fixes broken CI; bump macOS version to macos-11 (#20098)

* bump macOS image on Azure CI to macos-11
* fix CI error

(cherry picked from commit 8ef509b85b)

.builds/freebsd.yml

@@ -0,0 +1,31 @@
+# see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd
+image: freebsd/latest
+packages:
+- databases/sqlite3
+- devel/boehm-gc-threaded
+- devel/pcre
+- devel/sdl20
+- devel/sfml
+- www/node
+- devel/gmake
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpu)
+    bin/nim c --skipUserCfg --skipParentCfg koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_0.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "0_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_1.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "1_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_2.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "2_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.gitattributes

@@ -0,0 +1,6 @@
+# Avoids changelog conflicts by assuming additions-only, which is by far the common case.
+# In the rare case where branch b1 rebases against branch b2 and both branches
+# modified the same changelog entry, you'll end up with that changelog entry
+# duplicated, which is easily identifiable and fixable.
+/changelog.md merge=union
+

.github/FUNDING.yml

@@ -1,7 +1,7 @@
 # These are supported funding model platforms
 
 github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
+patreon: 'araq' # Replace with a single Patreon username
 open_collective: 'nim' # Replace with a single Open Collective username
 ko_fi: # Replace with a single Ko-fi username
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,46 +1,51 @@
 ---
 name: Bug report
 about: Have you found an unexpected behavior? Use this template.
+title: Think about the title, twice
+labels: ''
+assignees: ''
+
 ---
 
-<!-- Think about the title, twice. -->
-
-<!-- Summarize the problem here, keep it short and simple. -->
 Function `echo` outputs the wrong string.
 
-
 ### Example
-<!-- Paste your example in the code-block below. -->
 ```nim
 echo "Hello World!"
+# This code should be a minimum reproducible example:
+# try to simplify and minimize as much as possible. If it's a compiler
+# issue, try to minimize further by removing any imports if possible.
 ```
 
-
 ### Current Output
+please check whether the problem still exists in git head before posting,
+see [rebuilding the compiler](https://nim-lang.github.io/Nim/intern.html#rebuilding-the-compiler).
 ```
 Hola mundo!
 ```
 
-
 ### Expected Output
-<!-- What should be the correct output? -->
 ```
 Hello World!
 ```
 
-
 ### Possible Solution
-<!--- Have you found a possible solution? Post it here. -->
 
+* In file xyz there is a call that might be the cause of it.
 
 ### Additional Information
-<!--- For Example:
-* Your Nim version (output of `nim -v`).
-* Was it working in the previous Nim releases?
-* A link to a related issue or discussion.
-* A project reference where (and how) the issue causes problems.
--->
+If it's a regression, you can help us by identifying which version introduced
+the bug, see [Bisecting for regressions](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions),
+or at least try known past releases (eg `choosenim 1.2.0`).
+
+If it's a pre-existing compiler bug, see [Debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler)
+which should give more context on a compiler crash.
+
+* Issue #abc is related, but different because of ...
+* This issue is blocking my project xyz
+
 ```
 $ nim -v
 Nim Compiler Version 0.1.2
+# make sure to include the git hash if not using a tagged release
 ```

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,9 @@
 ---
 name: Feature request
 about: Do you want to suggest a new feature? Use this template.
+title: ''
+labels: ''
+assignees: ''
 
 ---
 

.github/stale.yml

@@ -0,0 +1,69 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 365
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 30
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - ARC
+  - bounty
+  - Codegen
+  - Crash
+  - Generics
+  - High Priority
+  - Macros
+  - Next release
+  - Showstopper
+  - Static[T]
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: stale
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This pull request has been automatically marked as stale because it has not had
+  recent activity.
+  If you think it is still a valid PR, please rebase it on the latest devel;
+  otherwise it will be closed. Thank you for your contributions.
+
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 20
+
+# Limit to only `issues` or `pulls`
+only: pulls
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.github/workflows/ci.yml.disabled

@@ -0,0 +1,122 @@
+name: Continous Integration
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64, i386]
+        cpp: ['false', 'true']
+        pkg: ['false', 'true']
+        exclude:
+          - os: ubuntu-18.04
+            cpp: 'true'
+          - os: ubuntu-18.04
+            pkg: 'true'
+          - os: macos-10.15
+            cpu: i386
+          - os: macos-10.15
+            pkg: 'true'
+          - os: windows-2019
+            cpu: i386
+          - os: windows-2019
+            cpp: 'true'
+    name: '${{ matrix.os }} (${{ matrix.cpu }}, cpp: ${{ matrix.cpp }}, pkg: ${{ matrix.pkg }})'
+    runs-on: ${{ matrix.os }}
+    env:
+      NIM_COMPILE_TO_CPP: ${{ matrix.cpp }}
+      NIM_TEST_PACKAGES: ${{ matrix.pkg }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install node.js 8.x'
+        uses: actions/setup-node@v1
+        with:
+          node-version: '8.x'
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -yq \
+              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
+              valgrind libc6-dbg
+      - name: 'Install dependencies (Linux i386)'
+        if: runner.os == 'Linux' && matrix.cpu == 'i386'
+        run: |
+          sudo dpkg --add-architecture i386
+
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+              g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
+              libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 \
+              libffi-dev:i386
+
+          cat << EOF > bin/gcc
+          #!/bin/bash
+
+          exec $(which gcc) -m32 "\$@"
+          EOF
+          cat << EOF > bin/g++
+          #!/bin/bash
+
+          exec $(which g++) -m32 "\$@"
+          EOF
+
+          chmod 755 bin/gcc
+          chmod 755 bin/g++
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install boehmgc make sfml
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+      - name: 'Run CI'
+        shell: bash
+        run: ./koch runCI
+
+      - name: 'Show failed tests'
+        if: failure()
+        shell: bash
+        run: nim c -r tools/ci_testresults.nim

.github/workflows/ci_docs.yml

@@ -0,0 +1,145 @@
+name: Nim Docs CI
+on:
+  push:
+    paths:
+      - 'compiler/docgen.nim'
+      - 'compiler/renderverbatim.nim'
+      - 'config/nimdoc.cfg'
+      - 'doc/**.rst'
+      - 'doc/nimdoc.css'
+      - 'lib/**.nim'
+      - 'nimdoc/testproject/expected/testproject.html'
+      - 'tools/dochack/dochack.nim'
+      - 'tools/kochdocs.nim'
+      - '.github/workflows/ci_docs.yml'
+      - 'koch.nim'
+  pull_request:
+    # Run only on changes on these files.
+    paths:
+      - 'compiler/docgen.nim'
+      - 'compiler/renderverbatim.nim'
+      - 'config/nimdoc.cfg'
+      - 'doc/**.rst'
+      - 'doc/nimdoc.css'
+      - 'lib/**.nim'
+      - 'nimdoc/testproject/expected/testproject.html'
+      - 'tools/dochack/dochack.nim'
+      - 'tools/kochdocs.nim'
+      - '.github/workflows/ci_docs.yml'
+
+
+jobs:
+  build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [linux, windows, osx]
+        include:
+          - target: linux
+            os: ubuntu-18.04
+          - target: windows
+            os: windows-2019
+          - target: osx
+            os: macos-11
+
+    name: ${{ matrix.target }}
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+
+      - name: 'Install build dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install make
+
+      - name: 'Install build dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Get current csources version'
+        id: csources-version
+        shell: bash
+        run: |
+          sha=$(git ls-remote https://github.com/nim-lang/csources_v1 master | cut -f 1)
+          echo "::set-output name=sha::$sha"
+
+      - name: 'Get prebuilt csources from cache'
+        id: csources-cache
+        uses: actions/cache@v1
+        with:
+          path: bin
+          key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}'
+
+      - name: 'Checkout csources'
+        if: steps.csources-cache.outputs.cache-hit != 'true'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources_v1
+          path: csources
+
+      - name: 'Build 1-stage compiler from csources'
+        shell: bash
+        run: |
+          ext=
+          [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe
+          if [[ ! -x bin/nim-csources$ext ]]; then
+            ncpu=
+            case '${{ runner.os }}' in
+            'Linux')
+              ncpu=$(nproc)
+              ;;
+            'macOS')
+              ncpu=$(sysctl -n hw.ncpu)
+              ;;
+            'Windows')
+              ncpu=$NUMBER_OF_PROCESSORS
+              ;;
+            esac
+            [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+            make -C csources -j $ncpu CC=gcc
+            cp bin/nim{,-csources}$ext
+          else
+            echo 'Cache hit, using prebuilt csources'
+            cp bin/nim{-csources,}$ext
+          fi
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build the real compiler'
+        shell: bash
+        run: ./koch boot -d:release
+
+      - name: 'Clone fusion'
+        shell: bash
+        run: ./koch fusion
+
+      - name: 'Build documentation'
+        shell: bash
+        run: ./koch doc --git.commit:devel
+
+      - name: 'Publish documentation to Github Pages'
+        if: |
+          github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
+          matrix.target == 'linux'
+        uses: crazy-max/ghaction-github-pages@v1
+        with:
+          build_dir: doc/html
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci_packages.yml

@@ -0,0 +1,85 @@
+name: Packages CI
+on: [push, pull_request]
+
+jobs:
+  build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, macos-11]
+        cpu: [amd64]
+        pkg: [1, 2]
+    name: '${{ matrix.os }} (pkg: ${{ matrix.pkg }})'
+    runs-on: ${{ matrix.os }}
+    env:
+      NIM_TEST_PACKAGES: ${{ matrix.pkg }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install node.js 12.x'
+        uses: actions/setup-node@v1
+        with:
+          node-version: '12.x'
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -yq \
+              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
+              valgrind libc6-dbg libblas-dev xorg-dev
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install boehmgc make sfml gtk+3
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+      - name: 'Run CI'
+        shell: bash
+        run: ./koch runCI
+
+      - name: 'Show failed tests'
+        if: failure()
+        shell: bash
+        run: nim c -r tools/ci_testresults.nim

.github/workflows/ci_ssl.yml

@@ -0,0 +1,93 @@
+name: Nim SSL CI
+on:
+  pull_request:
+    # Run only on changes on related files
+    paths:
+      - 'lib/pure/httpclient.nim'
+      - 'lib/pure/net.nim'
+      - 'lib/pure/ssl_certs.nim'
+      - 'lib/wrappers/openssl.nim'
+      - 'tests/stdlib/thttpclient_ssl*'
+      - 'tests/untestable/thttpclient_ssl*'
+
+jobs:
+  build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64]
+    name: '${{ matrix.os }} (${{ matrix.cpu }})'
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -y libssl1.1
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install make
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build 1-stage compiler from csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build the real compiler'
+        shell: bash
+        run: ./koch boot
+
+      - name: 'Run SSL nimDisableCertificateValidation integration tests'
+        shell: bash
+        run: nim c -d:nimDisableCertificateValidation -d:ssl -r -p:. tests/untestable/thttpclient_ssl_disabled.nim
+
+      - name: 'Run SSL certificate check integration tests'
+        # Not supported on Windows due to old openssl version
+        if: runner.os != 'Windows'
+        shell: bash
+        run: nim c -d:ssl -p:. --threads:on -r tests/untestable/thttpclient_ssl.nim

.gitignore

@@ -65,7 +65,8 @@ testament.db
 /tests/**/*.json
 /tests/**/*.js
 /csources
-dist/
+/dist/
+/lib/fusion
 
 # Private directories and files (IDEs)
 .*/
@@ -82,5 +83,20 @@ megatest.nim
 
 /outputExpected.txt
 /outputGotten.txt
+/t15148.txt
+/tests/vm/tfile_rw.txt
 
 /lib/pure/*.js
+
+!/.builds/
+
+!/.github
+
+# ignore debug dirs generated by dsymutil on OSX
+*.dSYM
+
+# for `nim c -r nimdoc/tester` etc; this can be in multiple places
+htmldocs
+
+## these are not needed anymore unless checkout old older versions
+nimdoc.out.css

.travis.yml

@@ -4,45 +4,12 @@ language: c
 dist: xenial
 
 matrix:
-  fast_finish: true
-
   include:
     - os: linux
       env:
         - NIM_COMPILE_TO_CPP=false
         - CPU=amd64
 
-    - os: linux
-      env:
-        - NIM_COMPILE_TO_CPP=false
-        - CPU=i386
-      addons:
-        apt:
-          packages:
-            - gcc-multilib
-            - g++-multilib
-            - libcurl4-openssl-dev:i386
-            - libgc-dev:i386
-            - libglib2.0-dev:i386
-            - libpulse-dev:i386
-            - libsdl1.2-dev:i386
-            - libsfml-dev:i386
-
-    - os: osx
-      env:
-        - NIM_COMPILE_TO_CPP=false
-        - CPU=amd64
-
-    - os: osx
-      env:
-        - NIM_COMPILE_TO_CPP=true
-        - CPU=amd64
-
-# To allow failures for a failing configuration, use something like:
-#  allow_failures:
-#    - env: NIM_COMPILE_TO_CPP=true
-#    - os: osx
-
 addons:
   apt:
     # update the list above if more deps are introduced
@@ -53,21 +20,9 @@ addons:
     - libsfml-dev
     - libc6-dbg
     - valgrind
-  homebrew:
-    packages:
-    - boehmgc
-    - make
-    - sfml
-    update: true
-
-install:
-  - if [[ $CPU = i386 ]]; then echo -e "#!/bin/bash\n$(which gcc) -m32 \"\$@\"" > bin/gcc; fi
-  - if [[ $CPU = i386 ]]; then chmod 755 bin/gcc; fi
-  - if [[ $CPU = i386 ]]; then echo -e "#!/bin/bash\n$(which g++) -m32 \"\$@\"" > bin/g++; fi
-  - if [[ $CPU = i386 ]]; then chmod 755 bin/g++; fi
 
 before_script:
-  - git clone --depth 1 https://github.com/nim-lang/csources.git
+  - git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
   - export PATH="$PWD/bin${PATH:+:$PATH}"
   - make -C csources -j 2 LD=$CC ucpu=$CPU
 
@@ -75,11 +30,17 @@ script:
   - echo "travis_fold:start:nim_c_koch"
   - nim c koch
   - echo "travis_fold:end:nim_c_koch"
-  - ./koch runCI
+  - echo "travis_fold:start:koch_boot"
+  - ./koch boot
+  - echo "travis_fold:end:koch_boot"
+  - echo "travis_fold:start:koch_doc"
+  - ./koch doc
+  - echo "travis_fold:end:koch_doc"
 
 before_deploy:
   # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html
   - cp -f ./doc/html/overview.html ./doc/html/index.html
+
 deploy:                         # https://nim-lang.github.io/Nim
   provider: pages
   # local-dir *has* to be a relative path from the repo root.
@@ -89,6 +50,3 @@ deploy:                         # https://nim-lang.github.io/Nim
   keep-history: false
   on:
     branch: devel
-
-# Extract failed tests
-after_failure: nim c -r tools/ci_testresults.nim

appveyor.yml

@@ -4,8 +4,8 @@ environment:
   DLLS_URL: https://nim-lang.org/download/dlls.zip
   DLLS_ARCHIVE: dlls.zip
   MINGW_DIR: mingw64
-  MINGW_URL: https://nim-lang.org/download/mingw64-6.3.0.7z
-  MINGW_ARCHIVE: mingw64-6.3.0.7z
+  MINGW_URL: https://nim-lang.org/download/mingw64.7z
+  MINGW_ARCHIVE: mingw64.7z
 
   matrix:
     - NIM_TEST_PACKAGES: false
@@ -29,6 +29,8 @@ install:
   - cd ..
 
 build_script:
+  - openssl version
+  - openssl version -d
   - bin\nim c koch
   - koch runCI
 

azure-pipelines.yml

@@ -1,148 +1,180 @@
-strategy:
-  matrix:
-    Linux_amd64:
-      vmImage: 'ubuntu-16.04'
-      CPU: amd64
-    Linux_i386:
-      vmImage: 'ubuntu-16.04'
-      CPU: i386
-    OSX_amd64:
-      vmImage: 'macOS-10.14'
-      CPU: amd64
-    OSX_amd64_cpp:
-      vmImage: 'macOS-10.14'
-      CPU: amd64
-      NIM_COMPILE_TO_CPP: true
-    Windows_amd64:
-      vmImage: 'windows-2019'
-      CPU: amd64
-    Windows_amd64_pkg:
-      vmImage: 'windows-2019'
-      CPU: amd64
-      NIM_TEST_PACKAGES: true
+trigger:
+  branches:
+    include:
+    - '*'
+pr:
+  branches:
+    include:
+    - '*'
 
-pool:
-  vmImage: $(vmImage)
+jobs:
+- job: packages
 
-workspace:
-  clean: all
+  timeoutInMinutes: 90 # default `60` led to lots of cancelled jobs; use 0 for unlimited (may be undesirable)
 
-steps:
-  - bash: git config --global core.autocrlf false
-    displayName: 'Disable auto conversion to CRLF by git (Windows-only)'
-    condition: eq(variables['Agent.OS'], 'Windows_NT')
+  strategy:
+    matrix:
+      Linux_amd64:
+        vmImage: 'ubuntu-18.04'
+        CPU: amd64
+      Linux_i386:
+        vmImage: 'ubuntu-18.04'
+        CPU: i386
+      OSX_amd64:
+        vmImage: 'macOS-11'
+        CPU: amd64
+      OSX_amd64_cpp:
+        vmImage: 'macOS-11'
+        CPU: amd64
+        NIM_COMPILE_TO_CPP: true
+      Windows_amd64_batch0_3:
+        vmImage: 'windows-2019'
+        CPU: amd64
+        # see also: `NIM_TEST_PACKAGES`
+        NIM_TESTAMENT_BATCH: "0_3"
+      Windows_amd64_batch1_3:
+        vmImage: 'windows-2019'
+        CPU: amd64
+        NIM_TESTAMENT_BATCH: "1_3"
+      Windows_amd64_batch2_3:
+        vmImage: 'windows-2019'
+        CPU: amd64
+        NIM_TESTAMENT_BATCH: "2_3"
 
-  - checkout: self
+  pool:
+    vmImage: $(vmImage)
 
-  - bash: git clone --depth 1 https://github.com/nim-lang/csources.git
-    displayName: 'Checkout csources'
+  workspace:
+    clean: all
 
-  - task: NodeTool@0
-    inputs:
-      versionSpec: '8.x'
-    displayName: 'Install node.js 8.x'
+  steps:
+    - bash: git config --global core.autocrlf false
+      displayName: 'Disable auto conversion to CRLF by git (Windows-only)'
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
 
-  - bash: |
-      sudo apt-fast update -qq
-      DEBIAN_FRONTEND='noninteractive' \
-        sudo apt-fast install --no-install-recommends -yq \
-          libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg
-    displayName: 'Install dependencies (amd64 Linux)'
-    condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
+    - checkout: self
+      fetchDepth: 1
 
-  - bash: |
-      sudo dpkg --add-architecture i386
+    - bash: git clone --depth 1 https://github.com/nim-lang/csources_v1 csources
+      displayName: 'Checkout Nim csources'
 
-      # Downgrade llvm, libgcc and libstdc++:
-      # - llvm has to be downgraded to have 32bit version installed for sfml.
-      # - libgcc and libstdc++ have to be downgraded as an optimization to
-      #   prevent the use of the toolchain ppa, which has a terrible download
-      #   speed.
-      cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
-      Package: libllvm6.0 libgcc1 libstdc++6
-      Pin: origin "azure.archive.ubuntu.com"
-      Pin-Priority: 1001
+    - task: NodeTool@0
+      inputs:
+        versionSpec: '12.x'
+      displayName: 'Install node.js 12.x'
 
-      Package: *
-      Pin: release o=LP-PPA-ubuntu-toolchain-r-test
-      Pin-Priority: 100
-      EOF
+    - bash: |
+        sudo apt-fast update -qq
+        DEBIAN_FRONTEND='noninteractive' \
+          sudo apt-fast install --no-install-recommends -yq \
+            libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg
+      displayName: 'Install dependencies (amd64 Linux)'
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
 
-      sudo apt-fast update -qq
-      DEBIAN_FRONTEND='noninteractive' \
-        sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
-          g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
-          libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386
+    - bash: |
+        sudo dpkg --add-architecture i386
 
-      cat << EOF > bin/gcc
-      #!/bin/bash
+        # Downgrade llvm:
+        # - llvm has to be downgraded to have 32bit version installed for sfml.
+        cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
+        Package: libllvm6.0
+        Pin: origin "azure.archive.ubuntu.com"
+        Pin-Priority: 1001
+        EOF
 
-      exec $(which gcc) -m32 "\$@"
-      EOF
-      cat << EOF > bin/g++
-      #!/bin/bash
+        sudo apt-fast update -qq
+        # `:i386` (e.g. in `libffi-dev:i386`) is needed otherwise you may get:
+        # `could not load: libffi.so` during dynamic loading.
+        DEBIAN_FRONTEND='noninteractive' \
+          sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+            g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
+            libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 libffi-dev:i386
 
-      exec $(which g++) -m32 "\$@"
-      EOF
+        cat << EOF > bin/gcc
+        #!/bin/bash
 
-      chmod 755 bin/gcc
-      chmod 755 bin/g++
+        exec $(which gcc) -m32 "\$@"
+        EOF
+        cat << EOF > bin/g++
+        #!/bin/bash
 
-    displayName: 'Install dependencies (i386 Linux)'
-    condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
+        exec $(which g++) -m32 "\$@"
+        EOF
 
-  - bash: brew install boehmgc make sfml
-    displayName: 'Install dependencies (OSX)'
-    condition: eq(variables['Agent.OS'], 'Darwin')
+        chmod 755 bin/gcc
+        chmod 755 bin/g++
 
-  - bash: |
-      mkdir dist
-      curl -L https://nim-lang.org/download/mingw64-6.3.0.7z -o dist/mingw64.7z
-      curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
-      7z x dist/mingw64.7z -odist
-      7z x dist/dlls.zip -obin
-      echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
+      displayName: 'Install dependencies (i386 Linux)'
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
 
-    displayName: 'Install dependencies (Windows)'
-    condition: eq(variables['Agent.OS'], 'Windows_NT')
+    - bash: brew install boehmgc make sfml
+      displayName: 'Install dependencies (OSX)'
+      condition: eq(variables['Agent.OS'], 'Darwin')
 
-  - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin'
-    displayName: 'Add build binaries to PATH'
+    - bash: |
+        mkdir dist
+        curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+        curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+        7z x dist/mingw64.7z -odist
+        7z x dist/dlls.zip -obin
+        echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
 
-  - bash: |
-      echo 'PATH:' "$PATH"
-      echo '##[section]gcc version'
-      gcc -v
-      echo '##[section]nodejs version'
-      node -v
-      echo '##[section]make version'
-      make -v
-    displayName: 'System information'
+      displayName: 'Install dependencies (Windows)'
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
 
-  - bash: |
-      ncpu=
-      case '$(Agent.OS)' in
-      'Linux')
-        ncpu=$(nproc)
-        ;;
-      'Darwin')
-        ncpu=$(sysctl -n hw.ncpu)
-        ;;
-      'Windows_NT')
-        ncpu=$NUMBER_OF_PROCESSORS
-        ;;
-      esac
-      [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+    - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin'
+      displayName: 'Add build binaries to PATH'
 
-      make -C csources -j $ncpu CC=gcc ucpu=$(CPU)
-    displayName: 'Build csources'
+    - bash: |
+        echo 'PATH:' "$PATH"
+        echo '##[section]gcc version'
+        gcc -v
+        echo '##[section]nodejs version'
+        node -v
+        echo '##[section]make version'
+        make -v
+      displayName: 'System information'
 
-  - bash: nim c koch
-    displayName: 'Build koch'
+    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
+      displayName: 'Get csources version'
 
-    # set result to omit the "bash exited with error code '1'" message
-  - bash: |
-      ./koch runCI || echo '##vso[task.complete result=Failed]'
-    displayName: 'Run CI'
-    env:
-      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+    - task: Cache@2
+      inputs:
+        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+        path: csources/bin
+      displayName: 'Restore built csources'
+
+    - bash: |
+        ncpu=
+        ext=
+        case '$(Agent.OS)' in
+        'Linux')
+          ncpu=$(nproc)
+          ;;
+        'Darwin')
+          ncpu=$(sysctl -n hw.ncpu)
+          ;;
+        'Windows_NT')
+          ncpu=$NUMBER_OF_PROCESSORS
+          ext=.exe
+          ;;
+        esac
+        [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+        if [[ -x csources/bin/nim$ext ]]; then
+          echo "Found cached compiler, skipping build"
+        else
+          make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
+        fi
+
+        cp csources/bin/nim$ext bin
+      displayName: 'Build 1-stage compiler from csources'
+
+    - bash: nim c koch
+      displayName: 'Build koch'
+
+      # set result to omit the "bash exited with error code '1'" message
+    - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
+      displayName: 'Run CI'
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)

bin/nim-gdb

@@ -5,9 +5,15 @@ set -e
 
 which nim > /dev/null || (echo "nim not in PATH"; exit 1)
 which gdb > /dev/null || (echo "gdb not in PATH"; exit 1)
+which readlink > /dev/null || (echo "readlink not in PATH."; exit 1)
+
+if [[ "$(uname -s)" == "Darwin" || "(uname -s)" == *"BSD" ]]; then
+  NIM_SYSROOT=$(dirname $(dirname $(readlink -f $(which nim))))
+else
+  NIM_SYSROOT=$(dirname $(dirname $(readlink -e $(which nim))))
+fi
 
 # Find out where the pretty printer Python module is
-NIM_SYSROOT=$(dirname $(dirname $(readlink -e $(which nim))))
 GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"
 
 # Run GDB with the additional arguments that load the pretty printers
@@ -16,6 +22,4 @@ GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"
 NIM_GDB="${NIM_GDB:-gdb}"
 # exec replaces the new process of bash with gdb. It is always good to
 # have fewer processes.
-exec ${NIM_GDB} \
-  -eval-command "source $GDB_PYTHON_MODULE_PATH" \
-  "$@"
+exec "${NIM_GDB}" -eval-command="source $GDB_PYTHON_MODULE_PATH" "$@"

bin/nim-gdb.bash

@@ -0,0 +1 @@
+nim-gdb

bin/nim-gdb.bat

@@ -0,0 +1,14 @@
+@echo off
+for %%i in (nim.exe) do (set NIM_BIN=%%~dp$PATH:i)
+
+for %%i in ("%NIM_BIN%\..\") do (set NIM_ROOT=%%~fi)
+
+set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\nim-gdb.py
+set @NIM_GDB=gdb.exe
+
+@echo source %@GDB_PYTHON_MODULE_PATH%> wingdbcommand.txt
+%@NIM_GDB% --command="wingdbcommand.txt" %*
+del wingdbcommand.txt /f /q
+
+EXIT /B %ERRORLEVEL%
+@echo on

build_all.bat

@@ -1,7 +1,7 @@
 @echo off
 rem build development version of the compiler; can be rerun safely
 if not exist csources (
-  git clone --depth 1 https://github.com/nim-lang/csources.git
+  git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 )
 if not exist bin\nim.exe (
   cd csources
@@ -12,5 +12,6 @@ if not exist bin\nim.exe (
   cd ..
 )
 bin\nim.exe c --skipUserCfg --skipParentCfg koch
-koch.exe boot -d:release
-koch.exe tools
+koch.exe boot -d:release --skipUserCfg --skipParentCfg
+koch.exe tools --skipUserCfg --skipParentCfg
+

build_all.sh

@@ -1,23 +1,41 @@
 #! /bin/sh
 
-# build development version of the compiler; can be rerun safely
+# build development version of the compiler; can be rerun safely.
+# arguments can be passed, e.g. `--os freebsd`
 
 set -u # error on undefined variables
 set -e # exit on first error
 
 echo_run(){
-  printf "\n$*\n"
+  echo "$*"
   "$@"
 }
 
-[ -d csources ] || echo_run git clone --depth 1 https://github.com/nim-lang/csources.git
+[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources_v1.git csources
 
 nim_csources=bin/nim_csources
+
+build_nim_csources_via_script(){
+  echo_run cd csources
+  echo_run sh build.sh "$@"
+}
+
 build_nim_csources(){
-  ## avoid changing dir in case of failure
+  # avoid changing dir in case of failure
   (
-    echo_run cd csources
-    echo_run sh build.sh $@
+    if [ $# -ne 0 ]; then
+      # some args were passed (e.g.: `--cpu i386`), need to call build.sh
+      build_nim_csources_via_script "$@"
+    else
+      # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s)
+      makeX=make
+      unamestr=$(uname)
+      if [ "$unamestr" = 'FreeBSD' ]; then
+        makeX=gmake
+      fi
+      nCPU=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
+      which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script
+    fi
   )
   # keep $nim_csources in case needed to investigate bootstrap issues
   # without having to rebuild from csources
@@ -29,5 +47,6 @@ build_nim_csources(){
 # Note: if fails, may need to `cd csources && git pull`
 echo_run bin/nim c --skipUserCfg --skipParentCfg koch
 
-echo_run ./koch boot -d:release
-echo_run ./koch tools # Compile Nimble and other tools.
+echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg
+echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools.
+

changelog.md

@@ -1,41 +1,83 @@
-# v1.1 - xxxx-xx-xx
-
-
-## Changes affecting backwards compatibility
+# v1.6.x - yyyy-mm-dd
 
 
 
-### Breaking changes in the standard library
+## Changes affecting backward compatibility
+
+- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected.
+  The downside is that these defines now have custom logic that doesn't apply for
+  other defines.
 
 
+## Standard library additions and changes
+- Added support for parenthesized expressions in `strformat`
 
-### Breaking changes in the compiler
+- Fixed buffer overflow bugs in `net`
+
+- Added `sections` iterator in `parsecfg`.
+
+- Make custom op in macros.quote work for all statements.
+
+- On Windows the SSL library now checks for valid certificates.
+  It uses the `cacert.pem` file for this purpose which was extracted
+  from `https://curl.se/ca/cacert.pem`. Besides
+  the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
+  now also need to ship `cacert.pem` with your `.exe` file.
 
 
+- Make `{.requiresInit.}` pragma to work for `distinct` types.
 
-## Library additions
+- Added a macros `enumLen` for returning the number of items in an enum to the
+  `typetraits.nim` module.
 
+- `prelude` now works with the JavaScript target.
 
+- Added `ioutils` module containing `duplicate` and `duplicateTo` to duplicate `FileHandle` using C function `dup` and `dup2`.
 
-## Library changes
+- The JSON module can now handle integer literals and floating point literals of arbitrary length and precision.
+  Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are kept as string literals and
+  one can use external BigNum libraries to handle these. The `parseFloat` family of functions also has now optional
+  `rawIntegers` and `rawFloats` parameters that can be used to enforce that all integer or float literals remain
+  in the "raw" string form so that client code can easily treat small and large numbers uniformly.
 
+- Added `randState` template that exposes the default random number generator. Useful for library authors.
 
+- Added `asyncdispatch.activeDescriptors` that returns the number of currently
+  active async event handles/file descriptors
 
-## Language additions
-
-
+- Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place.
 
 ## Language changes
 
+- `nimscript` now handles `except Exception as e`
 
-
-### Tool changes
+- The `cstring` doesn't support `[]=` operator in JS backend.
 
 
 
-### Compiler changes
+## Compiler changes
 
 
 
+- Added `unsafeIsolate` and `extract` to `std/isolation`.
+
+- `--hint:CC` now goes to stderr (like all other hints) instead of stdout.
+
+- json build instructions are now generated in `$nimcache/outFileBasename.json`
+  instead of `$nimcache/projectName.json`. This allows avoiding recompiling a given project
+  compiled with different options if the output file differs.
+
+- `--usenimcache` (implied by `nim r main`) now generates an output file that includes a hash of
+  some of the compilation options, which allows caching generated binaries:
+  nim r main # recompiles
+  nim r -d:foo main # recompiles
+  nim r main # uses cached binary
+  nim r main arg1 arg2 # ditto (runtime arguments are irrelevant)
+
+- The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch
+  enforces that every symbol is written as it was declared, not enforcing
+  the official Nim style guide. To be enabled, this has to be combined either
+  with `--styleCheck:error` or `--styleCheck:hint`.
+
+## Tool changes
 
-## Bugfixes

changelogs/changelog_1_0_2.md

@@ -43,7 +43,7 @@
 * threadpool: fix link in docs (#12258)
 * Fix spellings (#12277)
 * fix #12278, don't expose internal PCRE documentation
-* Fixed "Documentation of quitprocs is wrong" [#12279](https://github.com/nim-lang/Nim/issues/12279)
+* Fixed "Documentation of quitprocs is wrong" [#12279(https://github.com/nim-lang/Nim/issues/12279)
 * Fix typo in docs
 * Fix reference to parseSpec proc in readme
 * [doc/tut1] removed discard discussion in comments

changelogs/changelog_1_0_4.md

@@ -1,75 +0,0 @@
-# v1.0.4 - 2019-11-26
-
-
-
-## Language changes
-
-- Conversions to unsigned integers are unchecked at runtime, imitating earlier Nim
-  versions. The documentation was improved to acknowledge this special case.
-  See https://github.com/nim-lang/RFCs/issues/175 for more details. (#12688)
-
-- Add `or detectOs(Manjaro)` (#12587)
-
-- --os:ios needs to imply defined(macosx) (#12585)
-
-- export nim.cfg parser (#12602)
-
-
-
-## Bugfixes
-
-- Fixed "gorge(Ex) fails after a few calls on OSX"
-  ([#12337](https://github.com/nim-lang/Nim/issues/12337))
-- Fixed "Improve posix module"
-  ([#10723](https://github.com/nim-lang/Nim/issues/10723))
-- Fixed "nim check allows gorgeEx but doesn't allow writeFile"
-  ([#12491](https://github.com/nim-lang/Nim/issues/12491))
-- Fixed "Low level arithmetics bug: -3 mod 7 == 3"
-  ([#12514](https://github.com/nim-lang/Nim/issues/12514))
-- Fixed "Broken link to c2nim in backends.rst/.html"
-  ([#12537](https://github.com/nim-lang/Nim/issues/12537))
-- Fixed "generic argument with default value causes incorrect generic type resolution"
-  ([#12528](https://github.com/nim-lang/Nim/issues/12528))
-- Fixed "regression: compiler/vmgen.nim(354, 20) `false` leaking temporary 10 slotTempInt [AssertionError]
-  ([#12547](https://github.com/nim-lang/Nim/issues/12547))
-- Fixed "Compile/link broken for nim (64-bit only) on Windows"
-  ([#12536](https://github.com/nim-lang/Nim/issues/12536))
-- Fixed "No `=destroy` for elements of closure environments other than for latest devel --gc:destructors"
-  ([#12577](https://github.com/nim-lang/Nim/issues/12577))
-- Fixed "[1.0.0] Cannot compile anything with --cpu:avr"
-  ([#12395](https://github.com/nim-lang/Nim/issues/12395))
-- Fixed "Compiler crash with invalid object variant"
-  ([#12379](https://github.com/nim-lang/Nim/issues/12379))
-- Fixed "push pragma is silently ignored if position is in front of import statement"
-  ([#5050](https://github.com/nim-lang/Nim/issues/5050))
-- Fixed "Error with strformat + asyncdispatch + const"
-  ([#12612](https://github.com/nim-lang/Nim/issues/12612))
-- Fixed "The --nimblePath is additive; need a pain-free workaround"
-  ([#12601](https://github.com/nim-lang/Nim/issues/12601))
-- Fixed "nim.cfg syntax for --define:FOO:VAL undocumented or absent"
-  ([#12367](https://github.com/nim-lang/Nim/issues/12367))
-- Fixed "vm string literals do not work when generated by a macro"
-  ([#12670](https://github.com/nim-lang/Nim/issues/12670))
-- Fixed "StaticRead does force a recompile if static file changes."
-  ([#12663](https://github.com/nim-lang/Nim/issues/12663))
-- Fixed crash in terminate handler
-  ([#12572](https://github.com/nim-lang/Nim/pull/12572))
-- Fixed newLit for objects having string fields
-  ([#12542](https://github.com/nim-lang/Nim/pull/12542))
-
-
-
-## Documentation improvements
-
-- Documentation Math module (#12460)
-- Fix many broken links and prefer relative links within docs (#12463)
-- sequtils: replace deprecated 'random' call within example (#12515)
-- integer literal documentation (#12513)
-- Fix code style errors (#12545)
-- fix several typos in documentation and comments (#12553)
-- Add docs to better distinguish among getProjectPath, getCurrentDir and currentSourcePath (#12565)
-- doc/tut3.rst: Fix typo in Introduction (#12607)
-- Add links to packaging and distro pages (#12603)
-- fix documentation of `$`\*(dt: DateTime) (#12660)
-- Clarifies experimental / parallel example on manual.rst (#12472)
-- Fix wrong section hierarchy in the manual (#12724)

changelogs/changelog_1_0_6.md

@@ -1,68 +0,0 @@
-# v1.0.6 - 2020-01-24
-
-
-## Bugfixes
-
-### Fixed issues
-
-- Fixed "Nim stdlib style issues with --styleCheck:error"
-  ([#12687](https://github.com/nim-lang/Nim/issues/12687))
-- Fixed "new(ref MyObject) doesn't work at compile time"
-  ([#12488](https://github.com/nim-lang/Nim/issues/12488))
-- Fixed "Accessing the wrong variant field in the VM does not trigger the appropriate error message"
-  ([#11727](https://github.com/nim-lang/Nim/issues/11727))
-- Fixed "orderedTable.del() crashes with unitialized table"
-  ([#12798](https://github.com/nim-lang/Nim/issues/12798))
-- Fixed "semfold bug with negative value * 0"
-  ([#12783](https://github.com/nim-lang/Nim/issues/12783))
-- Fixed "nimsuggest `use` command does not return all instances of symbol"
-  ([#12832](https://github.com/nim-lang/Nim/issues/12832))
-- Fixed "Codegen ICE in allPathsAsgnResult"
-  ([#12827](https://github.com/nim-lang/Nim/issues/12827))
-- Fixed "Static[T] + syntactic error in the for loop = compiler crash"
-  ([#12148](https://github.com/nim-lang/Nim/issues/12148))
-- Fixed "Incorrect unused import warning involving templates and tables"
-  ([#12885](https://github.com/nim-lang/Nim/issues/12885))
-- Fixed "regression(1.0.4): undeclared identifier: 'readLines'; plus another regression and bug"
-  ([#13013](https://github.com/nim-lang/Nim/issues/13013))
-- Fixed "`nim doc` treats `export localSymbol` incorrectly"
-  ([#13100](https://github.com/nim-lang/Nim/issues/13100))
-- Fixed "symbols not defined in the grammar"
-  ([#10665](https://github.com/nim-lang/Nim/issues/10665))
-- Fixed "nim-gdb is missing from all released packages"
-  ([#13104](https://github.com/nim-lang/Nim/issues/13104))
-- Fixed "[JS] Move is not defined"
-  ([#9674](https://github.com/nim-lang/Nim/issues/9674))
-- Fixed "Error: usage of 'isNil' is a user-defined error"
-  ([#11440](https://github.com/nim-lang/Nim/issues/11440))
-
-
-### Other bugfixes
-
-- make addQuoted work on nimscript (#12717)
-- fix db_mysql getRow() when column is null error raised (#12806)
-- Fixed objects being erroneously zeroed out before object construction (#12814)
-- added cstrutils (#12858): fixed for 'csuCmpIgnoreStyle' error on hotcodereloading
-- Better clang_cl support (#12896)
-- fix cmdline bugs affecting nimBetterRun correctness (#12933)
-- fixes a bug that kept sugar.collect from working with for loop macros
-- Path substitution for --out and --outdir (#12796)
-- fix crash due to errant symbols in nim.cfg (#13073)
-- Allow `-o` option for `buildIndex` (#13037)
-- Deleted misplaced separator (#13085): Misplaced separator, which was constantly breaking compilation on Haiku OS, was deleted.
-- fixes an asyncftpclient bug; refs #13096
-- fix the ftp store function read the local file bug (#13108)
-- fix rtti sizeof for varargs in global scope (#13125)
-- Correctly remove a key from CountTable when it is set to zero.
-- fixes the distros.nim regression
-- fixes a critical times.nim bug reported on IRC
-- c_fflush() the rawWrite() buffer
-
-
-## Documentation improvements
-
-- Fixed "Documentation, Testament missing from "Tools available with Nim""
-  ([#12251](https://github.com/nim-lang/Nim/issues/12251))
-- Manual update: custom exceptions (#12847)
-- times/getClockStr(): fix mistake in doc
-- Fix typo and improve in code-block of 'lib/pure/parseutils.nim'

changelogs/changelog_1_2_0.md

@@ -0,0 +1,535 @@
+# v1.2.0 - 2020-04-02
+
+
+## Standard library additions and changes
+- Added overloaded `strformat.fmt` macro that use specified characters as
+  delimiter instead of '{' and '}'.
+- Added new procs in `tables.nim`: `OrderedTable.pop`, `CountTable.del`,
+  `CountTable.pop`, `Table.pop`.
+- Added `strtabs.clear` overload that reuses the existing mode.
+- Added `browsers.osOpen` const alias for the operating system specific *"open"* command.
+- Added `sugar.dup` for turning in-place algorithms like `sort` and `shuffle`
+  into operations that work on a copy of the data and return the mutated copy,
+  like the existing `sorted` does.
+- Added `sugar.collect` that does comprehension for seq/set/table collections.
+- Added `sugar.capture` for capturing some local loop variables when creating a
+  closure. This is an enhanced version of `closureScope`.
+- Added `typetraits.tupleLen` to get number of elements of a tuple/type tuple,
+  and `typetraits.get` to get the ith element of a type tuple.
+- Added `typetraits.genericParams` to return a tuple of generic params from a
+  generic instantiation.
+- `options` now treats `proc` like other pointer types, meaning `nil` proc variables
+  are converted to `None`.
+- Added `os.normalizePathEnd` for additional path sanitization.
+- Added `times.fromUnixFloat,toUnixFloat`, sub-second resolution versions of
+  `fromUnix`,`toUnixFloat`.
+- Added `wrapnils` module for chains of field-access and indexing where the LHS
+  can be nil. This simplifies code by reducing need for if-else branches around
+  intermediate maybe nil values. E.g. `echo ?.n.typ.kind`.
+- Added `minIndex`, `maxIndex` and `unzip` to the `sequtils` module.
+- Added `os.isRelativeTo` to tell whether a path is relative to another.
+- Added `resetOutputFormatters` to `unittest`.
+- Added `expectIdent` to the `macros` module.
+- Added `os.isValidFilename` that returns `true` if `filename` argument is valid
+  for cross-platform use.
+- Added `times.isLeapDay`.
+- `base64` adds URL-Safe Base64, implements RFC-4648 Section-7.
+- Added a new module, `std / compilesettings` for querying the compiler about
+  diverse configuration settings.
+- Added `net.getPeerCertificates` and `asyncnet.getPeerCertificates` for
+  retrieving the verified certificate chain of the peer we are connected to
+  through an SSL-wrapped `Socket`/`AsyncSocket`.
+- Added `browsers.openDefaultBrowser` without URL, implements IETF RFC-6694 Section-3.
+- Added `jsconsole.trace`, `jsconsole.table`, `jsconsole.exception` for JavaScript target.
+- Added `distinctBase` overload for values: `assert 12.MyInt.distinctBase == 12`
+- Added new module `std/stackframes`, in particular `setFrameMsg`, which enables
+  custom runtime annotation of stackframes, see #13351 for examples.
+  Turn on/off via `--stackTraceMsgs:on/off`.
+- Added `sequtils.countIt`, allowing for counting items using a predicate.
+- Added a `with` macro for easy function chaining that's available everywhere,
+  there is no need to concern your APIs with returning the first argument
+  to enable "chaining", instead use the dedicated macro `with` that
+  was designed for it. For example:
+
+```nim
+import std/with
+
+type
+  Foo = object
+    col, pos: string
+
+proc setColor(f: var Foo; r, g, b: int) = f.col = $(r, g, b)
+proc setPosition(f: var Foo; x, y: float) = f.pos = $(x, y)
+
+var f: Foo
+with(f, setColor(2, 3, 4), setPosition(0.0, 1.0))
+echo f
+```
+
+- `macros.newLit` now works for ref object types.
+- `macro pragmas` can now be used in type sections.
+- 5 new pragmas were added to Nim in order to make the upcoming tooling more
+  convenient to use. Nim compiler checks these pragmas for syntax but otherwise
+  ignores them. The pragmas are `requires`, `ensures`, `assume`, `assert`, `invariant`.
+- `system.writeFile` has been overloaded to also support `openarray[byte]`.
+- `asyncdispatch.drain` now properly takes into account `selector.hasPendingOperations`
+  and only returns once all pending async operations are guaranteed to have completed.
+- `sequtils.zip` now returns a sequence of anonymous tuples i.e. those tuples
+  now do not have fields named "a" and "b".
+- `distinctBase` has been moved from `sugar` to `typetraits` and now it is
+  implemented as compiler type trait instead of macro. `distinctBase` in sugar
+  module is now deprecated.
+- `CountTable.mget` has been removed from `tables.nim`. It didn't work, and it
+  was an oversight to be included in v1.0.
+- `tables.merge(CountTable, CountTable): CountTable` has been removed.
+  It didn't work well together with the existing inplace version of the same proc
+  (`tables.merge(var CountTable, CountTable)`).
+  It was an oversight to be included in v1.0.
+- `asyncdispatch.drain` now consistently uses the passed timeout value for all
+  iterations of the event loop, and not just the first iteration.
+  This is more consistent with the other asyncdispatch APIs, and allows
+  `asyncdispatch.drain` to be more efficient.
+- `base64.encode` and `base64.decode` were made faster by about 50%.
+- `htmlgen` adds [MathML](https://wikipedia.org/wiki/MathML) support
+  (ISO 40314).
+- `macros.eqIdent` is now invariant to export markers and backtick quotes.
+- `htmlgen.html` allows `lang` in the `<html>` tag and common valid attributes.
+- `macros.basename` and `basename=` got support for `PragmaExpr`,
+  so that an expression like `MyEnum {.pure.}` is handled correctly.
+- `httpclient.maxredirects` changed from `int` to `Natural`, because negative values
+  serve no purpose whatsoever.
+- `httpclient.newHttpClient` and `httpclient.newAsyncHttpClient` added `headers`
+  argument to set initial HTTP Headers, instead of a hardcoded empty `newHttpHeader()`.
+- `parseutils.parseUntil` has now a different behaviour if the `until` parameter is
+  empty. This was required for intuitive behaviour of the strscans module
+  (see bug #13605).
+- `strutils.formatFloat` with `precision = 0` has the same behavior in all
+  backends, and it is compatible with Python's behavior,
+  e.g. `formatFloat(3.14159, precision = 0)` is now `3`, not `3.`.
+- `times.parse` now only uses input to compute its result, and not `now`:
+  `parse("2020", "YYYY", utc())` is now `2020-01-01T00:00:00Z` instead of
+  `2020-03-02T00:00:00Z` if run on 03-02; it also doesn't crash anymore when
+  used on 29th, 30th, 31st of each month.
+- `httpcore.==(string, HttpCode)` is now deprecated due to lack of practical
+  usage. The `$` operator can be used to obtain the string form of `HttpCode`
+  for comparison if desired.
+- `std/oswalkdir` was buggy, it's now deprecated and reuses `std/os` procs.
+- `os.walkDir` and `os.walkDirRec` now have new flag, `checkDir` (default: false).
+  If it is set to true, it will throw if input dir is invalid instead of a noop
+  (which is the default behaviour, as it was before this change),
+  `os.walkDirRec` only throws if top-level dir is invalid, but ignores errors for
+  subdirs, otherwise it would be impossible to resume iteration.
+- The `FD` variant of `selector.unregister` for `ioselector_epoll` and
+  `ioselector_select` now properly handle the `Event.User` select event type.
+- `joinPath` path normalization when `/` is the first argument works correctly:
+  `assert "/" / "/a" == "/a"`. Fixed the edge case: `assert "" / "" == ""`.
+- `xmltree` now adds indentation consistently to child nodes for any number
+  of children nodes.
+- `os.splitPath()` behavior synchronized with `os.splitFile()` to return "/"
+  as the dir component of `/root_sub_dir` instead of the empty string.
+- The deprecated `lc` macro has been removed from `sugar`. It is now replaced
+  with the more powerful `collect` macro.
+- `os.relativePath("foo", "foo")` is now `"."`, not `""`, as `""` means invalid
+  path and shouldn't be conflated with `"."`; use `-d:nimOldRelativePathBehavior`
+  to restore the old behavior.
+- `os.joinPath(a, b)` now honors trailing slashes in `b` (or `a` if `b` = "").
+- `base64.encode` no longer supports `lineLen` and `newLine`.
+  Use `base64.encodeMime` instead.
+
+
+### Breaking changes
+
+- `net.newContext` now performs SSL Certificate checking on Linux and OSX.
+  Define `nimDisableCertificateValidation` to disable it globally.
+
+
+
+## Language changes
+
+- An `align` pragma can now be used for variables and object fields, similar
+  to the `alignas` declaration modifier in C/C++.
+- The `=sink` type bound operator is now optional. The compiler can now use a
+  combination of `=destroy` and `copyMem` to move objects efficiently.
+- Unsigned integer operators have been fixed to allow promotion of the first operand.
+- Conversions to unsigned integers are unchecked at runtime, imitating earlier Nim
+  versions. The documentation was improved to acknowledge this special case.
+  See https://github.com/nim-lang/RFCs/issues/175 for more details.
+- There is a new syntax for lvalue references: `var b {.byaddr.} = expr` enabled by
+  `import std/decls`.
+- `var a {.foo.}: MyType = expr` now lowers to `foo(a, MyType, expr)` for
+  non-builtin pragmas, enabling things like lvalue references (see `decls.byaddr`).
+
+
+
+## Compiler changes
+
+- The generated JS code uses spaces, instead of mixing spaces and tabs.
+- The Nim compiler now supports the ``--asm`` command option for easier
+  inspection of the produced assembler code.
+- The Nim compiler now supports a new pragma called ``.localPassc`` to
+  pass specific compiler options to the C(++) backend for the C(++) file
+  that was produced from the current Nim module.
+- The compiler now inferes "sink parameters". To disable this for a specific routine,
+  annotate it with `.nosinks`. To disable it for a section of code, use
+  `{.push sinkInference: off.}`...`{.pop.}`.
+- The compiler now supports a new switch `--panics:on` that turns runtime
+  errors like `IndexError` or `OverflowError` into fatal errors that **cannot**
+  be caught via Nim's `try` statement. `--panics:on` can improve the
+  runtime efficiency and code size of your program significantly.
+- The compiler now warns about inheriting directly from `system.Exception` as
+  this is **very bad** style. You should inherit from `ValueError`, `IOError`,
+  `OSError` or from a different specific exception type that inherits from
+  `CatchableError` and cannot be confused with a `Defect`.
+- The error reporting for Nim's effect system has been improved.
+- Implicit conversions for `const` behave correctly now, meaning that code like
+  `const SOMECONST = 0.int; procThatTakesInt32(SOMECONST)` will be illegal now.
+  Simply write `const SOMECONST = 0` instead.
+- The `{.dynlib.}` pragma is now required for exporting symbols when making
+  shared objects on POSIX and macOS, which make it consistent with the behavior
+  on Windows.
+- The compiler is now more strict about type conversions concerning proc
+  types: Type conversions cannot be used to hide `.raise` effects or side
+  effects, instead a `cast` must be used. With the flag `--useVersion:1.0` the
+  old behaviour is emulated.
+- The Nim compiler now implements a faster way to detect overflows based
+  on GCC's `__builtin_sadd_overflow` family of functions. (Clang also
+  supports these). Some versions of GCC lack this feature and unfortunately
+  we cannot detect this case reliably. So if you get compilation errors like
+  "undefined reference to `__builtin_saddll_overflow`" compile your programs
+  with `-d:nimEmulateOverflowChecks`.
+
+
+
+
+## Bugfixes
+
+- Fixed "`nimgrep --nocolor` is ignored on posix; should be instead: `--nimgrep --color=[auto]|true|false`"
+  ([#7591](https://github.com/nim-lang/Nim/issues/7591))
+- Fixed "Runtime index on const array (of converted obj) causes C-compiler error"
+  ([#10514](https://github.com/nim-lang/Nim/issues/10514))
+- Fixed "windows x86 with vcc compile error with "asmNoStackFrame""
+  ([#12298](https://github.com/nim-lang/Nim/issues/12298))
+- Fixed "[TODO] regression: Error: Locks requires --threads:on option"
+  ([#12330](https://github.com/nim-lang/Nim/issues/12330))
+- Fixed "Add --cc option to --help or --fullhelp output"
+  ([#12010](https://github.com/nim-lang/Nim/issues/12010))
+- Fixed "questionable `csize` definition in `system.nim`"
+  ([#12187](https://github.com/nim-lang/Nim/issues/12187))
+- Fixed "os.getAppFilename() returns incorrect results on OpenBSD"
+  ([#12389](https://github.com/nim-lang/Nim/issues/12389))
+- Fixed "HashSet[uint64] slow insertion depending on values"
+  ([#11764](https://github.com/nim-lang/Nim/issues/11764))
+- Fixed "Differences between compiling 'classic call syntax' vs 'method call syntax' ."
+  ([#12453](https://github.com/nim-lang/Nim/issues/12453))
+- Fixed "c -d:nodejs --> SIGSEGV: Illegal storage access"
+  ([#12502](https://github.com/nim-lang/Nim/issues/12502))
+- Fixed "Closure iterator crashes on --newruntime due to "dangling references""
+  ([#12443](https://github.com/nim-lang/Nim/issues/12443))
+- Fixed "No `=destroy` for elements of closure environments other than for latest devel --gc:destructors"
+  ([#12577](https://github.com/nim-lang/Nim/issues/12577))
+- Fixed "strutils:formatBiggestFloat() gives different results in JS mode"
+  ([#8242](https://github.com/nim-lang/Nim/issues/8242))
+- Fixed "Regression (devel): the new `csize_t` definition isn't consistently used, nor tested thoroughly..."
+  ([#12597](https://github.com/nim-lang/Nim/issues/12597))
+- Fixed "tables.take() is defined only for `Table` and missed for other table containers"
+  ([#12519](https://github.com/nim-lang/Nim/issues/12519))
+- Fixed "`pthread_key_t` errors on OpenBSD"
+  ([#12135](https://github.com/nim-lang/Nim/issues/12135))
+- Fixed "newruntime: simple seq pop at ct results in compile error"
+  ([#12644](https://github.com/nim-lang/Nim/issues/12644))
+- Fixed "[Windows] finish.exe C:\Users\<USERNAME>\.nimble\bin is not in your PATH environment variable."
+  ([#12319](https://github.com/nim-lang/Nim/issues/12319))
+- Fixed "Error with strformat + asyncdispatch + const"
+  ([#12612](https://github.com/nim-lang/Nim/issues/12612))
+- Fixed "MultipartData needs $"
+  ([#11863](https://github.com/nim-lang/Nim/issues/11863))
+- Fixed "Nim stdlib style issues with --styleCheck:error"
+  ([#12687](https://github.com/nim-lang/Nim/issues/12687))
+- Fixed "new $nimbleDir path substitution yields unexpected search paths"
+  ([#12767](https://github.com/nim-lang/Nim/issues/12767))
+- Fixed "Regression: inlined procs now get multiple rounds of destructor injection"
+  ([#12766](https://github.com/nim-lang/Nim/issues/12766))
+- Fixed "newruntime: compiler generates defective code"
+  ([#12669](https://github.com/nim-lang/Nim/issues/12669))
+- Fixed "broken windows modules path handling because of 'os.relativePath' breaking changes"
+  ([#12734](https://github.com/nim-lang/Nim/issues/12734))
+- Fixed "for loop tuple syntax not rendered correctly"
+  ([#12740](https://github.com/nim-lang/Nim/issues/12740))
+- Fixed "Crash when trying to use `type.name[0]`"
+  ([#12804](https://github.com/nim-lang/Nim/issues/12804))
+- Fixed "Enums should be considered Trivial types in Atomics"
+  ([#12812](https://github.com/nim-lang/Nim/issues/12812))
+- Fixed "Produce static/const initializations for variables when possible"
+  ([#12216](https://github.com/nim-lang/Nim/issues/12216))
+- Fixed "Assigning descriminator field leads to internal assert with --gc:destructors"
+  ([#12821](https://github.com/nim-lang/Nim/issues/12821))
+- Fixed "nimsuggest `use` command does not return all instances of symbol"
+  ([#12832](https://github.com/nim-lang/Nim/issues/12832))
+- Fixed "@[] is a problem for --gc:destructors"
+  ([#12820](https://github.com/nim-lang/Nim/issues/12820))
+- Fixed "Codegen ICE in allPathsAsgnResult"
+  ([#12827](https://github.com/nim-lang/Nim/issues/12827))
+- Fixed "seq[Object with ref and destructor type] doesn't work in old runtime"
+  ([#12882](https://github.com/nim-lang/Nim/issues/12882))
+- Fixed "Destructor not invoked because it is instantiated too late, old runtime"
+  ([#12883](https://github.com/nim-lang/Nim/issues/12883))
+- Fixed "The collect macro does not handle if/case correctly"
+  ([#12874](https://github.com/nim-lang/Nim/issues/12874))
+- Fixed "allow typed/untyped params in magic procs (even if not in stdlib)"
+  ([#12911](https://github.com/nim-lang/Nim/issues/12911))
+- Fixed "ARC/newruntime memory corruption"
+  ([#12899](https://github.com/nim-lang/Nim/issues/12899))
+- Fixed "tasyncclosestall.nim still flaky test: Address already in use"
+  ([#12919](https://github.com/nim-lang/Nim/issues/12919))
+- Fixed "newruntime and computed goto: variables inside the loop are in generated code uninitialised"
+  ([#12785](https://github.com/nim-lang/Nim/issues/12785))
+- Fixed "osx: dsymutil needs to be called for debug builds to keep debug info"
+  ([#12735](https://github.com/nim-lang/Nim/issues/12735))
+- Fixed "codegen ICE with ref objects, gc:destructors"
+  ([#12826](https://github.com/nim-lang/Nim/issues/12826))
+- Fixed "mutable iterator cannot yield named tuples"
+  ([#12945](https://github.com/nim-lang/Nim/issues/12945))
+- Fixed "parsecfg stores "\r\n" line breaks just as "\n""
+  ([#12970](https://github.com/nim-lang/Nim/issues/12970))
+- Fixed "db_postgres.getValue issues warning when no rows found"
+  ([#12973](https://github.com/nim-lang/Nim/issues/12973))
+- Fixed "ARC: Unpacking tuple with seq causes segfault"
+  ([#12989](https://github.com/nim-lang/Nim/issues/12989))
+- Fixed "ARC/newruntime: strutils.join on seq with only empty strings causes segfault"
+  ([#12965](https://github.com/nim-lang/Nim/issues/12965))
+- Fixed "regression (1.0.4): `{.push exportc.}` wrongly affects generic instantiations, causing codegen errors"
+  ([#12985](https://github.com/nim-lang/Nim/issues/12985))
+- Fixed "cdt, crash with --gc:arc, works fine with default gc"
+  ([#12978](https://github.com/nim-lang/Nim/issues/12978))
+- Fixed "ARC: No indexError thrown on out-of-bound seq access, SIGSEGV instead"
+  ([#12961](https://github.com/nim-lang/Nim/issues/12961))
+- Fixed "ARC/async: Returning in a try-block results in wrong codegen"
+  ([#12956](https://github.com/nim-lang/Nim/issues/12956))
+- Fixed "asm keyword is generating wrong output C code when --cc:tcc"
+  ([#12988](https://github.com/nim-lang/Nim/issues/12988))
+- Fixed "Destructor not invoked"
+  ([#13026](https://github.com/nim-lang/Nim/issues/13026))
+- Fixed "ARC/newruntime: Adding inherited var ref object to seq with base type causes segfault"
+  ([#12964](https://github.com/nim-lang/Nim/issues/12964))
+- Fixed "Style check error with JS compiler target"
+  ([#13032](https://github.com/nim-lang/Nim/issues/13032))
+- Fixed "regression(1.0.4): undeclared identifier: 'readLines'; plus another regression and bug"
+  ([#13013](https://github.com/nim-lang/Nim/issues/13013))
+- Fixed "regression(1.04) `invalid pragma: since` with nim js"
+  ([#12996](https://github.com/nim-lang/Nim/issues/12996))
+- Fixed "Sink to MemMove optimization in injectdestructors"
+  ([#13002](https://github.com/nim-lang/Nim/issues/13002))
+- Fixed "--gc:arc: `catch` doesn't work with exception subclassing"
+  ([#13072](https://github.com/nim-lang/Nim/issues/13072))
+- Fixed "nim c --gc:arc --exceptions:{setjmp,goto} incorrectly handles raise; `nim cpp --gc:arc` is ok"
+  ([#13070](https://github.com/nim-lang/Nim/issues/13070))
+- Fixed "typetraits feature request - get subtype of a generic type"
+  ([#6454](https://github.com/nim-lang/Nim/issues/6454))
+- Fixed "CountTable inconsistencies between keys() and len() after setting value to 0"
+  ([#12813](https://github.com/nim-lang/Nim/issues/12813))
+- Fixed "{.align.} pragma is not applied if there is a generic field"
+  ([#13122](https://github.com/nim-lang/Nim/issues/13122))
+- Fixed "ARC, finalizer, allow rebinding the same function multiple times"
+  ([#13112](https://github.com/nim-lang/Nim/issues/13112))
+- Fixed "`nim doc` treats `export localSymbol` incorrectly"
+  ([#13100](https://github.com/nim-lang/Nim/issues/13100))
+- Fixed "--gc:arc SIGSEGV (double free?)"
+  ([#13119](https://github.com/nim-lang/Nim/issues/13119))
+- Fixed "codegen bug with arc"
+  ([#13105](https://github.com/nim-lang/Nim/issues/13105))
+- Fixed "symbols not defined in the grammar"
+  ([#10665](https://github.com/nim-lang/Nim/issues/10665))
+- Fixed "[JS] Move is not defined"
+  ([#9674](https://github.com/nim-lang/Nim/issues/9674))
+- Fixed "[TODO] pathutils.`/` can return invalid AbsoluteFile"
+  ([#13121](https://github.com/nim-lang/Nim/issues/13121))
+- Fixed "regression(1.04) `nim doc main.nim` generates broken html (no css)"
+  ([#12998](https://github.com/nim-lang/Nim/issues/12998))
+- Fixed "Wrong supportsCopyMem on string in type section"
+  ([#13095](https://github.com/nim-lang/Nim/issues/13095))
+- Fixed "Arc, finalizer, out of memory"
+  ([#13157](https://github.com/nim-lang/Nim/issues/13157))
+- Fixed "`--genscript` messes up nimcache and future nim invocations"
+  ([#13144](https://github.com/nim-lang/Nim/issues/13144))
+- Fixed "--gc:arc with --exceptions:goto for "nim c" generate invalid c code"
+  ([#13186](https://github.com/nim-lang/Nim/issues/13186))
+- Fixed "[regression] duplicate member `_i1` codegen bug"
+  ([#13195](https://github.com/nim-lang/Nim/issues/13195))
+- Fixed "RTree investigations with valgrind for --gc:arc"
+  ([#13110](https://github.com/nim-lang/Nim/issues/13110))
+- Fixed "relativePath("foo", ".") returns wrong path"
+  ([#13211](https://github.com/nim-lang/Nim/issues/13211))
+- Fixed "asyncftpclient - problem with welcome.msg"
+  ([#4684](https://github.com/nim-lang/Nim/issues/4684))
+- Fixed "Unclear error message, lowest priority"
+  ([#13256](https://github.com/nim-lang/Nim/issues/13256))
+- Fixed "Channel messages are corrupted"
+  ([#13219](https://github.com/nim-lang/Nim/issues/13219))
+- Fixed "Codegen bug with exportc and case objects"
+  ([#13281](https://github.com/nim-lang/Nim/issues/13281))
+- Fixed "[bugfix] fix #11590: c compiler warnings silently ignored, giving undefined behavior"
+  ([#11591](https://github.com/nim-lang/Nim/issues/11591))
+- Fixed "[CI] tnetdial flaky test"
+  ([#13132](https://github.com/nim-lang/Nim/issues/13132))
+- Fixed "Cross-Compiling with -d:mingw fails to locate compiler under OSX"
+  ([#10717](https://github.com/nim-lang/Nim/issues/10717))
+- Fixed "`nim doc --project` broken with imports below main project file or duplicate names"
+  ([#13150](https://github.com/nim-lang/Nim/issues/13150))
+- Fixed "regression: isNamedTuple(MyGenericTuple[int]) is false, should be true"
+  ([#13349](https://github.com/nim-lang/Nim/issues/13349))
+- Fixed "--gc:arc codegen bug copying objects bound to C structs with missing C struct fields"
+  ([#13269](https://github.com/nim-lang/Nim/issues/13269))
+- Fixed "write requires conversion to string"
+  ([#13182](https://github.com/nim-lang/Nim/issues/13182))
+- Fixed "Some remarks to stdlib documentation"
+  ([#13352](https://github.com/nim-lang/Nim/issues/13352))
+- Fixed "a `check` in unittest generated by template doesn't show actual value"
+  ([#6736](https://github.com/nim-lang/Nim/issues/6736))
+- Fixed "Implicit return with case expression fails with 'var' return."
+  ([#3339](https://github.com/nim-lang/Nim/issues/3339))
+- Fixed "Segfault with closure on arc"
+  ([#13314](https://github.com/nim-lang/Nim/issues/13314))
+- Fixed "[Macro] Crash on malformed case statement with multiple else"
+  ([#13255](https://github.com/nim-lang/Nim/issues/13255))
+- Fixed "regression: `echo 'discard' | nim c -r -` generates a file '-' ; `-` should be treated specially"
+  ([#13374](https://github.com/nim-lang/Nim/issues/13374))
+- Fixed "on OSX, debugging (w gdb or lldb) a nim program crashes at the 1st call to `execCmdEx`"
+  ([#9634](https://github.com/nim-lang/Nim/issues/9634))
+- Fixed "Internal error in getTypeDescAux"
+  ([#13378](https://github.com/nim-lang/Nim/issues/13378))
+- Fixed "gc:arc mode breaks tuple let"
+  ([#13368](https://github.com/nim-lang/Nim/issues/13368))
+- Fixed "Nim compiler hangs for certain C/C++ compiler errors"
+  ([#8648](https://github.com/nim-lang/Nim/issues/8648))
+- Fixed "htmlgen does not support `data-*` attributes"
+  ([#13444](https://github.com/nim-lang/Nim/issues/13444))
+- Fixed "[gc:arc] setLen will cause string not to be null-terminated."
+  ([#13457](https://github.com/nim-lang/Nim/issues/13457))
+- Fixed "joinPath("", "") is "/" ; should be """
+  ([#13455](https://github.com/nim-lang/Nim/issues/13455))
+- Fixed "[CI] flaky test on windows: tests/osproc/texitcode.nim"
+  ([#13449](https://github.com/nim-lang/Nim/issues/13449))
+- Fixed "Casting to float32 on NimVM is broken"
+  ([#13479](https://github.com/nim-lang/Nim/issues/13479))
+- Fixed "`--hints:off` doesn't work (doesn't override ~/.config/nim.cfg)"
+  ([#8312](https://github.com/nim-lang/Nim/issues/8312))
+- Fixed "joinPath("", "") is "/" ; should be """
+  ([#13455](https://github.com/nim-lang/Nim/issues/13455))
+- Fixed "tables.values is broken"
+  ([#13496](https://github.com/nim-lang/Nim/issues/13496))
+- Fixed "global user config can override project specific config"
+  ([#9405](https://github.com/nim-lang/Nim/issues/9405))
+- Fixed "Non deterministic macros and id consistency problem"
+  ([#12627](https://github.com/nim-lang/Nim/issues/12627))
+- Fixed "try expression doesn't work with return on expect branch"
+  ([#13490](https://github.com/nim-lang/Nim/issues/13490))
+- Fixed "CI will break every 4 years on feb 28: times doesn't handle leap years properly"
+  ([#13543](https://github.com/nim-lang/Nim/issues/13543))
+- Fixed "[minor] `nimgrep --word` doesn't work with operators (eg misses  `1 +% 2`)"
+  ([#13528](https://github.com/nim-lang/Nim/issues/13528))
+- Fixed "`as` is usable as infix operator but its existence and precedence are not documented"
+  ([#13409](https://github.com/nim-lang/Nim/issues/13409))
+- Fixed "JSON unmarshalling drops seq's items"
+  ([#13531](https://github.com/nim-lang/Nim/issues/13531))
+- Fixed "os.joinPath returns wrong path when head ends '\' or '/' and tail starts '..'."
+  ([#13579](https://github.com/nim-lang/Nim/issues/13579))
+- Fixed "Block-local types with the same name lead to bad codegen (sighashes regression)"
+  ([#5170](https://github.com/nim-lang/Nim/issues/5170))
+- Fixed "tuple codegen error"
+  ([#12704](https://github.com/nim-lang/Nim/issues/12704))
+- Fixed "newHttpHeaders does not accept repeated headers"
+  ([#13573](https://github.com/nim-lang/Nim/issues/13573))
+- Fixed "regression: --incremental:on fails on simplest example"
+  ([#13319](https://github.com/nim-lang/Nim/issues/13319))
+- Fixed "strscan can't get value of last element in format"
+  ([#13605](https://github.com/nim-lang/Nim/issues/13605))
+- Fixed "hashes_examples crashes with "Bus Error" (unaligned access) on sparc64"
+  ([#12508](https://github.com/nim-lang/Nim/issues/12508))
+- Fixed "gc:arc bug with re-used `seq[T]`"
+  ([#13596](https://github.com/nim-lang/Nim/issues/13596))
+- Fixed "`raise CatchableError` is broken with --gc:arc  when throwing inside a proc"
+  ([#13599](https://github.com/nim-lang/Nim/issues/13599))
+- Fixed "cpp --gc:arc --exceptions:goto fails to raise with discard"
+  ([#13436](https://github.com/nim-lang/Nim/issues/13436))
+- Fixed "terminal doesn't compile with -d:useWinAnsi"
+  ([#13607](https://github.com/nim-lang/Nim/issues/13607))
+- Fixed "Parsing "sink ptr T" - region needs to be an object type"
+  ([#12757](https://github.com/nim-lang/Nim/issues/12757))
+- Fixed "gc:arc + threads:on + closures compilation error"
+  ([#13519](https://github.com/nim-lang/Nim/issues/13519))
+- Fixed "[ARC] segmentation fault"
+  ([#13240](https://github.com/nim-lang/Nim/issues/13240))
+- Fixed "times.toDateTime buggy on 29th, 30th and 31th of each month"
+  ([#13558](https://github.com/nim-lang/Nim/issues/13558))
+- Fixed "Deque misbehaves on VM"
+  ([#13310](https://github.com/nim-lang/Nim/issues/13310))
+- Fixed "Nimscript listFiles should throw exception when path is not found"
+  ([#12676](https://github.com/nim-lang/Nim/issues/12676))
+- Fixed "koch boot fails if even an empty config.nims is present in ~/.config/nims/ [devel regression]"
+  ([#13633](https://github.com/nim-lang/Nim/issues/13633))
+- Fixed "nim doc generates lots of false positive LockLevel warnings"
+  ([#13218](https://github.com/nim-lang/Nim/issues/13218))
+- Fixed "Arrays are passed by copy to iterators, causing crashes, unnecessary allocations and slowdowns"
+  ([#12747](https://github.com/nim-lang/Nim/issues/12747))
+- Fixed "Range types always uses signed integer as a base type"
+  ([#13646](https://github.com/nim-lang/Nim/issues/13646))
+- Fixed "Generate c code cannot compile with recent devel version"
+  ([#13645](https://github.com/nim-lang/Nim/issues/13645))
+- Fixed "[regression] VM: Error: cannot convert -1 to uint64"
+  ([#13661](https://github.com/nim-lang/Nim/issues/13661))
+- Fixed "Spurious raiseException(Exception) detected"
+  ([#13654](https://github.com/nim-lang/Nim/issues/13654))
+- Fixed "gc:arc memory leak"
+  ([#13659](https://github.com/nim-lang/Nim/issues/13659))
+- Fixed "Error: cannot convert -1 to uint (inside tuples)"
+  ([#13671](https://github.com/nim-lang/Nim/issues/13671))
+- Fixed "strformat issue with --gc:arc"
+  ([#13622](https://github.com/nim-lang/Nim/issues/13622))
+- Fixed "astToStr doesn't work inside generics"
+  ([#13524](https://github.com/nim-lang/Nim/issues/13524))
+- Fixed "oswalkdir.walkDirRec wont return folders"
+  ([#11458](https://github.com/nim-lang/Nim/issues/11458))
+- Fixed "`echo 'echo 1' | nim c -r -`  silently gives wrong results (nimBetterRun not updated for stdin)"
+  ([#13412](https://github.com/nim-lang/Nim/issues/13412))
+- Fixed "gc:arc destroys the global variable accidentally."
+  ([#13691](https://github.com/nim-lang/Nim/issues/13691))
+- Fixed "[minor] sigmatch errors should be sorted, for reproducible errors"
+  ([#13538](https://github.com/nim-lang/Nim/issues/13538))
+- Fixed "Exception when converting csize to clong"
+  ([#13698](https://github.com/nim-lang/Nim/issues/13698))
+- Fixed "ARC: variables are no copied on the thread spawn causing crashes"
+  ([#13708](https://github.com/nim-lang/Nim/issues/13708))
+- Fixed "Illegal distinct seq causes compiler crash"
+  ([#13720](https://github.com/nim-lang/Nim/issues/13720))
+- Fixed "cyclic seq definition crashes the compiler"
+  ([#13715](https://github.com/nim-lang/Nim/issues/13715))
+- Fixed "Iterator with openArray parameter make the argument evaluated many times"
+  ([#13417](https://github.com/nim-lang/Nim/issues/13417))
+- Fixed "net/asyncnet: Unable to access peer's certificate chain"
+  ([#13299](https://github.com/nim-lang/Nim/issues/13299))
+- Fixed "Accidentally "SIGSEGV: Illegal storage access" error after arc optimizations (#13325)"
+  ([#13709](https://github.com/nim-lang/Nim/issues/13709))
+- Fixed "Base64 Regression"
+  ([#13722](https://github.com/nim-lang/Nim/issues/13722))
+- Fixed "A regression (?) with --gc:arc and repr"
+  ([#13731](https://github.com/nim-lang/Nim/issues/13731))
+- Fixed "Internal compiler error when using the new variable pragmas"
+  ([#13737](https://github.com/nim-lang/Nim/issues/13737))
+- Fixed "bool conversion produces vcc 2019 warning at cpp compilation stage"
+  ([#13744](https://github.com/nim-lang/Nim/issues/13744))
+- Fixed "Compiler "does not detect" a type recursion error in the wrong code, remaining frozen"
+  ([#13763](https://github.com/nim-lang/Nim/issues/13763))
+- Fixed "[minor] regression: `Foo[0.0] is Foo[-0.0]` is now false"
+  ([#13730](https://github.com/nim-lang/Nim/issues/13730))
+- Fixed "`nim doc` - only whitespace on first line causes segfault"
+  ([#13631](https://github.com/nim-lang/Nim/issues/13631))
+- Fixed "hashset regression"
+  ([#13794](https://github.com/nim-lang/Nim/issues/13794))
+- Fixed "`os.getApplFreebsd` could return incorrect paths in the case of a long path"
+  ([#13806](https://github.com/nim-lang/Nim/issues/13806))
+- Fixed "Destructors are not inherited"
+  ([#13810](https://github.com/nim-lang/Nim/issues/13810))
+- Fixed "io.readLines AssertionError on devel"
+  ([#13829](https://github.com/nim-lang/Nim/issues/13829))
+- Fixed "exceptions:goto accidentally reset the variable during exception handling"
+  ([#13782](https://github.com/nim-lang/Nim/issues/13782))

changelogs/changelog_1_4_0.md

@@ -0,0 +1,874 @@
+# v1.4.0 - 2020-10-16
+
+
+
+## Standard library additions and changes
+
+- Added some enhancements to `std/jsonutils` module.
+  * Added a possibility to deserialize JSON arrays directly to `HashSet` and
+    `OrderedSet` types and respectively to serialize those types to JSON arrays
+    via `jsonutils.fromJson` and `jsonutils.toJson` procedures.
+  * Added a possibility to deserialize JSON `null` objects to Nim option objects
+    and respectively to serialize Nim option object to JSON object if `isSome`
+    or to JSON null object if `isNone` via `jsonutils.fromJson` and
+    `jsonutils.toJson` procedures.
+  * Added a `Joptions` parameter to `jsonutils.fromJson` currently
+    containing two boolean options `allowExtraKeys` and `allowMissingKeys`.
+    - If `allowExtraKeys` is `true` Nim's object to which the JSON is parsed is
+      not required to have a field for every JSON key.
+    - If `allowMissingKeys` is `true` Nim's object to which JSON is parsed is
+      allowed to have fields without corresponding JSON keys.
+- Added `bindParams`, `bindParam` to `db_sqlite` for binding parameters into a `SqlPrepared` statement.
+- Added `tryInsert`,`insert` procs to `db_*` libs which accept primary key column name.
+- Added `xmltree.newVerbatimText` support create `style`'s,`script`'s text.
+- `uri` module now implements RFC-2397.
+- Added [DOM Parser](https://developer.mozilla.org/en-US/docs/Web/API/DOMParser)
+  to the `dom` module for the JavaScript target.
+- The default hash for `Ordinal` has changed to something more bit-scrambling.
+  `import hashes; proc hash(x: myInt): Hash = hashIdentity(x)` recovers the old
+  one in an instantiation context while `-d:nimIntHash1` recovers it globally.
+- `deques.peekFirst` and `deques.peekLast` now have `var Deque[T] -> var T` overloads.
+- File handles created from high-level abstractions in the stdlib will no longer
+  be inherited by child processes. In particular, these modules are affected:
+  `asyncdispatch`, `asyncnet`, `system`, `nativesockets`, `net` and `selectors`.
+
+  For `asyncdispatch`, `asyncnet`, `net` and `nativesockets`, an `inheritable`
+  flag has been added to all `proc`s that create sockets, allowing the user to
+  control whether the resulting socket is inheritable. This flag is provided to
+  ease the writing of multi-process servers, where sockets inheritance is
+  desired.
+
+  For a transition period, define `nimInheritHandles` to enable file handle
+  inheritance by default. This flag does **not** affect the `selectors` module
+  due to the differing semantics between operating systems.
+
+  `asyncdispatch.setInheritable`, `system.setInheritable` and
+  `nativesockets.setInheritable` are also introduced for setting file handle or
+  socket inheritance. Not all platforms have these `proc`s defined.
+
+- The file descriptors created for internal bookkeeping by `ioselector_kqueue`
+  and `ioselector_epoll` will no longer be leaked to child processes.
+
+- `strutils.formatFloat` with `precision = 0` has been restored to the version
+  1 behaviour that produces a trailing dot, e.g. `formatFloat(3.14159, precision = 0)`
+  is now `3.`, not `3`.
+- Added `commonPrefixLen` to `critbits`.
+
+- `relativePath(rel, abs)` and `relativePath(abs, rel)` used to silently give wrong results
+  (see #13222); instead they now use `getCurrentDir` to resolve those cases,
+  and this can now throw in edge cases where `getCurrentDir` throws.
+  `relativePath` also now works for js with `-d:nodejs`.
+
+- JavaScript and NimScript standard library changes: `streams.StringStream` is
+  now supported in JavaScript, with the limitation that any buffer `pointer`s
+  used must be castable to `ptr string`, any incompatible pointer type will not
+  work. The `lexbase` and `streams` modules used to fail to compile on
+  NimScript due to a bug, but this has been fixed.
+
+  The following modules now compile on both JS and NimScript: `parsecsv`,
+  `parsecfg`, `parsesql`, `xmlparser`, `htmlparser` and `ropes`. Additionally
+  supported for JS is `cstrutils.startsWith` and `cstrutils.endsWith`, for
+  NimScript: `json`, `parsejson`, `strtabs` and `unidecode`.
+
+- Added `streams.readStr` and `streams.peekStr` overloads to
+  accept an existing string to modify, which avoids memory
+  allocations, similar to `streams.readLine` (#13857).
+
+- Added high-level `asyncnet.sendTo` and `asyncnet.recvFrom` UDP functionality.
+
+- `dollars.$` now works for unsigned ints with `nim js`.
+
+- Improvements to the `bitops` module, including bitslices, non-mutating versions
+  of the original masking functions, `mask`/`masked`, and varargs support for
+  `bitand`, `bitor`, and `bitxor`.
+
+- `sugar.=>` and `sugar.->` changes: Previously `(x, y: int)` was transformed
+  into `(x: auto, y: int)`, it now becomes `(x: int, y: int)` for consistency
+  with regular proc definitions (although you cannot use semicolons).
+
+  Pragmas and using a name are now allowed on the lefthand side of `=>`. Here
+  is an example of these changes:
+  ```nim
+  import sugar
+
+  foo(x, y: int) {.noSideEffect.} => x + y
+
+  # is transformed into
+
+  proc foo(x: int, y: int): auto {.noSideEffect.} = x + y
+  ```
+
+- The fields of `times.DateTime` are now private, and are accessed with getters and deprecated setters.
+
+- The `times` module now handles the default value for `DateTime` more consistently.
+  Most procs raise an assertion error when given
+  an uninitialized `DateTime`, the exceptions are `==` and `$` (which returns `"Uninitialized DateTime"`).
+  The proc `times.isInitialized` has been added which can be used to check if
+  a `DateTime` has been initialized.
+
+- Fix a bug where calling `close` on io streams in `osproc.startProcess` was a noop and led to
+  hangs if a process had both reads from stdin and writes (e.g. to stdout).
+
+- The callback that is passed to `system.onThreadDestruction` must now be `.raises: []`.
+- The callback that is assigned to `system.onUnhandledException` must now be `.gcsafe`.
+
+- `osproc.execCmdEx` now takes an optional `input` for stdin, `workingDir` and `env`
+  parameters.
+
+- Added a `ssl_config` module containing lists of secure ciphers as recommended by
+  [Mozilla OpSec](https://wiki.mozilla.org/Security/Server_Side_TLS)
+
+- `net.newContext` now defaults to the list of ciphers targeting
+  ["Intermediate compatibility"](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29)
+  per Mozilla's recommendation instead of `ALL`. This change should protect
+  users from the use of weak and insecure ciphers while still provides
+  adequate compatibility with the majority of the Internet.
+
+- A new module `std/jsonutils` with hookable `jsonTo,toJson,fromJson` operations for json
+  serialization/deserialization of custom types was added.
+
+- A new proc `heapqueue.find[T](heap: HeapQueue[T], x: T): int` to get index of element ``x``
+  was added.
+- Added `rstgen.rstToLatex` a convenience proc for `renderRstToOut` and `initRstGenerator`.
+- Added `os.normalizeExe`.
+- `macros.newLit` now preserves named vs unnamed tuples.
+- Added `random.gauss`, that uses the ratio of uniforms method of sampling from a Gaussian distribution.
+- Added `typetraits.elementType` to get the element type of an iterable.
+- `typetraits.$` changes: `$(int,)` is now `"(int,)"` instead of `"(int)"`;
+  `$tuple[]` is now `"tuple[]"` instead of `"tuple"`;
+  `$((int, float), int)` is now `"((int, float), int)"` instead of `"(tuple of (int, float), int)"`
+- Added `macros.extractDocCommentsAndRunnables` helper.
+
+- `strformat.fmt` and `strformat.&` support `specifier =`. `fmt"{expr=}"` now
+  expands to `fmt"expr={expr}"`.
+- Deprecations: instead of `os.existsDir` use `dirExists`, instead of `os.existsFile` use `fileExists`.
+
+- Added the `jsre` module, [Regular Expressions for the JavaScript target.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions).
+- Made `maxLines` argument `Positive` in `logging.newRollingFileLogger`,
+  because negative values will result in a new file being created for each logged
+  line which doesn't make sense.
+- Changed `log` in `logging` to use proper log level for JavaScript,
+  e.g. `debug` uses `console.debug`, `info` uses `console.info`, `warn` uses `console.warn`, etc.
+- Tables, HashSets, SharedTables and deques don't require anymore that the passed
+  initial size must be a power of two - this is done internally.
+  Proc `rightSize` for Tables and HashSets is deprecated, as it is not needed anymore.
+  `CountTable.inc` takes `val: int` again not `val: Positive`; i.e. it can "count down" again.
+- Removed deprecated symbols from `macros` module, some of which were deprecated already in `0.15`.
+- Removed `sugar.distinctBase`, deprecated since `0.19`. Use `typetraits.distinctBase`.
+- `asyncdispatch.PDispatcher.handles` is exported so that an external low-level libraries can access it.
+
+- `std/with`, `sugar.dup` now support object field assignment expressions:
+  ```nim
+  import std/with
+
+  type Foo = object
+    x, y: int
+
+  var foo = Foo()
+  with foo:
+    x = 10
+    y = 20
+
+  echo foo
+  ```
+
+- Proc `math.round` is no longer deprecated. The advice to use `strformat` instead
+  cannot be applied to every use case. The limitations and the (lack of) reliability
+  of `round` are well documented.
+
+- Added `getprotobyname` to `winlean`. Added `getProtoByname` to `nativesockets` which returns a protocol code
+  from the database that matches the protocol `name`.
+
+- Added missing attributes and methods to `dom.Navigator` like `deviceMemory`, `onLine`, `vibrate()`, etc.
+
+- Added `strutils.indentation` and `strutils.dedent` which enable indented string literals:
+  ```nim
+  import strutils
+  echo dedent """
+    This
+      is
+        cool!
+    """
+  ```
+
+- Added `initUri(isIpv6: bool)` to `uri` module, now `uri` supports parsing ipv6 hostname.
+
+- Added `readLines(p: Process)` to `osproc`.
+
+- Added the below `toX` procs for collections. The usage is similar to procs such as
+  `sets.toHashSet` and `tables.toTable`. Previously, it was necessary to create the
+  respective empty collection and add items manually.
+    * `critbits.toCritBitTree`, which creates a `CritBitTree` from an `openArray` of
+       items or an `openArray` of pairs.
+    * `deques.toDeque`, which creates a `Deque` from an `openArray`.
+    * `heapqueue.toHeapQueue`, which creates a `HeapQueue` from an `openArray`.
+    * `intsets.toIntSet`, which creates an `IntSet` from an `openArray`.
+
+- Added `progressInterval` argument to `asyncftpclient.newAsyncFtpClient` to control the interval
+  at which progress callbacks are called.
+
+- Added `os.copyFileToDir`.
+
+## Language changes
+
+- The `=destroy` hook no longer has to reset its target, as the compiler now automatically inserts
+  `wasMoved` calls where needed.
+- The `=` hook is now called `=copy` for clarity. The old name `=` is still available so there
+  is no need to update your code. This change was backported to 1.2 too so you can use the
+  more readable `=copy` without loss of compatibility.
+
+- In the newruntime it is now allowed to assign to the discriminator field
+  without restrictions as long as the case object doesn't have a custom destructor.
+  The discriminator value doesn't have to be a constant either. If you have a
+  custom destructor for a case object and you do want to freely assign discriminator
+  fields, it is recommended to refactor the object into 2 objects like this:
+
+  ```nim
+  type
+    MyObj = object
+      case kind: bool
+      of true: y: ptr UncheckedArray[float]
+      of false: z: seq[int]
+
+  proc `=destroy`(x: MyObj) =
+    if x.kind and x.y != nil:
+      deallocShared(x.y)
+  ```
+  Refactor into:
+  ```nim
+  type
+    MySubObj = object
+      val: ptr UncheckedArray[float]
+    MyObj = object
+      case kind: bool
+      of true: y: MySubObj
+      of false: z: seq[int]
+
+  proc `=destroy`(x: MySubObj) =
+    if x.val != nil:
+      deallocShared(x.val)
+  ```
+- `getImpl` on enum type symbols now returns field syms instead of idents. This helps
+  with writing typed macros. The old behavior for backwards compatibility can be restored
+  with `--useVersion:1.0`.
+- The typed AST for proc headers will now have the arguments be syms instead of idents.
+  This helps with writing typed macros. The old behaviour for backwards compatibility can
+  be restored with `--useVersion:1.0`.
+- ``let`` statements can now be used without a value if declared with
+  ``importc``/``importcpp``/``importjs``/``importobjc``.
+- The keyword `from` is now usable as an operator.
+- Exceptions inheriting from `system.Defect` are no longer tracked with
+  the `.raises: []` exception tracking mechanism. This is more consistent with the
+  built-in operations. The following always used to compile (and still does):
+  ```nim
+  proc mydiv(a, b): int {.raises: [].} =
+    a div b # can raise an DivByZeroDefect
+  ```
+
+  Now also this compiles:
+  ```nim
+  proc mydiv(a, b): int {.raises: [].} =
+    if b == 0: raise newException(DivByZeroDefect, "division by zero")
+    else: result = a div b
+  ```
+
+  The reason for this is that `DivByZeroDefect` inherits from `Defect` and
+  with `--panics:on` `Defects` become unrecoverable errors.
+
+- Added the `thiscall` calling convention as specified by Microsoft, mostly for hooking purposes.
+- Deprecated the `{.unroll.}` pragma, because it was always ignored by the compiler anyway.
+- Removed the deprecated `strutils.isNilOrWhitespace`.
+- Removed the deprecated `sharedtables.initSharedTable`.
+- Removed the deprecated `asyncdispatch.newAsyncNativeSocket`.
+- Removed the deprecated `dom.releaseEvents` and `dom.captureEvents`.
+
+- Removed `sharedlists.initSharedList`, was deprecated and produces undefined behaviour.
+
+- There is a new experimental feature called "strictFuncs" which makes the definition of
+  `.noSideEffect` stricter. [See here](manual_experimental.html#stricts-funcs)
+  for more information.
+
+- "for-loop macros" (see [the manual](manual.html#macros-for-loop-macros)) are no longer
+  an experimental feature. In other words, you don't have to write pragma
+  `{.experimental: "forLoopMacros".}` if you want to use them.
+
+- Added the ``.noalias`` pragma. It is mapped to C's ``restrict`` keyword for the increased
+  performance this keyword can enable.
+
+- `items` no longer compiles with enums with holes as its behavior was error prone, see #14004.
+- `system.deepcopy` has to be enabled explicitly for `--gc:arc` and `--gc:orc` via
+  `--deepcopy:on`.
+
+- Added the `std/effecttraits` module for introspection of the inferred effects.
+  We hope this enables `async` macros that are precise about the possible exceptions that
+  can be raised.
+- The pragma blocks `{.gcsafe.}: ...` and `{.noSideEffect.}: ...` can now also be
+  written as `{.cast(gcsafe).}: ...` and `{.cast(noSideEffect).}: ...`. This is the new
+  preferred way of writing these, emphasizing their unsafe nature.
+
+
+## Compiler changes
+
+- Specific warnings can now be turned into errors via `--warningAsError[X]:on|off`.
+- The `define` and `undef` pragmas have been de-deprecated.
+- New command: `nim r main.nim [args...]` which compiles and runs main.nim, and implies `--usenimcache`
+  so that the output is saved to $nimcache/main$exeExt, using the same logic as `nim c -r` to
+  avoid recompilations when sources don't change.
+  Example:
+  ```bash
+  nim r compiler/nim.nim --help # only compiled the first time
+  echo 'import os; echo getCurrentCompilerExe()' | nim r - # this works too
+  nim r compiler/nim.nim --fullhelp # no recompilation
+  nim r --nimcache:/tmp main # binary saved to /tmp/main
+  ```
+- `--hint:processing` is now supported and means `--hint:processing:on`
+  (likewise with other hints and warnings), which is consistent with all other bool flags.
+  (since 1.3.3).
+- `nim doc -r main` and `nim rst2html -r main` now call `openDefaultBrowser`.
+- Added the new hint `--hint:msgOrigin` will show where a compiler msg (hint|warning|error)
+  was generated; this helps in particular when it's non obvious where it came from
+  either because multiple locations generate the same message, or because the
+  message involves runtime formatting.
+- Added the new flag `--backend:js|c|cpp|objc` (or -b:js etc), to change the backend; can be
+  used with any command (e.g. nim r, doc, check etc); safe to re-assign.
+- Added the new flag `--doccmd:cmd` to pass additional flags for runnableExamples,
+  e.g.: `--doccmd:-d:foo --threads`
+  use `--doccmd:skip` to skip runnableExamples and rst test snippets.
+- Added the new flag `--usenimcache` to output binary files to nimcache.
+- `runnableExamples "-b:cpp -r:off": code` is now supported, allowing to override
+  how an example is compiled and run, for example to change the backend.
+- `nim doc` now outputs under `$projectPath/htmldocs` when `--outdir` is unspecified
+  (with or without `--project`); passing `--project` now automatically generates
+  an index and enables search.
+  See [docgen](docgen.html#introduction-quick-start) for details.
+- Removed the `--oldNewlines` switch.
+- Removed the `--laxStrings` switch for mutating the internal zero terminator on strings.
+- Removed the `--oldast` switch.
+- Removed the `--oldgensym` switch.
+- `$getType(untyped)` is now "untyped" instead of "expr", `$getType(typed)` is
+  now "typed" instead of "stmt".
+- Sink inference is now disabled per default and has to enabled explicitly via
+  `--sinkInference:on`. *Note*: For the standard library sink inference remains
+  enabled. This change is most relevant for the `--gc:arc`, `--gc:orc` memory
+  management modes.
+
+
+## Tool changes
+
+- `nimsuggest` now returns both the forward declaration and the
+  implementation location upon a `def` query. Previously the behavior was
+  to return the forward declaration only.
+
+
+## Bugfixes
+
+- Fixed "repr() not available for uint{,8,16,32,64} under --gc:arc"
+  ([#13872](https://github.com/nim-lang/Nim/issues/13872))
+- Fixed "Critical: 1 completed Future, multiple await: Only 1 await will be awakened (the last one)"
+  ([#13889](https://github.com/nim-lang/Nim/issues/13889))
+- Fixed "crash on openarray interator with argument in stmtListExpr"
+  ([#13739](https://github.com/nim-lang/Nim/issues/13739))
+- Fixed "Some compilers on Windows don't work"
+  ([#13910](https://github.com/nim-lang/Nim/issues/13910))
+- Fixed "httpclient hangs if it recieves an HTTP 204 (No Content)"
+  ([#13894](https://github.com/nim-lang/Nim/issues/13894))
+- Fixed ""distinct uint64" type corruption on 32-bit, when using {.borrow.} operators"
+  ([#13902](https://github.com/nim-lang/Nim/issues/13902))
+- Fixed "Regression: impossible to use typed pragmas with proc types"
+  ([#13909](https://github.com/nim-lang/Nim/issues/13909))
+- Fixed "openssl wrapper corrupts stack on OpenSSL 1.1.1f + Android"
+  ([#13903](https://github.com/nim-lang/Nim/issues/13903))
+- Fixed "C compile error with --gc:arc on version 1.2.0 "unknown type name 'TGenericSeq'"
+  ([#13863](https://github.com/nim-lang/Nim/issues/13863))
+- Fixed "var return type for proc doesn't work at c++ backend"
+  ([#13848](https://github.com/nim-lang/Nim/issues/13848))
+- Fixed "TimeFormat() should raise an error but craches at compilation time"
+  ([#12864](https://github.com/nim-lang/Nim/issues/12864))
+- Fixed "gc:arc cannot fully support threadpool with FlowVar"
+  ([#13781](https://github.com/nim-lang/Nim/issues/13781))
+- Fixed "simple 'var openarray[char]' assignment crash when the openarray source is a local string and using gc:arc"
+  ([#14003](https://github.com/nim-lang/Nim/issues/14003))
+- Fixed "Cant use expressions with `when` in `type` sections."
+  ([#14007](https://github.com/nim-lang/Nim/issues/14007))
+- Fixed "`for a in MyEnum` gives incorrect results with enum with holes"
+  ([#14001](https://github.com/nim-lang/Nim/issues/14001))
+- Fixed "Trivial crash"
+  ([#12741](https://github.com/nim-lang/Nim/issues/12741))
+- Fixed "Enum with holes cannot be used as Table index"
+  ([#12834](https://github.com/nim-lang/Nim/issues/12834))
+- Fixed "spawn proc that uses typedesc crashes the compiler"
+  ([#14014](https://github.com/nim-lang/Nim/issues/14014))
+- Fixed "Docs Search `Results` box styling is not Dark Mode Friendly"
+  ([#13972](https://github.com/nim-lang/Nim/issues/13972))
+- Fixed "--gc:arc -d:useSysAssert undeclared identifier `cstderr` with newSeq"
+  ([#14038](https://github.com/nim-lang/Nim/issues/14038))
+- Fixed "issues in the manual"
+  ([#12486](https://github.com/nim-lang/Nim/issues/12486))
+- Fixed "Annoying warning: inherit from a more precise exception type like ValueError, IOError or OSError [InheritFromException]"
+  ([#14052](https://github.com/nim-lang/Nim/issues/14052))
+- Fixed "relativePath("foo", "/") and relativePath("/", "foo") is wrong"
+  ([#13222](https://github.com/nim-lang/Nim/issues/13222))
+- Fixed "[regression] `parseEnum` does not work anymore for enums with holes"
+  ([#14030](https://github.com/nim-lang/Nim/issues/14030))
+- Fixed "Exception types in the stdlib should inherit from `CatchableError` or `Defect`, not `Exception`"
+  ([#10288](https://github.com/nim-lang/Nim/issues/10288))
+- Fixed "Make debugSend and debugRecv procs public in smtp.nim"
+  ([#12189](https://github.com/nim-lang/Nim/issues/12189))
+- Fixed "xmltree need add raw text, when add style element"
+  ([#14064](https://github.com/nim-lang/Nim/issues/14064))
+- Fixed "raises requirement does not propagate to derived methods"
+  ([#8481](https://github.com/nim-lang/Nim/issues/8481))
+- Fixed "tests/stdlib/tgetaddrinfo.nim fails on NetBSD"
+  ([#14091](https://github.com/nim-lang/Nim/issues/14091))
+- Fixed "tests/niminaction/Chapter8/sdl/sdl_test.nim fails on NetBSD"
+  ([#14088](https://github.com/nim-lang/Nim/issues/14088))
+- Fixed "Incorrect escape sequence for example in jsffi library documentation"
+  ([#14110](https://github.com/nim-lang/Nim/issues/14110))
+- Fixed "HCR: Can not link exported const, in external library"
+  ([#13915](https://github.com/nim-lang/Nim/issues/13915))
+- Fixed "Cannot import std/unidecode"
+  ([#14112](https://github.com/nim-lang/Nim/issues/14112))
+- Fixed "macOS: dsymutil should not be called on static libraries"
+  ([#14132](https://github.com/nim-lang/Nim/issues/14132))
+- Fixed "nim jsondoc -o:doc.json filename.nim fails when sequences without a type are used"
+  ([#14066](https://github.com/nim-lang/Nim/issues/14066))
+- Fixed "algorithm.sortedByIt template corrupts tuple input under --gc:arc"
+  ([#14079](https://github.com/nim-lang/Nim/issues/14079))
+- Fixed "Invalid C code with lvalue conversion"
+  ([#14160](https://github.com/nim-lang/Nim/issues/14160))
+- Fixed "strformat: doc example fails"
+  ([#14054](https://github.com/nim-lang/Nim/issues/14054))
+- Fixed "Nim doc fail to run for nim 1.2.0 (nim 1.0.4 is ok)"
+  ([#13986](https://github.com/nim-lang/Nim/issues/13986))
+- Fixed "Exception when converting csize to clong"
+  ([#13698](https://github.com/nim-lang/Nim/issues/13698))
+- Fixed "[Documentation] overloading using named arguments works but is not documented"
+  ([#11932](https://github.com/nim-lang/Nim/issues/11932))
+- Fixed "import os + use of existsDir/dirExists/existsFile/fileExists/findExe in config.nims causes "ambiguous call' error"
+  ([#14142](https://github.com/nim-lang/Nim/issues/14142))
+- Fixed "import os + use of existsDir/dirExists/existsFile/fileExists/findExe in config.nims causes "ambiguous call' error"
+  ([#14142](https://github.com/nim-lang/Nim/issues/14142))
+- Fixed "runnableExamples doc gen crashes compiler with `except Exception as e` syntax"
+  ([#14177](https://github.com/nim-lang/Nim/issues/14177))
+- Fixed "[ARC] Segfault with cyclic references (?)"
+  ([#14159](https://github.com/nim-lang/Nim/issues/14159))
+- Fixed "Semcheck regression when accessing a static parameter in proc"
+  ([#14136](https://github.com/nim-lang/Nim/issues/14136))
+- Fixed "iterator walkDir doesn't work with -d:useWinAnsi"
+  ([#14201](https://github.com/nim-lang/Nim/issues/14201))
+- Fixed "cas is wrong for tcc"
+  ([#14151](https://github.com/nim-lang/Nim/issues/14151))
+- Fixed "proc execCmdEx doesn't work with -d:useWinAnsi"
+  ([#14203](https://github.com/nim-lang/Nim/issues/14203))
+- Fixed "Use -d:nimEmulateOverflowChecks by default?"
+  ([#14209](https://github.com/nim-lang/Nim/issues/14209))
+- Fixed "Old sequences with destructor objects bug"
+  ([#14217](https://github.com/nim-lang/Nim/issues/14217))
+- Fixed "[ARC] ICE when changing the discriminant of a return value"
+  ([#14244](https://github.com/nim-lang/Nim/issues/14244))
+- Fixed "[ARC] ICE with static objects"
+  ([#14236](https://github.com/nim-lang/Nim/issues/14236))
+- Fixed "[ARC] "internal error: environment misses: a" in a finalizer"
+  ([#14243](https://github.com/nim-lang/Nim/issues/14243))
+- Fixed "[ARC] compile failure using repr with object containing `ref seq[string]`"
+  ([#14270](https://github.com/nim-lang/Nim/issues/14270))
+- Fixed "[ARC] implicit move on last use happening on non-last use"
+  ([#14269](https://github.com/nim-lang/Nim/issues/14269))
+- Fixed "[ARC] Compiler crash with a recursive non-ref object variant"
+  ([#14294](https://github.com/nim-lang/Nim/issues/14294))
+- Fixed "htmlparser.parseHtml behaves differently using --gc:arc or --gc:orc"
+  ([#13946](https://github.com/nim-lang/Nim/issues/13946))
+- Fixed "Invalid return value of openProcess is NULL rather than INVALID_HANDLE_VALUE(-1) in windows"
+  ([#14289](https://github.com/nim-lang/Nim/issues/14289))
+- Fixed "ARC codegen bug with inline iterators"
+  ([#14219](https://github.com/nim-lang/Nim/issues/14219))
+- Fixed "Building koch on OpenBSD fails unless the Nim directory is in `$PATH`"
+  ([#13758](https://github.com/nim-lang/Nim/issues/13758))
+- Fixed "[gc:arc] case object assignment SIGSEGV: destroy not called for primitive type "
+  ([#14312](https://github.com/nim-lang/Nim/issues/14312))
+- Fixed "Crash when using thread and --gc:arc "
+  ([#13881](https://github.com/nim-lang/Nim/issues/13881))
+- Fixed "Getting "Warning: Cannot prove that 'result' is initialized" for an importcpp'd proc with `var T` return type"
+  ([#14314](https://github.com/nim-lang/Nim/issues/14314))
+- Fixed "`nim cpp -r --gc:arc` segfaults on caught AssertionError"
+  ([#13071](https://github.com/nim-lang/Nim/issues/13071))
+- Fixed "tests/async/tasyncawait.nim is recently very flaky"
+  ([#14320](https://github.com/nim-lang/Nim/issues/14320))
+- Fixed "Documentation nonexistent quitprocs module"
+  ([#14331](https://github.com/nim-lang/Nim/issues/14331))
+- Fixed "SIGSEV encountered when creating threads in a loop w/ --gc:arc"
+  ([#13935](https://github.com/nim-lang/Nim/issues/13935))
+- Fixed "nim-gdb is missing from all released packages"
+  ([#13104](https://github.com/nim-lang/Nim/issues/13104))
+- Fixed "sysAssert error with gc:arc on 3 line program"
+  ([#13862](https://github.com/nim-lang/Nim/issues/13862))
+- Fixed "compiler error with inline async proc and pragma"
+  ([#13998](https://github.com/nim-lang/Nim/issues/13998))
+- Fixed "[ARC] Compiler crash when adding to a seq[ref Object]"
+  ([#14333](https://github.com/nim-lang/Nim/issues/14333))
+- Fixed "nimvm: sysFatal: unhandled exception: 'sons' is not accessible using discriminant 'kind' of type 'TNode' [FieldError]"
+  ([#14340](https://github.com/nim-lang/Nim/issues/14340))
+- Fixed "[Regression] karax events are not firing "
+  ([#14350](https://github.com/nim-lang/Nim/issues/14350))
+- Fixed "odbcsql module has some wrong integer types"
+  ([#9771](https://github.com/nim-lang/Nim/issues/9771))
+- Fixed "db_sqlite needs sqlPrepared"
+  ([#13559](https://github.com/nim-lang/Nim/issues/13559))
+- Fixed "[Regression] `createThread` is not GC-safe"
+  ([#14370](https://github.com/nim-lang/Nim/issues/14370))
+- Fixed "Broken example on hot code reloading"
+  ([#14380](https://github.com/nim-lang/Nim/issues/14380))
+- Fixed "runnableExamples block with `except` on specified error fails with `nim doc`"
+  ([#12746](https://github.com/nim-lang/Nim/issues/12746))
+- Fixed "compiler as a library: findNimStdLibCompileTime fails to find system.nim"
+  ([#12293](https://github.com/nim-lang/Nim/issues/12293))
+- Fixed "5 bugs with importcpp exceptions"
+  ([#14369](https://github.com/nim-lang/Nim/issues/14369))
+- Fixed "Docs shouldn't collapse pragmas inside runnableExamples/code blocks"
+  ([#14174](https://github.com/nim-lang/Nim/issues/14174))
+- Fixed "Bad codegen/emit for hashes.hiXorLo in some contexts."
+  ([#14394](https://github.com/nim-lang/Nim/issues/14394))
+- Fixed "Boehm GC does not scan thread-local storage"
+  ([#14364](https://github.com/nim-lang/Nim/issues/14364))
+- Fixed "RVO not exception safe"
+  ([#14126](https://github.com/nim-lang/Nim/issues/14126))
+- Fixed "runnableExamples that are only compiled"
+  ([#10731](https://github.com/nim-lang/Nim/issues/10731))
+- Fixed "`foldr` raises IndexError when called on sequence"
+  ([#14404](https://github.com/nim-lang/Nim/issues/14404))
+- Fixed "moveFile does not overwrite destination file"
+  ([#14057](https://github.com/nim-lang/Nim/issues/14057))
+- Fixed "doc2 outputs in current work dir"
+  ([#6583](https://github.com/nim-lang/Nim/issues/6583))
+- Fixed "[docgen] proc doc comments silently omitted after 1st runnableExamples"
+  ([#9227](https://github.com/nim-lang/Nim/issues/9227))
+- Fixed "`nim doc --project` shows '@@/' instead of '../' for relative paths to submodules"
+  ([#14448](https://github.com/nim-lang/Nim/issues/14448))
+- Fixed "re, nre have wrong `start` semantics"
+  ([#14284](https://github.com/nim-lang/Nim/issues/14284))
+- Fixed "runnableExamples should preserve source code doc comments, strings, and (maybe) formatting"
+  ([#8871](https://github.com/nim-lang/Nim/issues/8871))
+- Fixed "`nim doc ..` fails when runnableExamples uses `$`  [devel] [regression]"
+  ([#14485](https://github.com/nim-lang/Nim/issues/14485))
+- Fixed "`items` is 20%~30% slower than iteration via an index"
+  ([#14421](https://github.com/nim-lang/Nim/issues/14421))
+- Fixed "ARC: unreliable setLen "
+  ([#14495](https://github.com/nim-lang/Nim/issues/14495))
+- Fixed "lent is unsafe: after #14447 you can modify variables with "items" loop for sequences"
+  ([#14498](https://github.com/nim-lang/Nim/issues/14498))
+- Fixed "`var op = fn()` wrongly gives warning `ObservableStores` with `object of RootObj` type"
+  ([#14514](https://github.com/nim-lang/Nim/issues/14514))
+- Fixed "Compiler assertion"
+  ([#14562](https://github.com/nim-lang/Nim/issues/14562))
+- Fixed "Can't get `ord` of a value of a Range type in the JS backend "
+  ([#14570](https://github.com/nim-lang/Nim/issues/14570))
+- Fixed "js: can't take addr of param (including implicitly via `lent`)"
+  ([#14576](https://github.com/nim-lang/Nim/issues/14576))
+- Fixed "{.noinit.} ignored in for loop -> bad codegen for non-movable types"
+  ([#14118](https://github.com/nim-lang/Nim/issues/14118))
+- Fixed "generic destructor gives: `Error: unresolved generic parameter`"
+  ([#14315](https://github.com/nim-lang/Nim/issues/14315))
+- Fixed "Memory leak with arc gc"
+  ([#14568](https://github.com/nim-lang/Nim/issues/14568))
+- Fixed "escape analysis broken with `lent`"
+  ([#14557](https://github.com/nim-lang/Nim/issues/14557))
+- Fixed "`wrapWords` seems to ignore linebreaks when wrapping, leaving breaks in the wrong place"
+  ([#14579](https://github.com/nim-lang/Nim/issues/14579))
+- Fixed "`lent` gives wrong results with -d:release"
+  ([#14578](https://github.com/nim-lang/Nim/issues/14578))
+- Fixed "Nested await expressions regression: `await a(await expandValue())` doesnt compile"
+  ([#14279](https://github.com/nim-lang/Nim/issues/14279))
+- Fixed "windows CI docs fails with strange errors"
+  ([#14545](https://github.com/nim-lang/Nim/issues/14545))
+- Fixed "[CI] tests/async/tioselectors.nim flaky test for freebsd + OSX CI"
+  ([#13166](https://github.com/nim-lang/Nim/issues/13166))
+- Fixed "seq.setLen sometimes doesn't zero memory"
+  ([#14655](https://github.com/nim-lang/Nim/issues/14655))
+- Fixed "`nim dump` is roughly 100x slower in 1.3 versus 1.2"
+  ([#14179](https://github.com/nim-lang/Nim/issues/14179))
+- Fixed "Regression: devel docgen cannot generate document for method"
+  ([#14691](https://github.com/nim-lang/Nim/issues/14691))
+- Fixed "recently flaky tests/async/t7758.nim"
+  ([#14685](https://github.com/nim-lang/Nim/issues/14685))
+- Fixed "Bind no longer working in generic procs."
+  ([#11811](https://github.com/nim-lang/Nim/issues/11811))
+- Fixed "The pegs module doesn't work with generics!"
+  ([#14718](https://github.com/nim-lang/Nim/issues/14718))
+- Fixed "Defer is not properly working for asynchronous procedures."
+  ([#13899](https://github.com/nim-lang/Nim/issues/13899))
+- Fixed "Add an ARC test with threads in a loop"
+  ([#14690](https://github.com/nim-lang/Nim/issues/14690))
+- Fixed "[goto exceptions] {.noReturn.} pragma is not detected in a case expression"
+  ([#14458](https://github.com/nim-lang/Nim/issues/14458))
+- Fixed "[exceptions:goto] C compiler error with dynlib pragma calling a proc"
+  ([#14240](https://github.com/nim-lang/Nim/issues/14240))
+- Fixed "Cannot borrow var float64 in infix assignment"
+  ([#14440](https://github.com/nim-lang/Nim/issues/14440))
+- Fixed "lib/pure/memfiles.nim: compilation error with --taintMode:on"
+  ([#14760](https://github.com/nim-lang/Nim/issues/14760))
+- Fixed "newWideCString allocates a multiple of the memory needed"
+  ([#14750](https://github.com/nim-lang/Nim/issues/14750))
+- Fixed "Nim source archive install: 'install.sh' fails with error: cp: cannot stat 'bin/nim-gdb': No such file or directory"
+  ([#14748](https://github.com/nim-lang/Nim/issues/14748))
+- Fixed "`nim cpp -r tests/exception/t9657` hangs"
+  ([#10343](https://github.com/nim-lang/Nim/issues/10343))
+- Fixed "Detect tool fails on FreeBSD"
+  ([#14715](https://github.com/nim-lang/Nim/issues/14715))
+- Fixed "compiler crash: `findUnresolvedStatic` "
+  ([#14802](https://github.com/nim-lang/Nim/issues/14802))
+- Fixed "seq namespace (?) regression"
+  ([#4796](https://github.com/nim-lang/Nim/issues/4796))
+- Fixed "Possible out of bounds string access in std/colors parseColor and isColor"
+  ([#14839](https://github.com/nim-lang/Nim/issues/14839))
+- Fixed "compile error on latest devel with orc and ssl"
+  ([#14647](https://github.com/nim-lang/Nim/issues/14647))
+- Fixed "[minor] `$` wrong for type tuple"
+  ([#13432](https://github.com/nim-lang/Nim/issues/13432))
+- Fixed "Documentation missing on devel asyncftpclient"
+  ([#14846](https://github.com/nim-lang/Nim/issues/14846))
+- Fixed "nimpretty is confused with a trailing comma in enum definition"
+  ([#14401](https://github.com/nim-lang/Nim/issues/14401))
+- Fixed "Output arguments get ignored when compiling with --app:staticlib"
+  ([#12745](https://github.com/nim-lang/Nim/issues/12745))
+- Fixed "[ARC] destructive move destroys the object too early"
+  ([#14396](https://github.com/nim-lang/Nim/issues/14396))
+- Fixed "highlite.getNextToken() crashes if the buffer string is "echo "\"""
+  ([#14830](https://github.com/nim-lang/Nim/issues/14830))
+- Fixed "Memory corruption with --gc:arc with a seq of objects with an empty body."
+  ([#14472](https://github.com/nim-lang/Nim/issues/14472))
+- Fixed "Stropped identifiers don't work as field names in tuple literals"
+  ([#14911](https://github.com/nim-lang/Nim/issues/14911))
+- Fixed "Please revert my commit"
+  ([#14930](https://github.com/nim-lang/Nim/issues/14930))
+- Fixed "[ARC] C compiler error with inline iterators and imports"
+  ([#14864](https://github.com/nim-lang/Nim/issues/14864))
+- Fixed "AsyncHttpClient segfaults with gc:orc, possibly memory corruption"
+  ([#14402](https://github.com/nim-lang/Nim/issues/14402))
+- Fixed "[ARC] Template with a block evaluating to a GC'd value results in a compiler crash"
+  ([#14899](https://github.com/nim-lang/Nim/issues/14899))
+- Fixed "[ARC] Weird issue with if expressions and templates"
+  ([#14900](https://github.com/nim-lang/Nim/issues/14900))
+- Fixed "xmlparser does not compile on devel"
+  ([#14805](https://github.com/nim-lang/Nim/issues/14805))
+- Fixed "returning lent T from a var T param gives codegen errors or SIGSEGV"
+  ([#14878](https://github.com/nim-lang/Nim/issues/14878))
+- Fixed "[ARC] Weird issue with if expressions and templates"
+  ([#14900](https://github.com/nim-lang/Nim/issues/14900))
+- Fixed "threads:on + gc:orc + unittest = C compiler errors"
+  ([#14865](https://github.com/nim-lang/Nim/issues/14865))
+- Fixed "mitems, mpairs doesn't work at compile time anymore"
+  ([#12129](https://github.com/nim-lang/Nim/issues/12129))
+- Fixed "strange result from executing code in const expression"
+  ([#10465](https://github.com/nim-lang/Nim/issues/10465))
+- Fixed "Same warning printed 3 times"
+  ([#11009](https://github.com/nim-lang/Nim/issues/11009))
+- Fixed "type alias for generic typeclass doesn't work"
+  ([#4668](https://github.com/nim-lang/Nim/issues/4668))
+- Fixed "exceptions:goto Bug devel codegen lvalue NIM_FALSE=NIM_FALSE"
+  ([#14925](https://github.com/nim-lang/Nim/issues/14925))
+- Fixed "the --useVersion:1.0 no longer works in devel"
+  ([#14912](https://github.com/nim-lang/Nim/issues/14912))
+- Fixed "template declaration of iterator doesn't compile"
+  ([#4722](https://github.com/nim-lang/Nim/issues/4722))
+- Fixed "Compiler crash on type inheritance with static generic parameter and equality check"
+  ([#12571](https://github.com/nim-lang/Nim/issues/12571))
+- Fixed "Nim crashes while handling a cast in async circumstances."
+  ([#13815](https://github.com/nim-lang/Nim/issues/13815))
+- Fixed "[ARC] Internal compiler error when calling an iterator from an inline proc "
+  ([#14383](https://github.com/nim-lang/Nim/issues/14383))
+- Fixed ""Cannot instantiate" error when template uses generic type"
+  ([#5926](https://github.com/nim-lang/Nim/issues/5926))
+- Fixed "Different raises behaviour for newTerminal between Linux and Windows"
+  ([#12759](https://github.com/nim-lang/Nim/issues/12759))
+- Fixed "Expand on a type (that defines a proc type) in error message "
+  ([#6608](https://github.com/nim-lang/Nim/issues/6608))
+- Fixed "unittest require quits program with an exit code of 0"
+  ([#14475](https://github.com/nim-lang/Nim/issues/14475))
+- Fixed "Range type: Generics vs concrete type, semcheck difference."
+  ([#8426](https://github.com/nim-lang/Nim/issues/8426))
+- Fixed "[Macro] Type mismatch when parameter name is the same as a field"
+  ([#13253](https://github.com/nim-lang/Nim/issues/13253))
+- Fixed "Generic instantiation failure when converting a sequence of circular generic types to strings"
+  ([#10396](https://github.com/nim-lang/Nim/issues/10396))
+- Fixed "initOptParser ignores argument after value option with empty value."
+  ([#13086](https://github.com/nim-lang/Nim/issues/13086))
+- Fixed "[ARC] proc with both explicit and implicit return results in a C compiler error"
+  ([#14985](https://github.com/nim-lang/Nim/issues/14985))
+- Fixed "Alias type forgets implicit generic params depending on order"
+  ([#14990](https://github.com/nim-lang/Nim/issues/14990))
+- Fixed "[ARC] sequtils.insert has different behaviour between ARC/refc"
+  ([#14994](https://github.com/nim-lang/Nim/issues/14994))
+- Fixed "The documentation for "hot code reloading" references a non-existent npm package"
+  ([#13621](https://github.com/nim-lang/Nim/issues/13621))
+- Fixed "existsDir deprecated but breaking `dir` undeclared"
+  ([#15006](https://github.com/nim-lang/Nim/issues/15006))
+- Fixed "uri.decodeUrl crashes on incorrectly formatted input"
+  ([#14082](https://github.com/nim-lang/Nim/issues/14082))
+- Fixed "testament incorrectly reports time for tests, leading to wrong conclusions"
+  ([#14822](https://github.com/nim-lang/Nim/issues/14822))
+- Fixed "Calling peekChar with Stream returned from osproc.outputStream generate runtime error"
+  ([#14906](https://github.com/nim-lang/Nim/issues/14906))
+- Fixed "localPassC pragma should come *after* other flags"
+  ([#14194](https://github.com/nim-lang/Nim/issues/14194))
+- Fixed ""Could not load" dynamic library at runtime because of hidden dependency"
+  ([#2408](https://github.com/nim-lang/Nim/issues/2408))
+- Fixed "--gc:arc generate invalid code for {.global.} (`«nimErr_» in NIM_UNLIKELY`)"
+  ([#14480](https://github.com/nim-lang/Nim/issues/14480))
+- Fixed "Using `^` from stdlib/math along with converters gives a match for types that aren't SomeNumber"
+  ([#15033](https://github.com/nim-lang/Nim/issues/15033))
+- Fixed "[ARC] Weird exception behaviour from doAssertRaises"
+  ([#15026](https://github.com/nim-lang/Nim/issues/15026))
+- Fixed "[ARC] Compiler crash declaring a finalizer proc directly in 'new'"
+  ([#15044](https://github.com/nim-lang/Nim/issues/15044))
+- Fixed "[ARC] C compiler error when creating a var of a const seq"
+  ([#15036](https://github.com/nim-lang/Nim/issues/15036))
+- Fixed "code with named arguments in proc of winim/com can not been compiled"
+  ([#15056](https://github.com/nim-lang/Nim/issues/15056))
+- Fixed "javascript backend produces javascript code with syntax error in object syntax"
+  ([#14534](https://github.com/nim-lang/Nim/issues/14534))
+- Fixed "--gc:arc should be ignored in JS mode."
+  ([#14684](https://github.com/nim-lang/Nim/issues/14684))
+- Fixed "arc: C compilation error with imported global code using a closure iterator"
+  ([#12990](https://github.com/nim-lang/Nim/issues/12990))
+- Fixed "[ARC] Crash when modifying a string with mitems iterator"
+  ([#15052](https://github.com/nim-lang/Nim/issues/15052))
+- Fixed "[ARC] SIGSEGV when calling a closure as a tuple field in a seq"
+  ([#15038](https://github.com/nim-lang/Nim/issues/15038))
+- Fixed "pass varargs[seq[T]] to iterator give empty seq "
+  ([#12576](https://github.com/nim-lang/Nim/issues/12576))
+- Fixed "Compiler crashes when using string as object variant selector with else branch"
+  ([#14189](https://github.com/nim-lang/Nim/issues/14189))
+- Fixed "JS compiler error related to implicit return and return var type"
+  ([#11354](https://github.com/nim-lang/Nim/issues/11354))
+- Fixed "`nkRecWhen` causes internalAssert in semConstructFields"
+  ([#14698](https://github.com/nim-lang/Nim/issues/14698))
+- Fixed "Memory leaks with async (closure iterators?) under ORC"
+  ([#15076](https://github.com/nim-lang/Nim/issues/15076))
+- Fixed "strutil.insertSep() fails on negative numbers"
+  ([#11352](https://github.com/nim-lang/Nim/issues/11352))
+- Fixed "Constructing a uint64 range on a 32-bit machine leads to incorrect codegen"
+  ([#14616](https://github.com/nim-lang/Nim/issues/14616))
+- Fixed "heapqueue pushpop() proc doesn't compile"
+  ([#14139](https://github.com/nim-lang/Nim/issues/14139))
+- Fixed "[ARC] SIGSEGV when trying to swap in a literal/const string"
+  ([#15112](https://github.com/nim-lang/Nim/issues/15112))
+- Fixed "Defer and --gc:arc"
+  ([#15071](https://github.com/nim-lang/Nim/issues/15071))
+- Fixed "internal error: compiler/semobjconstr.nim(324, 20) example"
+  ([#15111](https://github.com/nim-lang/Nim/issues/15111))
+- Fixed "[ARC] Sequence "disappears" with a table inside of a table with an object variant"
+  ([#15122](https://github.com/nim-lang/Nim/issues/15122))
+- Fixed "[ARC] SIGSEGV with tuple assignment caused by cursor inference"
+  ([#15130](https://github.com/nim-lang/Nim/issues/15130))
+- Fixed "Issue with --gc:arc at compile time"
+  ([#15129](https://github.com/nim-lang/Nim/issues/15129))
+- Fixed "Writing an empty string to an AsyncFile raises an IndexDefect"
+  ([#15148](https://github.com/nim-lang/Nim/issues/15148))
+- Fixed "Compiler is confused about call convention of function with nested closure"
+  ([#5688](https://github.com/nim-lang/Nim/issues/5688))
+- Fixed "Nil check on each field fails in generic function"
+  ([#15101](https://github.com/nim-lang/Nim/issues/15101))
+- Fixed "{.nimcall.} convention won't avoid the creation of closures"
+  ([#8473](https://github.com/nim-lang/Nim/issues/8473))
+- Fixed "smtp.nim(161, 40) Error: type mismatch: got <typeof(nil)> but expected 'SslContext = void'"
+  ([#15177](https://github.com/nim-lang/Nim/issues/15177))
+- Fixed "[strscans] scanf doesn't match a single character with $+ if it's the end of the string"
+  ([#15064](https://github.com/nim-lang/Nim/issues/15064))
+- Fixed "Crash and incorrect return values when using readPasswordFromStdin on Windows."
+  ([#15207](https://github.com/nim-lang/Nim/issues/15207))
+- Fixed "Possible capture error with fieldPairs and genericParams"
+  ([#15221](https://github.com/nim-lang/Nim/issues/15221))
+- Fixed "The StmtList processing of template parameters can lead to unexpected errors"
+  ([#5691](https://github.com/nim-lang/Nim/issues/5691))
+- Fixed "[ARC] C compiler error when passing a var openArray to a sink openArray"
+  ([#15035](https://github.com/nim-lang/Nim/issues/15035))
+- Fixed "Inconsistent unsigned -> signed RangeDefect usage across integer sizes"
+  ([#15210](https://github.com/nim-lang/Nim/issues/15210))
+- Fixed "toHex results in RangeDefect exception when used with large uint64"
+  ([#15257](https://github.com/nim-lang/Nim/issues/15257))
+- Fixed "Arc sink arg crash"
+  ([#15238](https://github.com/nim-lang/Nim/issues/15238))
+- Fixed "SQL escape in db_mysql is not enough"
+  ([#15219](https://github.com/nim-lang/Nim/issues/15219))
+- Fixed "Mixing 'return' with expressions is allowed in 1.2"
+  ([#15280](https://github.com/nim-lang/Nim/issues/15280))
+- Fixed "os.getFileInfo() causes ICE with --gc:arc on Windows"
+  ([#15286](https://github.com/nim-lang/Nim/issues/15286))
+- Fixed "[ARC] Sequence "disappears" with a table inside of a table with an object variant"
+  ([#15122](https://github.com/nim-lang/Nim/issues/15122))
+- Fixed "Documentation regression jsre module missing"
+  ([#15183](https://github.com/nim-lang/Nim/issues/15183))
+- Fixed "CountTable.smallest/largest() on empty table either asserts or gives bogus answer"
+  ([#15021](https://github.com/nim-lang/Nim/issues/15021))
+- Fixed "[Regression] Parser regression"
+  ([#15305](https://github.com/nim-lang/Nim/issues/15305))
+- Fixed "[ARC] SIGSEGV with tuple unpacking caused by cursor inference"
+  ([#15147](https://github.com/nim-lang/Nim/issues/15147))
+- Fixed "LwIP/FreeRTOS compile error - missing SIGPIPE and more "
+  ([#15302](https://github.com/nim-lang/Nim/issues/15302))
+- Fixed "Memory leaks with async (closure iterators?) under ORC"
+  ([#15076](https://github.com/nim-lang/Nim/issues/15076))
+- Fixed "Bug compiling with --gc:arg or --gc:orc"
+  ([#15325](https://github.com/nim-lang/Nim/issues/15325))
+- Fixed "memory corruption in tmarshall.nim"
+  ([#9754](https://github.com/nim-lang/Nim/issues/9754))
+- Fixed "typed macros break generic proc definitions"
+  ([#15326](https://github.com/nim-lang/Nim/issues/15326))
+- Fixed "nim doc2 ignores --docSeeSrcUrl parameter"
+  ([#6071](https://github.com/nim-lang/Nim/issues/6071))
+- Fixed "The decodeData Iterator from cgi module crash"
+  ([#15369](https://github.com/nim-lang/Nim/issues/15369))
+- Fixed "|| iterator generates invalid code when compiling with --debugger:native"
+  ([#9710](https://github.com/nim-lang/Nim/issues/9710))
+- Fixed "Wrong number of variables"
+  ([#15360](https://github.com/nim-lang/Nim/issues/15360))
+- Fixed "Coercions with distinct types should traverse pointer modifiers transparently."
+  ([#7165](https://github.com/nim-lang/Nim/issues/7165))
+- Fixed "Error with distinct generic TableRef"
+  ([#6060](https://github.com/nim-lang/Nim/issues/6060))
+- Fixed "Support images in nim docgen"
+  ([#6430](https://github.com/nim-lang/Nim/issues/6430))
+- Fixed "Regression. Double sem check for procs."
+  ([#15389](https://github.com/nim-lang/Nim/issues/15389))
+- Fixed "uri.nim url with literal ipv6 address is printed wrong, and cannot parsed again"
+  ([#15333](https://github.com/nim-lang/Nim/issues/15333))
+- Fixed "[ARC] Object variant gets corrupted with cursor inference"
+  ([#15361](https://github.com/nim-lang/Nim/issues/15361))
+- Fixed "`nim doc ..` compiler crash (regression 0.19.6 => 1.0)"
+  ([#14474](https://github.com/nim-lang/Nim/issues/14474))
+- Fixed "cannot borrow result; what it borrows from is potentially mutated"
+  ([#15403](https://github.com/nim-lang/Nim/issues/15403))
+- Fixed "memory corruption for seq.add(seq) with gc:arc and d:useMalloc "
+  ([#14983](https://github.com/nim-lang/Nim/issues/14983))
+- Fixed "DocGen HTML output appears improperly when encountering text immediately after/before inline monospace; in some cases won't compile"
+  ([#11537](https://github.com/nim-lang/Nim/issues/11537))
+- Fixed "Deepcopy in arc crashes"
+  ([#15405](https://github.com/nim-lang/Nim/issues/15405))
+- Fixed "pop pragma takes invalid input"
+  ([#15430](https://github.com/nim-lang/Nim/issues/15430))
+- Fixed "tests/stdlib/tgetprotobyname fails on NetBSD"
+  ([#15452](https://github.com/nim-lang/Nim/issues/15452))
+- Fixed "defer doesnt work with block, break and await"
+  ([#15243](https://github.com/nim-lang/Nim/issues/15243))
+- Fixed "tests/stdlib/tssl failing on NetBSD"
+  ([#15493](https://github.com/nim-lang/Nim/issues/15493))
+- Fixed "strictFuncs doesn't seem to catch simple ref mutation"
+  ([#15508](https://github.com/nim-lang/Nim/issues/15508))
+- Fixed "Sizeof of case object is incorrect. Showstopper"
+  ([#15516](https://github.com/nim-lang/Nim/issues/15516))
+- Fixed "[ARC] Internal error when trying to use a parallel for loop"
+  ([#15512](https://github.com/nim-lang/Nim/issues/15512))
+- Fixed "[ARC] Type-bound assign op is not being generated"
+  ([#15510](https://github.com/nim-lang/Nim/issues/15510))
+- Fixed "[ARC] Crash when adding openArray proc argument to a local seq"
+  ([#15511](https://github.com/nim-lang/Nim/issues/15511))
+- Fixed "VM: const case object gets some fields zeroed out at runtime"
+  ([#13081](https://github.com/nim-lang/Nim/issues/13081))
+- Fixed "regression(1.2.6 => devel): VM: const case object field access gives: 'sons' is not accessible"
+  ([#15532](https://github.com/nim-lang/Nim/issues/15532))
+- Fixed "Csources: huge size increase (x2.3) in 0.20"
+  ([#12027](https://github.com/nim-lang/Nim/issues/12027))
+- Fixed "Out of date error message for GC options"
+  ([#15547](https://github.com/nim-lang/Nim/issues/15547))
+- Fixed "dbQuote additional escape regression"
+  ([#15560](https://github.com/nim-lang/Nim/issues/15560))

changelogs/changelog_X_XX_X.md

@@ -1,33 +1,20 @@
-# vx.xx.x - yyyy-mm-dd
+# v1.xx.x - yyyy-mm-dd
+
+This is an example file.
+The changes should go to changelog.md!
 
 
-## Changes affecting backwards compatibility
+## Standard library additions and changes
 
-- Example item: `Foo` changed to `Bar`.
-
-
-### Breaking changes in the standard library
-
-
-### Breaking changes in the compiler
-
-
-## Library additions
-- `set[T].len` is now an alias for `set[T].card` (cardinality)
-
-## Library changes
-
-
-## Language additions
+- Added `example.exampleProc`.
+- Changed `example.foo` to take additional `bar` parameter.
 
 
 ## Language changes
 
 
-### Tool changes
+## Compiler changes
 
 
-### Compiler changes
+## Tool changes
 
-
-## Bugfixes

ci/build.bat

@@ -2,7 +2,7 @@ REM Some debug info
 echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%."
 gcc -v
 
-git clone --depth 1 https://github.com/nim-lang/csources.git
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 cd csources
 call build64.bat
 cd ..

ci/build.sh

@@ -1,7 +1,7 @@
 sh ci/deps.sh
 
 # Build from C sources.
-git clone --depth 1 https://github.com/nim-lang/csources.git
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 cd csources
 sh build.sh
 cd ..

ci/nsis_build.bat

@@ -25,7 +25,7 @@ Rem Build csources
 koch csources -d:release || exit /b
 
 rem Grab C sources and nimsuggest
-git clone --depth 1 https://github.com/nim-lang/csources.git
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 
 set PATH=%CD%\bin;%PATH%
 

compiler/aliases.nim

@@ -22,17 +22,17 @@ proc isPartOfAux(n: PNode, b: PType, marker: var IntSet): TAnalysisResult =
   result = arNo
   case n.kind
   of nkRecList:
-    for i in 0 ..< len(n):
-      result = isPartOfAux(n.sons[i], b, marker)
+    for i in 0..<n.len:
+      result = isPartOfAux(n[i], b, marker)
       if result == arYes: return
   of nkRecCase:
-    assert(n.sons[0].kind == nkSym)
-    result = isPartOfAux(n.sons[0], b, marker)
+    assert(n[0].kind == nkSym)
+    result = isPartOfAux(n[0], b, marker)
     if result == arYes: return
-    for i in 1 ..< len(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = isPartOfAux(lastSon(n.sons[i]), b, marker)
+        result = isPartOfAux(lastSon(n[i]), b, marker)
         if result == arYes: return
       else: discard "isPartOfAux(record case branch)"
   of nkSym:
@@ -46,14 +46,14 @@ proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult =
   if compareTypes(a, b, dcEqIgnoreDistinct): return arYes
   case a.kind
   of tyObject:
-    if a.sons[0] != nil:
-      result = isPartOfAux(a.sons[0].skipTypes(skipPtrs), b, marker)
+    if a[0] != nil:
+      result = isPartOfAux(a[0].skipTypes(skipPtrs), b, marker)
     if result == arNo: result = isPartOfAux(a.n, b, marker)
   of tyGenericInst, tyDistinct, tyAlias, tySink:
     result = isPartOfAux(lastSon(a), b, marker)
   of tyArray, tySet, tyTuple:
-    for i in 0 ..< len(a):
-      result = isPartOfAux(a.sons[i], b, marker)
+    for i in 0..<a.len:
+      result = isPartOfAux(a[i], b, marker)
       if result == arYes: return
   else: discard
 
@@ -108,7 +108,7 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
           result = arMaybe
     of nkBracketExpr:
       result = isPartOf(a[0], b[0])
-      if len(a) >= 2 and len(b) >= 2:
+      if a.len >= 2 and b.len >= 2:
         # array accesses:
         if result == arYes and isDeepConstExpr(a[1]) and isDeepConstExpr(b[1]):
           # we know it's the same array and we have 2 constant indexes;
@@ -181,14 +181,14 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
       else: discard
     of nkObjConstr:
       result = arNo
-      for i in 1 ..< b.len:
+      for i in 1..<b.len:
         let res = isPartOf(a, b[i][1])
         if res != arNo:
           result = res
           if res == arYes: break
     of nkCallKinds:
       result = arNo
-      for i in 1 ..< b.len:
+      for i in 1..<b.len:
         let res = isPartOf(a, b[i])
         if res != arNo:
           result = res

compiler/astalgo.nim

@@ -24,55 +24,55 @@ proc typeToYaml*(conf: ConfigRef; n: PType, indent: int = 0, maxRecDepth: int =
 proc symToYaml*(conf: ConfigRef; n: PSym, indent: int = 0, maxRecDepth: int = - 1): Rope
 proc lineInfoToStr*(conf: ConfigRef; info: TLineInfo): Rope
 
-when declared(echo):
-  # these are for debugging only: They are not really deprecated, but I want
-  # the warning so that release versions do not contain debugging statements:
-  proc debug*(n: PSym; conf: ConfigRef = nil) {.exportc: "debugSym", deprecated.}
-  proc debug*(n: PType; conf: ConfigRef = nil) {.exportc: "debugType", deprecated.}
-  proc debug*(n: PNode; conf: ConfigRef = nil) {.exportc: "debugNode", deprecated.}
 
-  proc typekinds*(t: PType) {.deprecated.} =
-    var t = t
-    var s = ""
-    while t != nil and t.len > 0:
-      s.add $t.kind
-      s.add " "
-      t = t.lastSon
-    echo s
+# these are for debugging only: They are not really deprecated, but I want
+# the warning so that release versions do not contain debugging statements:
+proc debug*(n: PSym; conf: ConfigRef = nil) {.exportc: "debugSym", deprecated.}
+proc debug*(n: PType; conf: ConfigRef = nil) {.exportc: "debugType", deprecated.}
+proc debug*(n: PNode; conf: ConfigRef = nil) {.exportc: "debugNode", deprecated.}
 
-  template debug*(x: PSym|PType|PNode) {.deprecated.} =
-    when compiles(c.config):
-      debug(c.config, x)
-    elif compiles(c.graph.config):
-      debug(c.graph.config, x)
+proc typekinds*(t: PType) {.deprecated.} =
+  var t = t
+  var s = ""
+  while t != nil and t.len > 0:
+    s.add $t.kind
+    s.add " "
+    t = t.lastSon
+  echo s
+
+template debug*(x: PSym|PType|PNode) {.deprecated.} =
+  when compiles(c.config):
+    debug(c.config, x)
+  elif compiles(c.graph.config):
+    debug(c.graph.config, x)
+  else:
+    error()
+
+template debug*(x: auto) {.deprecated.} =
+  echo x
+
+template mdbg*: bool {.deprecated.} =
+  when compiles(c.graph):
+    c.module.fileIdx == c.graph.config.projectMainIdx
+  elif compiles(c.module):
+    c.module.fileIdx == c.config.projectMainIdx
+  elif compiles(c.c.module):
+    c.c.module.fileIdx == c.c.config.projectMainIdx
+  elif compiles(m.c.module):
+    m.c.module.fileIdx == m.c.config.projectMainIdx
+  elif compiles(cl.c.module):
+    cl.c.module.fileIdx == cl.c.config.projectMainIdx
+  elif compiles(p):
+    when compiles(p.lex):
+      p.lex.fileIdx == p.lex.config.projectMainIdx
     else:
-      error()
-
-  template debug*(x: auto) {.deprecated.} =
-    echo x
-
-  template mdbg*: bool {.deprecated.} =
-    when compiles(c.graph):
-      c.module.fileIdx == c.graph.config.projectMainIdx
-    elif compiles(c.module):
-      c.module.fileIdx == c.config.projectMainIdx
-    elif compiles(c.c.module):
-      c.c.module.fileIdx == c.c.config.projectMainIdx
-    elif compiles(m.c.module):
-      m.c.module.fileIdx == m.c.config.projectMainIdx
-    elif compiles(cl.c.module):
-      cl.c.module.fileIdx == cl.c.config.projectMainIdx
-    elif compiles(p):
-      when compiles(p.lex):
-        p.lex.fileIdx == p.lex.config.projectMainIdx
-      else:
-        p.module.module.fileIdx == p.config.projectMainIdx
-    elif compiles(m.module.fileIdx):
-      m.module.fileIdx == m.config.projectMainIdx
-    elif compiles(L.fileIdx):
-      L.fileIdx == L.config.projectMainIdx
-    else:
-      error()
+      p.module.module.fileIdx == p.config.projectMainIdx
+  elif compiles(m.module.fileIdx):
+    m.module.fileIdx == m.config.projectMainIdx
+  elif compiles(L.fileIdx):
+    L.fileIdx == L.config.projectMainIdx
+  else:
+    error()
 
 # --------------------------- ident tables ----------------------------------
 proc idTableGet*(t: TIdTable, key: PIdObj): RootRef
@@ -110,15 +110,15 @@ proc iiTablePut*(t: var TIITable, key, val: int)
 
 # implementation
 
-proc skipConvAndClosure*(n: PNode): PNode =
+proc skipConvCastAndClosure*(n: PNode): PNode =
   result = n
   while true:
     case result.kind
     of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64,
        nkClosure:
-      result = result.sons[0]
-    of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-      result = result.sons[1]
+      result = result[0]
+    of nkHiddenStdConv, nkHiddenSubConv, nkConv, nkCast:
+      result = result[1]
     else: break
 
 proc sameValue*(a, b: PNode): bool =
@@ -160,17 +160,17 @@ proc lookupInRecord(n: PNode, field: PIdent): PSym =
   result = nil
   case n.kind
   of nkRecList:
-    for i in 0 ..< len(n):
-      result = lookupInRecord(n.sons[i], field)
+    for i in 0..<n.len:
+      result = lookupInRecord(n[i], field)
       if result != nil: return
   of nkRecCase:
-    if (n.sons[0].kind != nkSym): return nil
-    result = lookupInRecord(n.sons[0], field)
+    if (n[0].kind != nkSym): return nil
+    result = lookupInRecord(n[0], field)
     if result != nil: return
-    for i in 1 ..< len(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = lookupInRecord(lastSon(n.sons[i]), field)
+        result = lookupInRecord(lastSon(n[i]), field)
         if result != nil: return
       else: return nil
   of nkSym:
@@ -182,10 +182,11 @@ proc getModule*(s: PSym): PSym =
   assert((result.kind == skModule) or (result.owner != result))
   while result != nil and result.kind != skModule: result = result.owner
 
+proc fromSystem*(op: PSym): bool {.inline.} = sfSystemModule in getModule(op).flags
 proc getSymFromList*(list: PNode, ident: PIdent, start: int = 0): PSym =
-  for i in start ..< len(list):
-    if list.sons[i].kind == nkSym:
-      result = list.sons[i].sym
+  for i in start..<list.len:
+    if list[i].kind == nkSym:
+      result = list[i].sym
       if result.name.id == ident.id: return
     else: return nil
   result = nil
@@ -218,14 +219,14 @@ proc sameIgnoreBacktickGensymInfo(a, b: string): bool =
 
 proc getNamedParamFromList*(list: PNode, ident: PIdent): PSym =
   ## Named parameters are special because a named parameter can be
-  ## gensym'ed and then they have '`<number>' suffix that we need to
+  ## gensym'ed and then they have '\`<number>' suffix that we need to
   ## ignore, see compiler / evaltempl.nim, snippet:
   ##
-  ## .. code-block:: nim
+  ##..code-block:: nim
   ##
-  ##    result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $x.id),
+  ##    result.add newIdentNode(getIdent(c.ic, x.name.s & "\`gensym" & $x.id),
   ##            if c.instLines: actual.info else: templ.info)
-  for i in 1 ..< len(list):
+  for i in 1..<list.len:
     let it = list[i].sym
     if it.name.id == ident.id or
         sameIgnoreBacktickGensymInfo(it.name.s, ident.s): return it
@@ -254,15 +255,15 @@ proc makeYamlString*(s: string): Rope =
   const MaxLineLength = 64
   result = nil
   var res = "\""
-  for i in 0 ..< s.len:
+  for i in 0..<s.len:
     if (i + 1) mod MaxLineLength == 0:
-      add(res, '\"')
-      add(res, "\n")
-      add(result, rope(res))
+      res.add('\"')
+      res.add("\n")
+      result.add(rope(res))
       res = "\""              # reset
-    add(res, toYamlChar(s[i]))
-  add(res, '\"')
-  add(result, rope(res))
+    res.add(toYamlChar(s[i]))
+  res.add('\"')
+  result.add(rope(res))
 
 proc flagsToStr[T](flags: set[T]): Rope =
   if flags == {}:
@@ -270,8 +271,8 @@ proc flagsToStr[T](flags: set[T]): Rope =
   else:
     result = nil
     for x in items(flags):
-      if result != nil: add(result, ", ")
-      add(result, makeYamlString($x))
+      if result != nil: result.add(", ")
+      result.add(makeYamlString($x))
     result = "[" & result & "]"
 
 proc lineInfoToStr(conf: ConfigRef; info: TLineInfo): Rope =
@@ -298,25 +299,25 @@ proc symToYamlAux(conf: ConfigRef; n: PSym, marker: var IntSet, indent: int,
                                  #  indent + 2, maxRecDepth - 1),
     let istr = rspaces(indent + 2)
     result = rope("{")
-    addf(result, "$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
-    addf(result, "$N$1\"name\": $2", [istr, makeYamlString(n.name.s)])
-    addf(result, "$N$1\"typ\": $2", [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
+    result.addf("$N$1\"name\": $2", [istr, makeYamlString(n.name.s)])
+    result.addf("$N$1\"typ\": $2", [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth - 1)])
     if conf != nil:
       # if we don't pass the config, we probably don't care about the line info
-      addf(result, "$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
+      result.addf("$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
     if card(n.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
-    addf(result, "$N$1\"magic\": $2", [istr, makeYamlString($n.magic)])
-    addf(result, "$N$1\"ast\": $2", [istr, ast])
-    addf(result, "$N$1\"options\": $2", [istr, flagsToStr(n.options)])
-    addf(result, "$N$1\"position\": $2", [istr, rope(n.position)])
-    addf(result, "$N$1\"k\": $2", [istr, makeYamlString($n.loc.k)])
-    addf(result, "$N$1\"storage\": $2", [istr, makeYamlString($n.loc.storage)])
+      result.addf("$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
+    result.addf("$N$1\"magic\": $2", [istr, makeYamlString($n.magic)])
+    result.addf("$N$1\"ast\": $2", [istr, ast])
+    result.addf("$N$1\"options\": $2", [istr, flagsToStr(n.options)])
+    result.addf("$N$1\"position\": $2", [istr, rope(n.position)])
+    result.addf("$N$1\"k\": $2", [istr, makeYamlString($n.loc.k)])
+    result.addf("$N$1\"storage\": $2", [istr, makeYamlString($n.loc.storage)])
     if card(n.loc.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, makeYamlString($n.loc.flags)])
-    addf(result, "$N$1\"r\": $2", [istr, n.loc.r])
-    addf(result, "$N$1\"lode\": $2", [istr, treeToYamlAux(conf, n.loc.lode, marker, indent + 2, maxRecDepth - 1)])
-    addf(result, "$N$1}", [rspaces(indent)])
+      result.addf("$N$1\"flags\": $2", [istr, makeYamlString($n.loc.flags)])
+    result.addf("$N$1\"r\": $2", [istr, n.loc.r])
+    result.addf("$N$1\"lode\": $2", [istr, treeToYamlAux(conf, n.loc.lode, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1}", [rspaces(indent)])
 
 proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int,
                    maxRecDepth: int): Rope =
@@ -327,27 +328,27 @@ proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int,
     sonsRope = "\"$1 @$2\"" % [rope($n.kind), rope(
         strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))]
   else:
-    if len(n) > 0:
+    if n.len > 0:
       sonsRope = rope("[")
-      for i in 0 ..< len(n):
-        if i > 0: add(sonsRope, ",")
-        addf(sonsRope, "$N$1$2", [rspaces(indent + 4), typeToYamlAux(conf, n.sons[i],
+      for i in 0..<n.len:
+        if i > 0: sonsRope.add(",")
+        sonsRope.addf("$N$1$2", [rspaces(indent + 4), typeToYamlAux(conf, n[i],
             marker, indent + 4, maxRecDepth - 1)])
-      addf(sonsRope, "$N$1]", [rspaces(indent + 2)])
+      sonsRope.addf("$N$1]", [rspaces(indent + 2)])
     else:
       sonsRope = rope("null")
 
     let istr = rspaces(indent + 2)
     result = rope("{")
-    addf(result, "$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
-    addf(result, "$N$1\"sym\": $2",  [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth - 1)])
-    addf(result, "$N$1\"n\": $2",     [istr, treeToYamlAux(conf, n.n, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
+    result.addf("$N$1\"sym\": $2",  [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1\"n\": $2",     [istr, treeToYamlAux(conf, n.n, marker, indent + 2, maxRecDepth - 1)])
     if card(n.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
-    addf(result, "$N$1\"callconv\": $2", [istr, makeYamlString(CallingConvToStr[n.callConv])])
-    addf(result, "$N$1\"size\": $2", [istr, rope(n.size)])
-    addf(result, "$N$1\"align\": $2", [istr, rope(n.align)])
-    addf(result, "$N$1\"sons\": $2", [istr, sonsRope])
+      result.addf("$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
+    result.addf("$N$1\"callconv\": $2", [istr, makeYamlString($n.callConv)])
+    result.addf("$N$1\"size\": $2", [istr, rope(n.size)])
+    result.addf("$N$1\"align\": $2", [istr, rope(n.align)])
+    result.addf("$N$1\"sons\": $2", [istr, sonsRope])
 
 proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
                    maxRecDepth: int): Rope =
@@ -358,34 +359,34 @@ proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
     result = "{$N$1\"kind\": $2" % [istr, makeYamlString($n.kind)]
     if maxRecDepth != 0:
       if conf != nil:
-        addf(result, ",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
+        result.addf(",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
       case n.kind
       of nkCharLit..nkInt64Lit:
-        addf(result, ",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
+        result.addf(",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
       of nkFloatLit, nkFloat32Lit, nkFloat64Lit:
-        addf(result, ",$N$1\"floatVal\": $2",
+        result.addf(",$N$1\"floatVal\": $2",
             [istr, rope(n.floatVal.toStrMaxPrecision)])
       of nkStrLit..nkTripleStrLit:
-        addf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
+        result.addf(",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
       of nkSym:
-        addf(result, ",$N$1\"sym\": $2",
+        result.addf(",$N$1\"sym\": $2",
              [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth)])
       of nkIdent:
         if n.ident != nil:
-          addf(result, ",$N$1\"ident\": $2", [istr, makeYamlString(n.ident.s)])
+          result.addf(",$N$1\"ident\": $2", [istr, makeYamlString(n.ident.s)])
         else:
-          addf(result, ",$N$1\"ident\": null", [istr])
+          result.addf(",$N$1\"ident\": null", [istr])
       else:
-        if len(n) > 0:
-          addf(result, ",$N$1\"sons\": [", [istr])
-          for i in 0 ..< len(n):
-            if i > 0: add(result, ",")
-            addf(result, "$N$1$2", [rspaces(indent + 4), treeToYamlAux(conf, n.sons[i],
+        if n.len > 0:
+          result.addf(",$N$1\"sons\": [", [istr])
+          for i in 0..<n.len:
+            if i > 0: result.add(",")
+            result.addf("$N$1$2", [rspaces(indent + 4), treeToYamlAux(conf, n[i],
                 marker, indent + 4, maxRecDepth - 1)])
-          addf(result, "$N$1]", [istr])
-      addf(result, ",$N$1\"typ\": $2",
+          result.addf("$N$1]", [istr])
+      result.addf(",$N$1\"typ\": $2",
            [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth)])
-    addf(result, "$N$1}", [rspaces(indent)])
+    result.addf("$N$1}", [rspaces(indent)])
 
 proc treeToYaml(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope =
   var marker = initIntSet()
@@ -427,7 +428,7 @@ proc indentLess(this: var DebugPrinter) =
 proc newlineAndIndent(this: var DebugPrinter) =
   this.res.add "\n"
   this.currentLine += 1
-  for i in 0 ..< this.indent:
+  for i in 0..<this.indent:
     this.res.add ' '
 
 proc openCurly(this: var DebugPrinter) =
@@ -563,12 +564,12 @@ proc value(this: var DebugPrinter; value: PType) =
     this.key "n"
     this.value value.n
 
-  if len(value) > 0:
+  if value.len > 0:
     this.key "sons"
     this.openBracket
-    for i in 0 ..< len(value):
-      this.value value.sons[i]
-      if i != len(value) - 1:
+    for i in 0..<value.len:
+      this.value value[i]
+      if i != value.len - 1:
         this.comma
     this.closeBracket
 
@@ -590,10 +591,17 @@ proc value(this: var DebugPrinter; value: PNode) =
   if this.conf != nil:
     this.key "info"
     this.value $lineInfoToStr(this.conf, value.info)
-  if card(value.flags) > 0:
+  if value.flags != {}:
     this.key "flags"
     this.value value.flags
 
+  if value.typ != nil:
+    this.key "typ"
+    this.value value.typ.kind
+  else:
+    this.key "typ"
+    this.value "nil"
+
   case value.kind
   of nkCharLit..nkUInt64Lit:
     this.key "intVal"
@@ -616,41 +624,41 @@ proc value(this: var DebugPrinter; value: PNode) =
     if this.renderSymType and value.typ != nil:
       this.key "typ"
       this.value value.typ
-    if len(value) > 0:
+    if value.len > 0:
       this.key "sons"
       this.openBracket
-      for i in 0 ..< len(value):
-        this.value value.sons[i]
-        if i != len(value) - 1:
+      for i in 0..<value.len:
+        this.value value[i]
+        if i != value.len - 1:
           this.comma
       this.closeBracket
 
   this.closeCurly
 
-when declared(echo):
-  proc debug(n: PSym; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
 
-  proc debug(n: PType; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
+proc debug(n: PSym; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
 
-  proc debug(n: PNode; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    #this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
+proc debug(n: PType; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
+
+proc debug(n: PNode; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  #this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
 
 proc nextTry(h, maxHash: Hash): Hash =
   result = ((5 * h) + 1) and maxHash
@@ -677,13 +685,13 @@ proc objectSetRawInsert(data: var TObjectSeq, obj: RootRef) =
 
 proc objectSetEnlarge(t: var TObjectSet) =
   var n: TObjectSeq
-  newSeq(n, len(t.data) * GrowthFactor)
-  for i in 0 .. high(t.data):
+  newSeq(n, t.data.len * GrowthFactor)
+  for i in 0..high(t.data):
     if t.data[i] != nil: objectSetRawInsert(n, t.data[i])
   swap(t.data, n)
 
 proc objectSetIncl*(t: var TObjectSet, obj: RootRef) =
-  if mustRehash(len(t.data), t.counter): objectSetEnlarge(t)
+  if mustRehash(t.data.len, t.counter): objectSetEnlarge(t)
   objectSetRawInsert(t.data, obj)
   inc(t.counter)
 
@@ -696,7 +704,7 @@ proc objectSetContainsOrIncl*(t: var TObjectSet, obj: RootRef): bool =
     if it == obj:
       return true             # found it
     h = nextTry(h, high(t.data))
-  if mustRehash(len(t.data), t.counter):
+  if mustRehash(t.data.len, t.counter):
     objectSetEnlarge(t)
     objectSetRawInsert(t.data, obj)
   else:
@@ -739,13 +747,13 @@ proc symTabReplace*(t: var TStrTable, prevSym: PSym, newSym: PSym) =
 
 proc strTableEnlarge(t: var TStrTable) =
   var n: seq[PSym]
-  newSeq(n, len(t.data) * GrowthFactor)
-  for i in 0 .. high(t.data):
+  newSeq(n, t.data.len * GrowthFactor)
+  for i in 0..high(t.data):
     if t.data[i] != nil: strTableRawInsert(n, t.data[i])
   swap(t.data, n)
 
 proc strTableAdd*(t: var TStrTable, n: PSym) =
-  if mustRehash(len(t.data), t.counter): strTableEnlarge(t)
+  if mustRehash(t.data.len, t.counter): strTableEnlarge(t)
   strTableRawInsert(t.data, n)
   inc(t.counter)
 
@@ -772,7 +780,7 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym;
     if not onConflictKeepOld:
       t.data[replaceSlot] = n # overwrite it with newer definition!
     return t.data[replaceSlot] # found it
-  elif mustRehash(len(t.data), t.counter):
+  elif mustRehash(t.data.len, t.counter):
     strTableEnlarge(t)
     strTableRawInsert(t.data, n)
   else:
@@ -876,7 +884,7 @@ iterator items*(tab: TStrTable): PSym =
     s = nextIter(it, tab)
 
 proc hasEmptySlot(data: TIdPairSeq): bool =
-  for h in 0 .. high(data):
+  for h in 0..high(data):
     if data[h].key == nil:
       return true
   result = false
@@ -929,9 +937,9 @@ proc idTablePut(t: var TIdTable, key: PIdObj, val: RootRef) =
     assert(t.data[index].key != nil)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(t.data):
+    if mustRehash(t.data.len, t.counter):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(t.data):
         if t.data[i].key != nil:
           idTableRawInsert(n, t.data[i].key, t.data[i].val)
       assert(hasEmptySlot(n))
@@ -940,7 +948,7 @@ proc idTablePut(t: var TIdTable, key: PIdObj, val: RootRef) =
     inc(t.counter)
 
 iterator idTablePairs*(t: TIdTable): tuple[key: PIdObj, val: RootRef] =
-  for i in 0 .. high(t.data):
+  for i in 0..high(t.data):
     if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val)
 
 proc idNodeTableRawGet(t: TIdNodeTable, key: PIdObj): int =
@@ -974,10 +982,10 @@ proc idNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) =
     assert(t.data[index].key != nil)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
+    if mustRehash(t.data.len, t.counter):
       var n: TIdNodePairSeq
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(t.data):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(t.data):
         if t.data[i].key != nil:
           idNodeTableRawInsert(n, t.data[i].key, t.data[i].val)
       swap(t.data, n)
@@ -985,13 +993,13 @@ proc idNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) =
     inc(t.counter)
 
 iterator pairs*(t: TIdNodeTable): tuple[key: PIdObj, val: PNode] =
-  for i in 0 .. high(t.data):
+  for i in 0..high(t.data):
     if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val)
 
 proc initIITable(x: var TIITable) =
   x.counter = 0
   newSeq(x.data, StartSize)
-  for i in 0 ..< StartSize: x.data[i].key = InvalidKey
+  for i in 0..<StartSize: x.data[i].key = InvalidKey
 
 proc iiTableRawGet(t: TIITable, key: int): int =
   var h: Hash
@@ -1022,11 +1030,11 @@ proc iiTablePut(t: var TIITable, key, val: int) =
     assert(t.data[index].key != InvalidKey)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
+    if mustRehash(t.data.len, t.counter):
       var n: TIIPairSeq
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(n): n[i].key = InvalidKey
-      for i in 0 .. high(t.data):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(n): n[i].key = InvalidKey
+      for i in 0..high(t.data):
         if t.data[i].key != InvalidKey:
           iiTableRawInsert(n, t.data[i].key, t.data[i].val)
       swap(t.data, n)
@@ -1040,3 +1048,14 @@ proc isAddrNode*(n: PNode): bool =
       if n[0].kind == nkSym and n[0].sym.magic == mAddr: true
       else: false
     else: false
+
+proc listSymbolNames*(symbols: openArray[PSym]): string =
+  for sym in symbols:
+    if result.len > 0:
+      result.add ", "
+    result.add sym.name.s
+
+proc isDiscriminantField*(n: PNode): bool =
+  if n.kind == nkCheckedFieldExpr: sfDiscriminant in n[0][1].sym.flags
+  elif n.kind == nkDotExpr: sfDiscriminant in n[1].sym.flags
+  else: false

compiler/bitsets.nim

@@ -52,25 +52,25 @@ proc bitSetInit(b: var TBitSet, length: int) =
   newSeq(b, length)
 
 proc bitSetUnion(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] or y[i]
+  for i in 0..high(x): x[i] = x[i] or y[i]
 
 proc bitSetDiff(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] and not y[i]
+  for i in 0..high(x): x[i] = x[i] and not y[i]
 
 proc bitSetSymDiff(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] xor y[i]
+  for i in 0..high(x): x[i] = x[i] xor y[i]
 
 proc bitSetIntersect(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] and y[i]
+  for i in 0..high(x): x[i] = x[i] and y[i]
 
 proc bitSetEquals(x, y: TBitSet): bool =
-  for i in 0 .. high(x):
+  for i in 0..high(x):
     if x[i] != y[i]:
       return false
   result = true
 
 proc bitSetContains(x, y: TBitSet): bool =
-  for i in 0 .. high(x):
+  for i in 0..high(x):
     if (x[i] and not y[i]) != Zero:
       return false
   result = true

compiler/btrees.nim

@@ -16,7 +16,7 @@ const
   Mhalf = M div 2
 
 type
-  Node[Key, Val] = ref object
+  Node[Key, Val] {.acyclic.} = ref object
     entries: int
     keys: array[M, Key]
     case isInternal: bool
@@ -37,34 +37,34 @@ template eq(a, b): bool = cmp(a, b) == 0
 proc getOrDefault*[Key, Val](b: BTree[Key, Val], key: Key): Val =
   var x = b.root
   while x.isInternal:
-    for j in 0 ..< x.entries:
+    for j in 0..<x.entries:
       if j+1 == x.entries or less(key, x.keys[j+1]):
         x = x.links[j]
         break
   assert(not x.isInternal)
-  for j in 0 ..< x.entries:
+  for j in 0..<x.entries:
     if eq(key, x.keys[j]): return x.vals[j]
 
 proc contains*[Key, Val](b: BTree[Key, Val], key: Key): bool =
   var x = b.root
   while x.isInternal:
-    for j in 0 ..< x.entries:
+    for j in 0..<x.entries:
       if j+1 == x.entries or less(key, x.keys[j+1]):
         x = x.links[j]
         break
   assert(not x.isInternal)
-  for j in 0 ..< x.entries:
+  for j in 0..<x.entries:
     if eq(key, x.keys[j]): return true
   return false
 
 proc copyHalf[Key, Val](h, result: Node[Key, Val]) =
-  for j in 0 ..< Mhalf:
+  for j in 0..<Mhalf:
     result.keys[j] = h.keys[Mhalf + j]
   if h.isInternal:
-    for j in 0 ..< Mhalf:
+    for j in 0..<Mhalf:
       result.links[j] = h.links[Mhalf + j]
   else:
-    for j in 0 ..< Mhalf:
+    for j in 0..<Mhalf:
       shallowCopy(result.vals[j], h.vals[Mhalf + j])
 
 proc split[Key, Val](h: Node[Key, Val]): Node[Key, Val] =
@@ -79,6 +79,9 @@ proc insert[Key, Val](h: Node[Key, Val], key: Key, val: Val): Node[Key, Val] =
   var j = 0
   if not h.isInternal:
     while j < h.entries:
+      if eq(key, h.keys[j]):
+        h.vals[j] = val
+        return
       if less(key, h.keys[j]): break
       inc j
     for i in countdown(h.entries, j+1):
@@ -132,8 +135,7 @@ proc `$`[Key, Val](b: BTree[Key, Val]): string =
   result = ""
   toString(b.root, "", result)
 
-proc hasNext*[Key, Val](b: BTree[Key, Val]; index: int): bool =
-  result = index < b.entries
+proc hasNext*[Key, Val](b: BTree[Key, Val]; index: int): bool = index < b.entries
 
 proc countSubTree[Key, Val](it: Node[Key, Val]): int =
   if it.isInternal:
@@ -166,68 +168,3 @@ iterator pairs*[Key, Val](b: BTree[Key, Val]): (Key, Val) =
     yield (k, v)
 
 proc len*[Key, Val](b: BTree[Key, Val]): int {.inline.} = b.entries
-
-when isMainModule:
-
-  import random, tables
-
-  proc main =
-    var st = initBTree[string, string]()
-    st.add("www.cs.princeton.edu", "abc")
-    st.add("www.princeton.edu",    "128.112.128.15")
-    st.add("www.yale.edu",         "130.132.143.21")
-    st.add("www.simpsons.com",     "209.052.165.60")
-    st.add("www.apple.com",        "17.112.152.32")
-    st.add("www.amazon.com",       "207.171.182.16")
-    st.add("www.ebay.com",         "66.135.192.87")
-    st.add("www.cnn.com",          "64.236.16.20")
-    st.add("www.google.com",       "216.239.41.99")
-    st.add("www.nytimes.com",      "199.239.136.200")
-    st.add("www.microsoft.com",    "207.126.99.140")
-    st.add("www.dell.com",         "143.166.224.230")
-    st.add("www.slashdot.org",     "66.35.250.151")
-    st.add("www.espn.com",         "199.181.135.201")
-    st.add("www.weather.com",      "63.111.66.11")
-    st.add("www.yahoo.com",        "216.109.118.65")
-
-    assert st.getOrDefault("www.cs.princeton.edu") == "abc"
-    assert st.getOrDefault("www.harvardsucks.com") == nil
-
-    assert st.getOrDefault("www.simpsons.com") == "209.052.165.60"
-    assert st.getOrDefault("www.apple.com") == "17.112.152.32"
-    assert st.getOrDefault("www.ebay.com") == "66.135.192.87"
-    assert st.getOrDefault("www.dell.com") == "143.166.224.230"
-    assert(st.entries == 16)
-
-    for k, v in st:
-      echo k, ": ", v
-
-    when false:
-      var b2 = initBTree[string, string]()
-      const iters = 10_000
-      for i in 1..iters:
-        b2.add($i, $(iters - i))
-      for i in 1..iters:
-        let x = b2.getOrDefault($i)
-        if x != $(iters - i):
-          echo "got ", x, ", but expected ", iters - i
-      echo b2.entries
-
-    when true:
-      var b2 = initBTree[int, string]()
-      var t2 = initTable[int, string]()
-      const iters = 100_000
-      for i in 1..iters:
-        let x = rand(high(int))
-        if not t2.hasKey(x):
-          doAssert b2.getOrDefault(x).len == 0, " what, tree has this element " & $x
-          t2[x] = $x
-          b2.add(x, $x)
-
-      doAssert b2.entries == t2.len
-      echo "unique entries ", b2.entries
-      for k, v in t2:
-        doAssert $k == v
-        doAssert b2.getOrDefault(k) == $k
-
-  main()

compiler/canonicalizer.nim

@@ -102,7 +102,7 @@ proc hashTree(c: var MD5Context, n: PNode) =
   of nkStrLit..nkTripleStrLit:
     c &= n.strVal
   else:
-    for i in 0..<n.len: hashTree(c, n.sons[i])
+    for i in 0..<n.len: hashTree(c, n[i])
 
 proc hashType(c: var MD5Context, t: PType) =
   # modelled after 'typeToString'
@@ -120,44 +120,44 @@ proc hashType(c: var MD5Context, t: PType) =
 
   case t.kind
   of tyGenericBody, tyGenericInst, tyGenericInvocation:
-    for i in 0 ..< len(t)-ord(t.kind != tyGenericInvocation):
-      c.hashType t.sons[i]
+    for i in 0..<t.len-ord(t.kind != tyGenericInvocation):
+      c.hashType t[i]
   of tyUserTypeClass:
     internalAssert t.sym != nil and t.sym.owner != nil
     c &= t.sym.owner.name.s
   of tyUserTypeClassInst:
     let body = t.base
     c.hashSym body.sym
-    for i in 1 .. len(t) - 2:
-      c.hashType t.sons[i]
+    for i in 1..<t.len-1:
+      c.hashType t[i]
   of tyFromExpr:
     c.hashTree(t.n)
   of tyArray:
-    c.hashTree(t.sons[0].n)
-    c.hashType(t.sons[1])
+    c.hashTree(t[0].n)
+    c.hashType(t[1])
   of tyTuple:
     if t.n != nil:
-      assert(len(t.n) == len(t))
-      for i in 0 ..< len(t.n):
-        assert(t.n.sons[i].kind == nkSym)
-        c &= t.n.sons[i].sym.name.s
+      assert(t.n.len == t.len)
+      for i in 0..<t.n.len:
+        assert(t.n[i].kind == nkSym)
+        c &= t.n[i].sym.name.s
         c &= ":"
-        c.hashType(t.sons[i])
+        c.hashType(t[i])
         c &= ","
     else:
-      for i in 0 ..< len(t): c.hashType t.sons[i]
+      for i in 0..<t.len: c.hashType t[i]
   of tyRange:
     c.hashTree(t.n)
-    c.hashType(t.sons[0])
+    c.hashType(t[0])
   of tyProc:
     c &= (if tfIterator in t.flags: "iterator " else: "proc ")
-    for i in 0..<t.len: c.hashType(t.sons[i])
+    for i in 0..<t.len: c.hashType(t[i])
     md5Update(c, cast[cstring](addr(t.callConv)), 1)
 
     if tfNoSideEffect in t.flags: c &= ".noSideEffect"
     if tfThread in t.flags: c &= ".thread"
   else:
-    for i in 0..<t.len: c.hashType(t.sons[i])
+    for i in 0..<t.len: c.hashType(t[i])
   if tfNotNil in t.flags: c &= "not nil"
 
 proc canonConst(n: PNode): TUid =
@@ -238,35 +238,35 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
     encodeVInt(n.sym.id, result)
     pushSym(w, n.sym)
   else:
-    for i in 0 ..< len(n):
-      encodeNode(w, n.info, n.sons[i], result)
-  add(result, ')')
+    for i in 0..<n.len:
+      encodeNode(w, n.info, n[i], result)
+  result.add(')')
 
 proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
   var oldLen = result.len
   result.add('<')
   if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
   if loc.s != low(loc.s):
-    add(result, '*')
+    result.add('*')
     encodeVInt(ord(loc.s), result)
   if loc.flags != {}:
-    add(result, '$')
+    result.add('$')
     encodeVInt(cast[int32](loc.flags), result)
   if loc.t != nil:
-    add(result, '^')
+    result.add('^')
     encodeVInt(cast[int32](loc.t.id), result)
     pushType(w, loc.t)
   if loc.r != nil:
-    add(result, '!')
+    result.add('!')
     encodeStr($loc.r, result)
   if loc.a != 0:
-    add(result, '?')
+    result.add('?')
     encodeVInt(loc.a, result)
   if oldLen + 1 == result.len:
     # no data was necessary, so remove the '<' again:
     setLen(result, oldLen)
   else:
-    add(result, '>')
+    result.add('>')
 
 proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t == nil:
@@ -277,47 +277,47 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t.kind == tyForward: internalError("encodeType: tyForward")
   # for the new rodfile viewer we use a preceding [ so that the data section
   # can easily be disambiguated:
-  add(result, '[')
+  result.add('[')
   encodeVInt(ord(t.kind), result)
-  add(result, '+')
+  result.add('+')
   encodeVInt(t.id, result)
   if t.n != nil:
-    encodeNode(w, unknownLineInfo(), t.n, result)
+    encodeNode(w, unknownLineInfo, t.n, result)
   if t.flags != {}:
-    add(result, '$')
+    result.add('$')
     encodeVInt(cast[int32](t.flags), result)
   if t.callConv != low(t.callConv):
-    add(result, '?')
+    result.add('?')
     encodeVInt(ord(t.callConv), result)
   if t.owner != nil:
-    add(result, '*')
+    result.add('*')
     encodeVInt(t.owner.id, result)
     pushSym(w, t.owner)
   if t.sym != nil:
-    add(result, '&')
+    result.add('&')
     encodeVInt(t.sym.id, result)
     pushSym(w, t.sym)
   if t.size != - 1:
-    add(result, '/')
+    result.add('/')
     encodeVBiggestInt(t.size, result)
   if t.align != - 1:
-    add(result, '=')
+    result.add('=')
     encodeVInt(t.align, result)
   encodeLoc(w, t.loc, result)
-  for i in 0 ..< len(t):
-    if t.sons[i] == nil:
-      add(result, "^()")
+  for i in 0..<t.len:
+    if t[i] == nil:
+      result.add("^()")
     else:
-      add(result, '^')
-      encodeVInt(t.sons[i].id, result)
-      pushType(w, t.sons[i])
+      result.add('^')
+      encodeVInt(t[i].id, result)
+      pushType(w, t[i])
 
 proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) =
-  add(result, '|')
+  result.add('|')
   encodeVInt(ord(lib.kind), result)
-  add(result, '|')
+  result.add('|')
   encodeStr($lib.name, result)
-  add(result, '|')
+  result.add('|')
   encodeNode(w, info, lib.path, result)
 
 proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
@@ -363,8 +363,8 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
   encodeLoc(w, s.loc, result)
   if s.annex != nil: encodeLib(w, s.annex, s.info, result)
   if s.constraint != nil:
-    add(result, '#')
-    encodeNode(w, unknownLineInfo(), s.constraint, result)
+    result.add('#')
+    encodeNode(w, unknownLineInfo, s.constraint, result)
   # lazy loading will soon reload the ast lazily, so the ast needs to be
   # the last entry of a symbol:
   if s.ast != nil:

compiler/ccgcalls.nim

@@ -9,43 +9,95 @@
 #
 # included from cgen.nim
 
-proc leftAppearsOnRightSide(le, ri: PNode): bool =
+proc canRaiseDisp(p: BProc; n: PNode): bool =
+  # we assume things like sysFatal cannot raise themselves
+  if n.kind == nkSym and {sfNeverRaises, sfImportc, sfCompilerProc} * n.sym.flags != {}:
+    result = false
+  elif optPanics in p.config.globalOptions or
+      (n.kind == nkSym and sfSystemModule in getModule(n.sym).flags):
+    # we know we can be strict:
+    result = canRaise(n)
+  else:
+    # we have to be *very* conservative:
+    result = canRaiseConservative(n)
+
+proc preventNrvo(p: BProc; le, ri: PNode): bool =
+  proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
+    var n = le
+    while true:
+      # do NOT follow nkHiddenDeref here!
+      case n.kind
+      of nkSym:
+        # we don't own the location so it escapes:
+        if n.sym.owner != p.prc:
+          return true
+        elif inTryStmt and sfUsedInFinallyOrExcept in n.sym.flags:
+          # it is also an observable store if the location is used
+          # in 'except' or 'finally'
+          return true
+        return false
+      of nkDotExpr, nkBracketExpr, nkObjUpConv, nkObjDownConv,
+          nkCheckedFieldExpr:
+        n = n[0]
+      of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+        n = n[1]
+      else:
+        # cannot analyse the location; assume the worst
+        return true
+
   if le != nil:
-    for i in 1 ..< ri.len:
+    for i in 1..<ri.len:
       let r = ri[i]
       if isPartOf(le, r) != arNo: return true
+    # we use the weaker 'canRaise' here in order to prevent too many
+    # annoying warnings, see #14514
+    if canRaise(ri[0]) and
+        locationEscapes(p, le, p.nestedTryStmts.len > 0):
+      message(p.config, le.info, warnObservableStores, $le)
 
 proc hasNoInit(call: PNode): bool {.inline.} =
-  result = call.sons[0].kind == nkSym and sfNoInit in call.sons[0].sym.flags
+  result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
+
+proc isHarmlessStore(p: BProc; canRaise: bool; d: TLoc): bool =
+  if d.k in {locTemp, locNone} or not canRaise:
+    result = true
+  elif d.k == locLocalVar and p.withinTryWithExcept == 0:
+    # we cannot observe a store to a local variable if the current proc
+    # has no error handler:
+    result = true
+  else:
+    result = false
 
 proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
                callee, params: Rope) =
+  let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
   genLineDir(p, ri)
   var pl = callee & ~"(" & params
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
+  var typ = skipTypes(ri[0].typ, abstractInst)
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
       if params != nil: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
-        if d.k == locNone: getTemp(p, typ.sons[0], d, needsInit=true)
+        if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
           discard "resetLoc(p, d)"
-        add(pl, addrLoc(p.config, d))
-        add(pl, ~");$n")
+        pl.add(addrLoc(p.config, d))
+        pl.add(~");$n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
-        add(pl, ~");$n")
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
+        pl.add(~");$n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
+      if canRaise: raiseExit(p)
     else:
-      add(pl, ~")")
+      pl.add(~")")
       if p.module.compileToCpp:
         if lfSingleUse in d.flags:
           # do not generate spurious temporaries for C++! For C we're better off
@@ -56,29 +108,94 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
           excl d.flags, lfSingleUse
         else:
           if d.k == locNone and p.splitDecls == 0:
-            getTempCpp(p, typ.sons[0], d, pl)
+            getTempCpp(p, typ[0], d, pl)
           else:
-            if d.k == locNone: getTemp(p, typ.sons[0], d)
+            if d.k == locNone: getTemp(p, typ[0], d)
             var list: TLoc
             initLoc(list, locCall, d.lode, OnUnknown)
             list.r = pl
             genAssignment(p, d, list, {}) # no need for deep copying
-      else:
-        if d.k == locNone: getTemp(p, typ.sons[0], d)
+            if canRaise: raiseExit(p)
+
+      elif isHarmlessStore(p, canRaise, d):
+        if d.k == locNone: getTemp(p, typ[0], d)
         assert(d.t != nil)        # generate an assignment to d:
         var list: TLoc
         initLoc(list, locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
+        if canRaise: raiseExit(p)
+      else:
+        var tmp: TLoc
+        getTemp(p, typ[0], tmp, needsInit=true)
+        var list: TLoc
+        initLoc(list, locCall, d.lode, OnUnknown)
+        list.r = pl
+        genAssignment(p, tmp, list, {}) # no need for deep copying
+        if canRaise: raiseExit(p)
+        genAssignment(p, d, tmp, {})
   else:
-    add(pl, ~");$n")
+    pl.add(~");$n")
     line(p, cpsStmts, pl)
+    if canRaise: raiseExit(p)
 
 proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
 
-proc openArrayLoc(p: BProc, n: PNode): Rope =
-  var a: TLoc
+proc reifiedOpenArray(n: PNode): bool {.inline.} =
+  var x = n
+  while x.kind in {nkAddr, nkHiddenAddr, nkHiddenStdConv, nkHiddenDeref}:
+    x = x[0]
+  if x.kind == nkSym and x.sym.kind == skParam:
+    result = false
+  else:
+    result = true
 
+proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope, Rope) =
+  var a, b, c: TLoc
+  initLocExpr(p, q[1], a)
+  initLocExpr(p, q[2], b)
+  initLocExpr(p, q[3], c)
+  # but first produce the required index checks:
+  if optBoundsCheck in p.options:
+    genBoundsCheck(p, a, b, c)
+  let ty = skipTypes(a.t, abstractVar+{tyPtr})
+  let dest = getTypeDesc(p.module, destType)
+  let lengthExpr = "($1)-($2)+1" % [rdLoc(c), rdLoc(b)]
+  case ty.kind
+  of tyArray:
+    let first = toInt64(firstOrd(p.config, ty))
+    if first == 0:
+      result = ("($3*)(($1)+($2))" % [rdLoc(a), rdLoc(b), dest],
+                lengthExpr)
+    else:
+      result = ("($4*)($1)+(($2)-($3))" %
+        [rdLoc(a), rdLoc(b), intLiteral(first), dest],
+        lengthExpr)
+  of tyOpenArray, tyVarargs:
+    if reifiedOpenArray(q[1]):
+      result = ("($3*)($1.Field0)+($2)" % [rdLoc(a), rdLoc(b), dest],
+                lengthExpr)
+    else:
+      result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest],
+                lengthExpr)
+  of tyUncheckedArray, tyCString:
+    result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest],
+              lengthExpr)
+  of tyString, tySequence:
+    let atyp = skipTypes(a.t, abstractInst)
+    if formalType.skipTypes(abstractInst).kind in {tyVar} and atyp.kind == tyString and
+        optSeqDestructors in p.config.globalOptions:
+      linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+    if atyp.kind in {tyVar} and not compileToCpp(p.module):
+      result = ("($4*)(*$1)$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
+                lengthExpr)
+    else:
+      result = ("($4*)$1$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
+                lengthExpr)
+  else:
+    internalError(p.config, "openArrayLoc: " & typeToString(a.t))
+
+proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
   var q = skipConv(n)
   var skipped = false
   while q.kind == nkStmtListExpr and q.len > 0:
@@ -89,44 +206,29 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
     if skipped:
       q = skipConv(n)
       while q.kind == nkStmtListExpr and q.len > 0:
-        for i in 0..q.len-2:
+        for i in 0..<q.len-1:
           genStmts(p, q[i])
         q = q.lastSon
-    var b, c: TLoc
-    initLocExpr(p, q[1], a)
-    initLocExpr(p, q[2], b)
-    initLocExpr(p, q[3], c)
-    # but first produce the required index checks:
-    if optBoundsCheck in p.options:
-      genBoundsCheck(p, a, b, c)
-    let ty = skipTypes(a.t, abstractVar+{tyPtr})
-    let dest = getTypeDesc(p.module, n.typ.sons[0])
-    case ty.kind
-    of tyArray:
-      let first = toInt64(firstOrd(p.config, ty))
-      if first == 0:
-        result = "($4*)(($1)+($2)), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dest]
-      else:
-        result = "($5*)($1)+(($2)-($4)), ($3)-($2)+1" %
-          [rdLoc(a), rdLoc(b), rdLoc(c), intLiteral(first), dest]
-    of tyOpenArray, tyVarargs, tyUncheckedArray, tyCString:
-      result = "($4*)($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dest]
-    of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-          not compileToCpp(p.module):
-        result = "($5*)(*$1)$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
-      else:
-        result = "($5*)$1$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
-    else:
-      internalError(p.config, "openArrayLoc: " & typeToString(a.t))
+    let (x, y) = genOpenArraySlice(p, q, formalType, n.typ[0])
+    result = x & ", " & y
   else:
-    initLocExpr(p, n, a)
-    case skipTypes(a.t, abstractVar).kind
+    var a: TLoc
+    initLocExpr(p, if n.kind == nkHiddenStdConv: n[1] else: n, a)
+    case skipTypes(a.t, abstractVar+{tyStatic}).kind
     of tyOpenArray, tyVarargs:
-      result = "$1, $1Len_0" % [rdLoc(a)]
+      if reifiedOpenArray(n):
+        if a.t.kind in {tyVar, tyLent}:
+          result = "$1->Field0, $1->Field1" % [rdLoc(a)]
+        else:
+          result = "$1.Field0, $1.Field1" % [rdLoc(a)]
+      else:
+        result = "$1, $1Len_0" % [rdLoc(a)]
     of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-            not compileToCpp(p.module):
+      let ntyp = skipTypes(n.typ, abstractInst)
+      if formalType.skipTypes(abstractInst).kind in {tyVar} and ntyp.kind == tyString and
+          optSeqDestructors in p.config.globalOptions:
+        linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+      if ntyp.kind in {tyVar} and not compileToCpp(p.module):
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
         result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
@@ -146,27 +248,38 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
         internalError(p.config, "openArrayLoc: " & typeToString(a.t))
     else: internalError(p.config, "openArrayLoc: " & typeToString(a.t))
 
-proc genArgStringToCString(p: BProc, n: PNode): Rope {.inline.} =
-  var a: TLoc
-  initLocExpr(p, n.sons[0], a)
-  result = ropecg(p.module, "#nimToCStringConv($1)", [a.rdLoc])
+proc withTmpIfNeeded(p: BProc, a: TLoc, needsTmp: bool): TLoc =
+  # Bug https://github.com/status-im/nimbus-eth2/issues/1549
+  # Aliasing is preferred over stack overflows.
+  # Also don't regress for non ARC-builds, too risky.
+  if needsTmp and a.lode.typ != nil and p.config.selectedGC in {gcArc, gcOrc} and
+      getSize(p.config, a.lode.typ) < 1024:
+    getTemp(p, a.lode.typ, result, needsInit=false)
+    genAssignment(p, result, a, {})
+  else:
+    result = a
 
-proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
+proc genArgStringToCString(p: BProc, n: PNode, needsTmp: bool): Rope {.inline.} =
+  var a: TLoc
+  initLocExpr(p, n[0], a)
+  ropecg(p.module, "#nimToCStringConv($1)", [withTmpIfNeeded(p, a, needsTmp).rdLoc])
+
+proc genArg(p: BProc, n: PNode, param: PSym; call: PNode, needsTmp = false): Rope =
   var a: TLoc
   if n.kind == nkStringToCString:
-    result = genArgStringToCString(p, n)
+    result = genArgStringToCString(p, n, needsTmp)
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
-    var n = if n.kind != nkHiddenAddr: n else: n.sons[0]
-    result = openArrayLoc(p, n)
+    var n = if n.kind != nkHiddenAddr: n else: n[0]
+    result = openArrayLoc(p, param.typ, n)
   elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
-    result = addrLoc(p.config, a)
-  elif p.module.compileToCpp and param.typ.kind == tyVar and
+    result = addrLoc(p.config, withTmpIfNeeded(p, a, needsTmp))
+  elif p.module.compileToCpp and param.typ.kind in {tyVar} and
       n.kind == nkHiddenAddr:
-    initLocExprSingleUse(p, n.sons[0], a)
+    initLocExprSingleUse(p, n[0], a)
     # if the proc is 'importc'ed but not 'importcpp'ed then 'var T' still
     # means '*T'. See posix.nim for lots of examples that do that in the wild.
-    let callee = call.sons[0]
+    let callee = call[0]
     if callee.kind == nkSym and
         {sfImportc, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportc} and
         {lfHeader, lfNoDecl} * callee.sym.loc.flags != {}:
@@ -175,26 +288,85 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
       result = rdLoc(a)
   else:
     initLocExprSingleUse(p, n, a)
-    result = rdLoc(a)
+    result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
 
-proc genArgNoParam(p: BProc, n: PNode): Rope =
+proc genArgNoParam(p: BProc, n: PNode, needsTmp = false): Rope =
   var a: TLoc
   if n.kind == nkStringToCString:
-    result = genArgStringToCString(p, n)
+    result = genArgStringToCString(p, n, needsTmp)
   else:
     initLocExprSingleUse(p, n, a)
-    result = rdLoc(a)
+    result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
 
-template genParamLoop(params) {.dirty.} =
-  if i < len(typ):
-    assert(typ.n.sons[i].kind == nkSym)
-    let paramType = typ.n.sons[i]
-    if not paramType.typ.isCompileTimeOnly:
-      if params != nil: add(params, ~", ")
-      add(params, genArg(p, ri.sons[i], paramType.sym, ri))
+from dfa import instrTargets, InstrTargetKind
+
+proc potentialAlias(n: PNode, potentialWrites: seq[PNode]): bool =
+  for p in potentialWrites:
+    if instrTargets(p, n) != None:
+      return true
+
+proc skipTrivialIndirections(n: PNode): PNode =
+  result = n
+  while true:
+    case result.kind
+    of {nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv}:
+      result = result[0]
+    of {nkHiddenStdConv, nkHiddenSubConv}:
+      result = result[1]
+    else: break
+
+proc getPotentialWrites(n: PNode, mutate = false): seq[PNode] =
+  case n.kind:
+  of nkLiterals, nkIdent: discard
+  of nkSym:
+    if mutate: result.add n
+  of nkAsgn, nkFastAsgn:
+    result.add getPotentialWrites(n[0], true)
+    result.add getPotentialWrites(n[1], mutate)
+  of nkAddr, nkHiddenAddr:
+    result.add getPotentialWrites(n[0], true)
+  of nkCallKinds: #TODO: Find out why in f += 1, f is a nkSym and not a nkHiddenAddr
+    for s in n.sons:
+      result.add getPotentialWrites(s, true)
   else:
-    if params != nil: add(params, ~", ")
-    add(params, genArgNoParam(p, ri.sons[i]))
+    for s in n.sons:
+      result.add getPotentialWrites(s, mutate)
+
+proc getPotentialReads(n: PNode): seq[PNode] =
+  case n.kind:
+  of nkLiterals, nkIdent: discard
+  of nkSym: result.add n
+  else:
+    for s in n.sons:
+      result.add getPotentialReads(s)
+
+proc genParams(p: BProc, ri: PNode, typ: PType): Rope =
+  # We must generate temporaries in cases like #14396
+  # to keep the strict Left-To-Right evaluation
+  var needTmp = newSeq[bool](ri.len - 1)
+  var potentialWrites: seq[PNode]
+  for i in countdown(ri.len - 1, 1):
+    if ri[i].skipTrivialIndirections.kind == nkSym:
+      needTmp[i - 1] = potentialAlias(ri[i], potentialWrites)
+    else:
+      for n in getPotentialReads(ri[i]):
+        if not needTmp[i - 1]:
+          needTmp[i - 1] = potentialAlias(n, potentialWrites)
+      potentialWrites.add getPotentialWrites(ri[i])
+    if ri[i].kind == nkHiddenAddr:
+      # Optimization: don't use a temp, if we would only take the adress anyway
+      needTmp[i - 1] = false
+
+  for i in 1..<ri.len:
+    if i < typ.len:
+      assert(typ.n[i].kind == nkSym)
+      let paramType = typ.n[i]
+      if not paramType.typ.isCompileTimeOnly:
+        if result != nil: result.add(~", ")
+        result.add(genArg(p, ri[i], paramType.sym, ri, needTmp[i-1]))
+    else:
+      if result != nil: result.add(~", ")
+      result.add(genArgNoParam(p, ri[i], needTmp[i-1]))
 
 proc addActualSuffixForHCR(res: var Rope, module: PSym, sym: PSym) =
   if sym.flags * {sfImportc, sfNonReloadable} == {} and sym.loc.k == locProc and
@@ -204,40 +376,36 @@ proc addActualSuffixForHCR(res: var Rope, module: PSym, sym: PSym) =
 proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var op: TLoc
   # this is a hotspot in the compiler
-  initLocExpr(p, ri.sons[0], op)
-  var params: Rope
+  initLocExpr(p, ri[0], op)
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInstOwned)
+  var typ = skipTypes(ri[0].typ, abstractInstOwned)
   assert(typ.kind == tyProc)
-  assert(len(typ) == len(typ.n))
-  var length = len(ri)
-  for i in 1 ..< length:
-    genParamLoop(params)
+  assert(typ.len == typ.n.len)
+
+  var params = genParams(p, ri, typ)
+
   var callee = rdLoc(op)
-  if p.hcrOn and ri.sons[0].kind == nkSym:
-    callee.addActualSuffixForHCR(p.module.module, ri.sons[0].sym)
+  if p.hcrOn and ri[0].kind == nkSym:
+    callee.addActualSuffixForHCR(p.module.module, ri[0].sym)
   fixupCall(p, le, ri, d, callee, params)
 
 proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
 
-  proc getRawProcType(p: BProc, t: PType): Rope =
-    result = getClosureType(p.module, t, clHalf)
-
   proc addComma(r: Rope): Rope =
-    result = if r == nil: r else: r & ~", "
+    if r == nil: r else: r & ~", "
 
   const PatProc = "$1.ClE_0? $1.ClP_0($3$1.ClE_0):(($4)($1.ClP_0))($2)"
   const PatIter = "$1.ClP_0($3$1.ClE_0)" # we know the env exists
-  var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
-  var pl: Rope
 
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var op: TLoc
+  initLocExpr(p, ri[0], op)
+
+  # getUniqueType() is too expensive here:
+  var typ = skipTypes(ri[0].typ, abstractInstOwned)
   assert(typ.kind == tyProc)
-  var length = len(ri)
-  for i in 1 ..< length:
-    assert(len(typ) == len(typ.n))
-    genParamLoop(pl)
+  assert(typ.len == typ.n.len)
+
+  var pl = genParams(p, ri, typ)
 
   template genCallPattern {.dirty.} =
     if tfIterator in typ.flags:
@@ -245,28 +413,30 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
     else:
       lineF(p, cpsStmts, PatProc & ";$n", [rdLoc(op), pl, pl.addComma, rawProc])
 
-  let rawProc = getRawProcType(p, typ)
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
-      if len(ri) > 1: add(pl, ~", ")
+  let rawProc = getClosureType(p.module, typ, clHalf)
+  let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
+      if ri.len > 1: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
-          getTemp(p, typ.sons[0], d, needsInit=true)
+          getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
           discard "resetLoc(p, d)"
-        add(pl, addrLoc(p.config, d))
+        pl.add(addrLoc(p.config, d))
         genCallPattern()
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
         genCallPattern()
+        if canRaise: raiseExit(p)
         genAssignment(p, d, tmp, {}) # no need for deep copying
-    else:
-      if d.k == locNone: getTemp(p, typ.sons[0], d)
+    elif isHarmlessStore(p, canRaise, d):
+      if d.k == locNone: getTemp(p, typ[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
       initLoc(list, locCall, d.lode, OnUnknown)
@@ -274,29 +444,43 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
         list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
       else:
         list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
-
       genAssignment(p, d, list, {}) # no need for deep copying
+      if canRaise: raiseExit(p)
+    else:
+      var tmp: TLoc
+      getTemp(p, typ[0], tmp)
+      assert(d.t != nil)        # generate an assignment to d:
+      var list: TLoc
+      initLoc(list, locCall, d.lode, OnUnknown)
+      if tfIterator in typ.flags:
+        list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
+      else:
+        list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
+      genAssignment(p, tmp, list, {})
+      if canRaise: raiseExit(p)
+      genAssignment(p, d, tmp, {})
   else:
     genCallPattern()
+    if canRaise: raiseExit(p)
 
 proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
-  if i < len(typ):
+  if i < typ.len:
     # 'var T' is 'T&' in C++. This means we ignore the request of
     # any nkHiddenAddr when it's a 'var T'.
-    let paramType = typ.n.sons[i]
+    let paramType = typ.n[i]
     assert(paramType.kind == nkSym)
     if paramType.typ.isCompileTimeOnly:
       result = nil
-    elif typ.sons[i].kind == tyVar and ri.sons[i].kind == nkHiddenAddr:
-      result = genArgNoParam(p, ri.sons[i][0])
+    elif typ[i].kind in {tyVar} and ri[i].kind == nkHiddenAddr:
+      result = genArgNoParam(p, ri[i][0])
     else:
-      result = genArgNoParam(p, ri.sons[i]) #, typ.n.sons[i].sym)
+      result = genArgNoParam(p, ri[i]) #, typ.n[i].sym)
   else:
     if tfVarargs notin typ.flags:
       localError(p.config, ri.info, "wrong argument count")
       result = nil
     else:
-      result = genArgNoParam(p, ri.sons[i])
+      result = genArgNoParam(p, ri[i])
 
 discard """
 Dot call syntax in C++
@@ -340,16 +524,16 @@ proc skipAddrDeref(node: PNode): PNode =
   var isAddr = false
   case n.kind
   of nkAddr, nkHiddenAddr:
-    n = n.sons[0]
+    n = n[0]
     isAddr = true
   of nkDerefExpr, nkHiddenDeref:
-    n = n.sons[0]
+    n = n[0]
   else: return n
-  if n.kind == nkObjDownConv: n = n.sons[0]
+  if n.kind == nkObjDownConv: n = n[0]
   if isAddr and n.kind in {nkDerefExpr, nkHiddenDeref}:
-    result = n.sons[0]
+    result = n[0]
   elif n.kind in {nkAddr, nkHiddenAddr}:
-    result = n.sons[0]
+    result = n[0]
   else:
     result = node
 
@@ -357,14 +541,14 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   # for better or worse c2nim translates the 'this' argument to a 'var T'.
   # However manual wrappers may also use 'ptr T'. In any case we support both
   # for convenience.
-  internalAssert p.config, i < len(typ)
-  assert(typ.n.sons[i].kind == nkSym)
+  internalAssert p.config, i < typ.len
+  assert(typ.n[i].kind == nkSym)
   # if the parameter is lying (tyVar) and thus we required an additional deref,
   # skip the deref:
   var ri = ri[i]
   while ri.kind == nkObjDownConv: ri = ri[0]
-  let t = typ.sons[i].skipTypes({tyGenericInst, tyAlias, tySink})
-  if t.kind == tyVar:
+  let t = typ[i].skipTypes({tyGenericInst, tyAlias, tySink})
+  if t.kind in {tyVar}:
     let x = if ri.kind == nkHiddenAddr: ri[0] else: ri
     if x.typ.kind == tyPtr:
       result = genArgNoParam(p, x)
@@ -385,7 +569,7 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   else:
     ri = skipAddrDeref(ri)
     if ri.kind in {nkAddr, nkHiddenAddr}: ri = ri[0]
-    result = genArgNoParam(p, ri) #, typ.n.sons[i].sym)
+    result = genArgNoParam(p, ri) #, typ.n[i].sym)
     result.add(".")
 
 proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
@@ -395,7 +579,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
     case pat[i]
     of '@':
       var first = true
-      for k in j ..< ri.len:
+      for k in j..<ri.len:
         let arg = genOtherArg(p, ri, k, typ)
         if arg.len > 0:
           if not first:
@@ -407,12 +591,12 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
       if i+1 < pat.len and pat[i+1] in {'+', '@'}:
         let ri = ri[j]
         if ri.kind in nkCallKinds:
-          let typ = skipTypes(ri.sons[0].typ, abstractInst)
-          if pat[i+1] == '+': result.add genArgNoParam(p, ri.sons[0])
+          let typ = skipTypes(ri[0].typ, abstractInst)
+          if pat[i+1] == '+': result.add genArgNoParam(p, ri[0])
           result.add(~"(")
           if 1 < ri.len:
             result.add genOtherArg(p, ri, 1, typ)
-          for k in j+1 ..< ri.len:
+          for k in j+1..<ri.len:
             result.add(~", ")
             result.add genOtherArg(p, ri, k, typ)
           result.add(~")")
@@ -423,7 +607,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
         result.add genThisArg(p, ri, j, typ)
         inc i
       elif i+1 < pat.len and pat[i+1] == '[':
-        var arg = ri.sons[j].skipAddrDeref
+        var arg = ri[j].skipAddrDeref
         while arg.kind in {nkAddr, nkHiddenAddr, nkObjDownConv}: arg = arg[0]
         result.add genArgNoParam(p, arg)
         #result.add debugTree(arg, 0, 10)
@@ -443,24 +627,23 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
         if pat[i] notin {'@', '#', '\''}: inc(i)
         else: break
       if i - 1 >= start:
-        add(result, substr(pat, start, i - 1))
+        result.add(substr(pat, start, i - 1))
 
 proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
-  var length = len(ri)
-  assert(len(typ) == len(typ.n))
+  assert(typ.len == typ.n.len)
   # don't call '$' here for efficiency:
-  let pat = ri.sons[0].sym.loc.r.data
+  let pat = ri[0].sym.loc.r.data
   internalAssert p.config, pat.len > 0
   if pat.contains({'#', '(', '@', '\''}):
     var pl = genPatternCall(p, ri, pat, typ)
     # simpler version of 'fixupCall' that works with the pl+params combination:
-    var typ = skipTypes(ri.sons[0].typ, abstractInst)
-    if typ.sons[0] != nil:
+    var typ = skipTypes(ri[0].typ, abstractInst)
+    if typ[0] != nil:
       if p.module.compileToCpp and lfSingleUse in d.flags:
         # do not generate spurious temporaries for C++! For C we're better off
         # with them to prevent undefined behaviour and because the codegen
@@ -469,116 +652,132 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
         d.r = pl
         excl d.flags, lfSingleUse
       else:
-        if d.k == locNone: getTemp(p, typ.sons[0], d)
+        if d.k == locNone: getTemp(p, typ[0], d)
         assert(d.t != nil)        # generate an assignment to d:
         var list: TLoc
         initLoc(list, locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
     else:
-      add(pl, ~";$n")
+      pl.add(~";$n")
       line(p, cpsStmts, pl)
   else:
     var pl: Rope = nil
-    #var param = typ.n.sons[1].sym
+    #var param = typ.n[1].sym
     if 1 < ri.len:
-      add(pl, genThisArg(p, ri, 1, typ))
-    add(pl, op.r)
+      pl.add(genThisArg(p, ri, 1, typ))
+    pl.add(op.r)
     var params: Rope
-    for i in 2 ..< length:
+    for i in 2..<ri.len:
       if params != nil: params.add(~", ")
-      assert(len(typ) == len(typ.n))
-      add(params, genOtherArg(p, ri, i, typ))
+      assert(typ.len == typ.n.len)
+      params.add(genOtherArg(p, ri, i, typ))
     fixupCall(p, le, ri, d, pl, params)
 
 proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
   # generates a crappy ObjC call
   var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   var pl = ~"["
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
-  var length = len(ri)
-  assert(len(typ) == len(typ.n))
+  assert(typ.len == typ.n.len)
 
   # don't call '$' here for efficiency:
-  let pat = ri.sons[0].sym.loc.r.data
+  let pat = ri[0].sym.loc.r.data
   internalAssert p.config, pat.len > 0
   var start = 3
   if ' ' in pat:
     start = 1
-    add(pl, op.r)
-    if length > 1:
-      add(pl, ~": ")
-      add(pl, genArg(p, ri.sons[1], typ.n.sons[1].sym, ri))
+    pl.add(op.r)
+    if ri.len > 1:
+      pl.add(~": ")
+      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
       start = 2
   else:
-    if length > 1:
-      add(pl, genArg(p, ri.sons[1], typ.n.sons[1].sym, ri))
-      add(pl, ~" ")
-    add(pl, op.r)
-    if length > 2:
-      add(pl, ~": ")
-      add(pl, genArg(p, ri.sons[2], typ.n.sons[2].sym, ri))
-  for i in start ..< length:
-    assert(len(typ) == len(typ.n))
-    if i >= len(typ):
+    if ri.len > 1:
+      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
+      pl.add(~" ")
+    pl.add(op.r)
+    if ri.len > 2:
+      pl.add(~": ")
+      pl.add(genArg(p, ri[2], typ.n[2].sym, ri))
+  for i in start..<ri.len:
+    assert(typ.len == typ.n.len)
+    if i >= typ.len:
       internalError(p.config, ri.info, "varargs for objective C method?")
-    assert(typ.n.sons[i].kind == nkSym)
-    var param = typ.n.sons[i].sym
-    add(pl, ~" ")
-    add(pl, param.name.s)
-    add(pl, ~": ")
-    add(pl, genArg(p, ri.sons[i], param, ri))
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
-      if len(ri) > 1: add(pl, ~" ")
+    assert(typ.n[i].kind == nkSym)
+    var param = typ.n[i].sym
+    pl.add(~" ")
+    pl.add(param.name.s)
+    pl.add(~": ")
+    pl.add(genArg(p, ri[i], param, ri))
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
+      if ri.len > 1: pl.add(~" ")
       # beware of 'result = p(result)'. We always allocate a temporary:
       if d.k in {locTemp, locNone}:
         # We already got a temp. Great, special case it:
-        if d.k == locNone: getTemp(p, typ.sons[0], d, needsInit=true)
-        add(pl, ~"Result: ")
-        add(pl, addrLoc(p.config, d))
-        add(pl, ~"];$n")
+        if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
+        pl.add(~"Result: ")
+        pl.add(addrLoc(p.config, d))
+        pl.add(~"];$n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
-        add(pl, ~"];$n")
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
+        pl.add(~"];$n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
     else:
-      add(pl, ~"]")
-      if d.k == locNone: getTemp(p, typ.sons[0], d)
+      pl.add(~"]")
+      if d.k == locNone: getTemp(p, typ[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
       initLoc(list, locCall, ri, OnUnknown)
       list.r = pl
       genAssignment(p, d, list, {}) # no need for deep copying
   else:
-    add(pl, ~"];$n")
+    pl.add(~"];$n")
     line(p, cpsStmts, pl)
 
-proc genCall(p: BProc, e: PNode, d: var TLoc) =
-  if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
-    genClosureCall(p, nil, e, d)
-  elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags:
-    genInfixCall(p, nil, e, d)
-  elif e.sons[0].kind == nkSym and sfNamedParamCall in e.sons[0].sym.flags:
-    genNamedParamCall(p, e, d)
-  else:
-    genPrefixCall(p, nil, e, d)
-  postStmtActions(p)
+proc notYetAlive(n: PNode): bool {.inline.} =
+  let r = getRoot(n)
+  result = r != nil and r.loc.lode == nil
+
+proc isInactiveDestructorCall(p: BProc, e: PNode): bool =
+  #[ Consider this example.
+
+    var :tmpD_3281815
+    try:
+      if true:
+        return
+      let args_3280013 =
+        wasMoved_3281816(:tmpD_3281815)
+        `=_3280036`(:tmpD_3281815, [1])
+        :tmpD_3281815
+    finally:
+      `=destroy_3280027`(args_3280013)
+
+  We want to return early but the 'finally' section is traversed before
+  the 'let args = ...' statement. We exploit this to generate better
+  code for 'return'. ]#
+  result = e.len == 2 and e[0].kind == nkSym and
+    e[0].sym.name.s == "=destroy" and notYetAlive(e[1].skipAddr)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
-  if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
+  if p.withinBlockLeaveActions > 0 and isInactiveDestructorCall(p, ri):
+    return
+  if ri[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
-  elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags:
+  elif ri[0].kind == nkSym and sfInfixCall in ri[0].sym.flags:
     genInfixCall(p, le, ri, d)
-  elif ri.sons[0].kind == nkSym and sfNamedParamCall in ri.sons[0].sym.flags:
+  elif ri[0].kind == nkSym and sfNamedParamCall in ri[0].sym.flags:
     genNamedParamCall(p, ri, d)
   else:
     genPrefixCall(p, le, ri, d)
   postStmtActions(p)
+
+proc genCall(p: BProc, e: PNode, d: var TLoc) = genAsgnCall(p, nil, e, d)

compiler/ccgliterals.nim

@@ -35,8 +35,8 @@ proc detectSeqVersion(m: BModule): int =
 proc genStringLiteralDataOnlyV1(m: BModule, s: string): Rope =
   discard cgsym(m, "TGenericSeq")
   result = getTempName(m)
-  addf(m.s[cfsData], "STRING_LITERAL($1, $2, $3);$n",
-       [result, makeCString(s), rope(len(s))])
+  m.s[cfsData].addf("STRING_LITERAL($1, $2, $3);$n",
+       [result, makeCString(s), rope(s.len)])
 
 proc genStringLiteralV1(m: BModule; n: PNode): Rope =
   if s.isNil:
@@ -53,29 +53,31 @@ proc genStringLiteralV1(m: BModule; n: PNode): Rope =
 
 # ------ Version 2: destructor based strings and seqs -----------------------
 
-proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope) =
-  addf(m.s[cfsData], "static const struct {$n" &
-       "  NI cap; void* allocator; NIM_CHAR data[$2+1];$n" &
-       "} $1 = { $2, NIM_NIL, $3 };$n",
-       [result, rope(len(s)), makeCString(s)])
+proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope; isConst: bool) =
+  m.s[cfsData].addf("static $4 struct {$n" &
+       "  NI cap; NIM_CHAR data[$2+1];$n" &
+       "} $1 = { $2 | NIM_STRLIT_FLAG, $3 };$n",
+       [result, rope(s.len), makeCString(s),
+       rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2(m: BModule; n: PNode): Rope =
+proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   if id == m.labels:
     let pureLit = getTempName(m)
-    genStringLiteralDataOnlyV2(m, n.strVal, pureLit)
+    genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
     result = getTempName(m)
     discard cgsym(m, "NimStrPayload")
     discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    addf(m.s[cfsData], "static const NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [result, rope(len(n.strVal)), pureLit])
+    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [result, rope(n.strVal.len), pureLit, rope(if isConst: "const" else: "")])
   else:
     result = getTempName(m)
-    addf(m.s[cfsData], "static const NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [result, rope(len(n.strVal)), m.tmpBase & rope(id)])
+    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [result, rope(n.strVal.len), m.tmpBase & rope(id),
+          rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2Const(m: BModule; n: PNode): Rope =
+proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   var pureLit: Rope
   if id == m.labels:
@@ -83,19 +85,20 @@ proc genStringLiteralV2Const(m: BModule; n: PNode): Rope =
     discard cgsym(m, "NimStrPayload")
     discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    genStringLiteralDataOnlyV2(m, n.strVal, pureLit)
+    genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
   else:
     pureLit = m.tmpBase & rope(id)
-  result = "{$1, (NimStrPayload*)&$2}" % [rope(len(n.strVal)), pureLit]
+  result = "{$1, (NimStrPayload*)&$2}" % [rope(n.strVal.len), pureLit]
 
 # ------ Version selector ---------------------------------------------------
 
-proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo): Rope =
+proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo;
+                              isConst: bool): Rope =
   case detectStrVersion(m)
   of 0, 1: result = genStringLiteralDataOnlyV1(m, s)
   of 2:
     result = getTempName(m)
-    genStringLiteralDataOnlyV2(m, s, result)
+    genStringLiteralDataOnlyV2(m, s, result, isConst)
   else:
     localError(m.config, info, "cannot determine how to produce code for string literal")
 
@@ -105,6 +108,6 @@ proc genNilStringLiteral(m: BModule; info: TLineInfo): Rope =
 proc genStringLiteral(m: BModule; n: PNode): Rope =
   case detectStrVersion(m)
   of 0, 1: result = genStringLiteralV1(m, n)
-  of 2: result = genStringLiteralV2(m, n)
+  of 2: result = genStringLiteralV2(m, n, isConst = true)
   else:
     localError(m.config, n.info, "cannot determine how to produce code for string literal")

compiler/ccgmerge.nim

@@ -28,8 +28,8 @@ const
     cfsFieldInfo: "NIM_merge_FIELD_INFO",
     cfsTypeInfo: "NIM_merge_TYPE_INFO",
     cfsProcHeaders: "NIM_merge_PROC_HEADERS",
-    cfsVars: "NIM_merge_VARS",
     cfsData: "NIM_merge_DATA",
+    cfsVars: "NIM_merge_VARS",
     cfsProcs: "NIM_merge_PROCS",
     cfsInitProc: "NIM_merge_INIT_PROC",
     cfsDatInitProc: "NIM_merge_DATINIT_PROC",
@@ -48,11 +48,15 @@ const
   NimMergeEndMark = "/*\tNIM_merge_END:*/"
 
 proc genSectionStart*(fs: TCFileSection; conf: ConfigRef): Rope =
+  # useful for debugging and only adds at most a few lines in each file
+  result.add("\n/* section: ")
+  result.add(CFileSectionNames[fs])
+  result.add(" */\n")
   if compilationCachePresent(conf):
     result = nil
-    add(result, "\n/*\t")
-    add(result, CFileSectionNames[fs])
-    add(result, ":*/\n")
+    result.add("\n/*\t")
+    result.add(CFileSectionNames[fs])
+    result.add(":*/\n")
 
 proc genSectionEnd*(fs: TCFileSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
@@ -61,9 +65,9 @@ proc genSectionEnd*(fs: TCFileSection; conf: ConfigRef): Rope =
 proc genSectionStart*(ps: TCProcSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
     result = rope("")
-    add(result, "\n/*\t")
-    add(result, CProcSectionNames[ps])
-    add(result, ":*/\n")
+    result.add("\n/*\t")
+    result.add(CProcSectionNames[ps])
+    result.add(":*/\n")
 
 proc genSectionEnd*(ps: TCProcSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
@@ -278,7 +282,7 @@ proc mergeRequired*(m: BModule): bool =
     if m.s[i] != nil:
       #echo "not empty: ", i, " ", m.s[i]
       return true
-  for i in low(TCProcSection)..high(TCProcSection):
+  for i in TCProcSection:
     if m.initProc.s(i) != nil:
       #echo "not empty: ", i, " ", m.initProc.s[i]
       return true
@@ -288,7 +292,7 @@ proc mergeFiles*(cfilename: AbsoluteFile, m: BModule) =
   var old: TMergeSections
   readMergeSections(cfilename, old)
   # do the merge; old section before new section:
-  for i in low(TCFileSection)..high(TCFileSection):
+  for i in TCFileSection:
     m.s[i] = old.f[i] & m.s[i]
-  for i in low(TCProcSection)..high(TCProcSection):
+  for i in TCProcSection:
     m.initProc.s(i) = old.p[i] & m.initProc.s(i)

compiler/ccgreset.nim

@@ -0,0 +1,94 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# included from cgen.nim
+
+## Code specialization instead of the old, incredibly slow 'genericReset'
+## implementation.
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType)
+
+proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
+                     typ: PType) =
+  if n == nil: return
+  case n.kind
+  of nkRecList:
+    for i in 0..<n.len:
+      specializeResetN(p, accessor, n[i], typ)
+  of nkRecCase:
+    if (n[0].kind != nkSym): internalError(p.config, n.info, "specializeResetN")
+    let disc = n[0].sym
+    if disc.loc.r == nil: fillObjectFields(p.module, typ)
+    if disc.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
+    for i in 1..<n.len:
+      let branch = n[i]
+      assert branch.kind in {nkOfBranch, nkElse}
+      if branch.kind == nkOfBranch:
+        genCaseRange(p, branch)
+      else:
+        lineF(p, cpsStmts, "default:$n", [])
+      specializeResetN(p, accessor, lastSon(branch), typ)
+      lineF(p, cpsStmts, "break;$n", [])
+    lineF(p, cpsStmts, "} $n", [])
+    specializeResetT(p, "$1.$2" % [accessor, disc.loc.r], disc.loc.t)
+  of nkSym:
+    let field = n.sym
+    if field.typ.kind == tyVoid: return
+    if field.loc.r == nil: fillObjectFields(p.module, typ)
+    if field.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    specializeResetT(p, "$1.$2" % [accessor, field.loc.r], field.loc.t)
+  else: internalError(p.config, n.info, "specializeResetN()")
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
+  if typ == nil: return
+
+  case typ.kind
+  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred,
+     tySink, tyOwned:
+    specializeResetT(p, accessor, lastSon(typ))
+  of tyArray:
+    let arraySize = lengthOrd(p.config, typ[0])
+    var i: TLoc
+    getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i)
+    linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
+            [i.r, arraySize])
+    specializeResetT(p, ropecg(p.module, "$1[$2]", [accessor, i.r]), typ[1])
+    lineF(p, cpsStmts, "}$n", [])
+  of tyObject:
+    for i in 0..<typ.len:
+      var x = typ[i]
+      if x != nil: x = x.skipTypes(skipPtrs)
+      specializeResetT(p, accessor.parentObj(p.module), x)
+    if typ.n != nil: specializeResetN(p, accessor, typ.n, typ)
+  of tyTuple:
+    let typ = getUniqueType(typ)
+    for i in 0..<typ.len:
+      specializeResetT(p, ropecg(p.module, "$1.Field$2", [accessor, i]), typ[i])
+
+  of tyString, tyRef, tySequence:
+    lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1, NIM_NIL);$n", [accessor])
+
+  of tyProc:
+    if typ.callConv == ccClosure:
+      lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1.ClE_0, NIM_NIL);$n", [accessor])
+      lineCg(p, cpsStmts, "$1.ClP_0 = NIM_NIL;$n", [accessor])
+    else:
+      lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  of tyChar, tyBool, tyEnum, tyInt..tyUInt64:
+    lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
+  of tyCString, tyPointer, tyPtr, tyVar, tyLent:
+    lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  else:
+    discard
+
+proc specializeReset(p: BProc, a: TLoc) =
+  specializeResetT(p, rdLoc(a), a.t)

compiler/ccgthreadvars.nim

@@ -16,11 +16,11 @@ proc emulatedThreadVars(conf: ConfigRef): bool =
   result = {optThreads, optTlsEmulation} <= conf.globalOptions
 
 proc accessThreadLocalVar(p: BProc, s: PSym) =
-  if emulatedThreadVars(p.config) and not p.threadVarAccessed:
-    p.threadVarAccessed = true
+  if emulatedThreadVars(p.config) and threadVarAccessed notin p.flags:
+    p.flags.incl threadVarAccessed
     incl p.module.flags, usesThreadVars
-    addf(p.procSec(cpsLocals), "\tNimThreadVars* NimTV_;$n", [])
-    add(p.procSec(cpsInit),
+    p.procSec(cpsLocals).addf("\tNimThreadVars* NimTV_;$n", [])
+    p.procSec(cpsInit).add(
       ropecg(p.module, "\tNimTV_ = (NimThreadVars*) #GetThreadLocalVars();$n", []))
 
 proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
@@ -30,23 +30,26 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
     # allocator for it :-(
     if not containsOrIncl(m.g.nimtvDeclared, s.id):
       m.g.nimtvDeps.add(s.loc.t)
-      addf(m.g.nimtv, "$1 $2;$n", [getTypeDesc(m, s.loc.t), s.loc.r])
+      m.g.nimtv.addf("$1 $2;$n", [getTypeDesc(m, s.loc.t), s.loc.r])
   else:
-    if isExtern: add(m.s[cfsVars], "extern ")
-    if optThreads in m.config.globalOptions: add(m.s[cfsVars], "NIM_THREADVAR ")
-    add(m.s[cfsVars], getTypeDesc(m, s.loc.t))
-    addf(m.s[cfsVars], " $1;$n", [s.loc.r])
+    if isExtern: m.s[cfsVars].add("extern ")
+    elif lfExportLib in s.loc.flags: m.s[cfsVars].add("N_LIB_EXPORT_VAR ")
+    else: m.s[cfsVars].add("N_LIB_PRIVATE ")
+    if optThreads in m.config.globalOptions: m.s[cfsVars].add("NIM_THREADVAR ")
+    m.s[cfsVars].add(getTypeDesc(m, s.loc.t))
+    m.s[cfsVars].addf(" $1;$n", [s.loc.r])
 
 proc generateThreadLocalStorage(m: BModule) =
   if m.g.nimtv != nil and (usesThreadVars in m.flags or sfMainModule in m.module.flags):
     for t in items(m.g.nimtvDeps): discard getTypeDesc(m, t)
-    addf(m.s[cfsSeqTypes], "typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
+    finishTypeDescriptions(m)
+    m.s[cfsSeqTypes].addf("typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
 
 proc generateThreadVarsSize(m: BModule) =
   if m.g.nimtv != nil:
-    let externc = if m.config.cmd == cmdCompileToCpp or
+    let externc = if m.config.backend == backendCpp or
                        sfCompileToCpp in m.module.flags: "extern \"C\" "
                   else: ""
-    addf(m.s[cfsProcs],
+    m.s[cfsProcs].addf(
       "$#NI NimThreadVarsSize(){return (NI)sizeof(NimThreadVars);}$n",
       [externc.rope])

compiler/ccgtrav.nim

@@ -28,18 +28,18 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
   if n == nil: return
   case n.kind
   of nkRecList:
-    for i in 0 ..< len(n):
-      genTraverseProc(c, accessor, n.sons[i], typ)
+    for i in 0..<n.len:
+      genTraverseProc(c, accessor, n[i], typ)
   of nkRecCase:
-    if (n.sons[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
+    if (n[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
     var p = c.p
-    let disc = n.sons[0].sym
+    let disc = n[0].sym
     if disc.loc.r == nil: fillObjectFields(c.p.module, typ)
     if disc.loc.t == nil:
       internalError(c.p.config, n.info, "genTraverseProc()")
     lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
-    for i in 1 ..< len(n):
-      let branch = n.sons[i]
+    for i in 1..<n.len:
+      let branch = n[i]
       assert branch.kind in {nkOfBranch, nkElse}
       if branch.kind == nkOfBranch:
         genCaseRange(c.p, branch)
@@ -73,33 +73,33 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType) =
      tySink, tyOwned:
     genTraverseProc(c, accessor, lastSon(typ))
   of tyArray:
-    let arraySize = lengthOrd(c.p.config, typ.sons[0])
+    let arraySize = lengthOrd(c.p.config, typ[0])
     var i: TLoc
-    getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo(), tyInt), i)
+    getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
     let oldCode = p.s(cpsStmts)
     linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
             [i.r, arraySize])
     let oldLen = p.s(cpsStmts).len
-    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", [accessor, i.r]), typ.sons[1])
+    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", [accessor, i.r]), typ[1])
     if p.s(cpsStmts).len == oldLen:
       # do not emit dummy long loops for faster debug builds:
       p.s(cpsStmts) = oldCode
     else:
       lineF(p, cpsStmts, "}$n", [])
   of tyObject:
-    for i in 0 ..< len(typ):
-      var x = typ.sons[i]
+    for i in 0..<typ.len:
+      var x = typ[i]
       if x != nil: x = x.skipTypes(skipPtrs)
       genTraverseProc(c, accessor.parentObj(c.p.module), x)
     if typ.n != nil: genTraverseProc(c, accessor, typ.n, typ)
   of tyTuple:
     let typ = getUniqueType(typ)
-    for i in 0 ..< len(typ):
-      genTraverseProc(c, ropecg(c.p.module, "$1.Field$2", [accessor, i]), typ.sons[i])
+    for i in 0..<typ.len:
+      genTraverseProc(c, ropecg(c.p.module, "$1.Field$2", [accessor, i]), typ[i])
   of tyRef:
     lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
   of tySequence:
-    if tfHasAsgn notin typ.flags:
+    if optSeqDestructors notin c.p.module.config.globalOptions:
       lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
     elif containsGarbageCollectedRef(typ.lastSon):
       # destructor based seqs are themselves not traced but their data is, if
@@ -119,7 +119,7 @@ proc genTraverseProcSeq(c: TTraversalClosure, accessor: Rope, typ: PType) =
   var p = c.p
   assert typ.kind == tySequence
   var i: TLoc
-  getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo(), tyInt), i)
+  getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
   let oldCode = p.s(cpsStmts)
   var a: TLoc
   a.r = accessor
@@ -127,7 +127,7 @@ proc genTraverseProcSeq(c: TTraversalClosure, accessor: Rope, typ: PType) =
   lineF(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
       [i.r, lenExpr(c.p, a)])
   let oldLen = p.s(cpsStmts).len
-  genTraverseProc(c, "$1$3[$2]" % [accessor, i.r, dataField(c.p)], typ.sons[0])
+  genTraverseProc(c, "$1$3[$2]" % [accessor, i.r, dataField(c.p)], typ[0])
   if p.s(cpsStmts).len == oldLen:
     # do not emit dummy long loops for faster debug builds:
     p.s(cpsStmts) = oldCode
@@ -155,11 +155,11 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope =
   if typ.kind == tySequence:
     genTraverseProcSeq(c, "a".rope, typ)
   else:
-    if skipTypes(typ.sons[0], typedescInst+{tyOwned}).kind == tyArray:
+    if skipTypes(typ[0], typedescInst+{tyOwned}).kind == tyArray:
       # C's arrays are broken beyond repair:
-      genTraverseProc(c, "a".rope, typ.sons[0])
+      genTraverseProc(c, "a".rope, typ[0])
     else:
-      genTraverseProc(c, "(*a)".rope, typ.sons[0])
+      genTraverseProc(c, "(*a)".rope, typ[0])
 
   let generatedProc = "$1 {$n$2$3$4}\n" %
         [header, p.s(cpsLocals), p.s(cpsInit), p.s(cpsStmts)]
@@ -168,12 +168,12 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope =
   m.s[cfsProcs].add(generatedProc)
 
   if hcrOn:
-    addf(m.s[cfsProcHeaders], "N_NIMCALL_PTR(void, $1)(void*, NI);\n", [result])
-    addf(m.s[cfsDynLibInit], "\t$1 = (N_NIMCALL_PTR(void, )(void*, NI)) hcrRegisterProc($3, \"$1\", (void*)$2);\n",
+    m.s[cfsProcHeaders].addf("N_NIMCALL_PTR(void, $1)(void*, NI);\n", [result])
+    m.s[cfsDynLibInit].addf("\t$1 = (N_NIMCALL_PTR(void, )(void*, NI)) hcrRegisterProc($3, \"$1\", (void*)$2);\n",
          [result, markerName, getModuleDllPath(m)])
 
 proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope =
-  discard genTypeInfo(m, s.loc.t, info)
+  discard genTypeInfoV1(m, s.loc.t, info)
 
   var c: TTraversalClosure
   var p = newProc(nil, m)

compiler/ccgutils.nim

@@ -16,11 +16,11 @@ import
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
   case n.kind
   of nkStmtList:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       result = getPragmaStmt(n[i], w)
       if result != nil: break
   of nkPragma:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       if whichPragma(n[i]) == w: return n[i]
   else: discard
 
@@ -33,7 +33,7 @@ proc hashString*(conf: ConfigRef; s: string): BiggestInt =
     # we have to use the same bitwidth
     # as the target CPU
     var b = 0'u64
-    for i in 0 ..< len(s):
+    for i in 0..<s.len:
       b = b + uint(s[i])
       b = b + (b shl 10)
       b = b xor (b shr 6)
@@ -43,7 +43,7 @@ proc hashString*(conf: ConfigRef; s: string): BiggestInt =
     result = cast[Hash](b)
   else:
     var a = 0'u32
-    for i in 0 ..< len(s):
+    for i in 0..<s.len:
       a = a + uint32(s[i])
       a = a + (a shl 10)
       a = a xor (a shr 6)
@@ -70,18 +70,18 @@ proc mangle*(name: string): string =
   template special(x) =
     result.add x
     requiresUnderscore = true
-  for i in start..(name.len-1):
+  for i in start..<name.len:
     let c = name[i]
     case c
     of 'a'..'z', '0'..'9', 'A'..'Z':
-      add(result, c)
+      result.add(c)
     of '_':
       # we generate names like 'foo_9' for scope disambiguations and so
       # disallow this here:
       if i > 0 and i < name.len-1 and name[i+1] in Digits:
         discard
       else:
-        add(result, c)
+        result.add(c)
     of '$': special "dollar"
     of '%': special "percent"
     of '&': special "amp"
@@ -102,7 +102,7 @@ proc mangle*(name: string): string =
     of '@': special "at"
     of '|': special "bar"
     else:
-      add(result, "X" & toHex(ord(c), 2))
+      result.add("X" & toHex(ord(c), 2))
       requiresUnderscore = true
   if requiresUnderscore:
     result.add "_"

compiler/cgendata.nim

@@ -11,7 +11,7 @@
 
 import
   ast, ropes, options, intsets,
-  tables, ndi, lineinfos, pathutils, modulegraphs
+  tables, ndi, lineinfos, pathutils, modulegraphs, sets
 
 type
   TLabel* = Rope              # for the C generator a label is just a rope
@@ -25,10 +25,10 @@ type
                               # this is needed for strange type generation
                               # reasons
     cfsFieldInfo,             # section for field information
-    cfsTypeInfo,              # section for type information
+    cfsTypeInfo,              # section for type information (ag ABI checks)
     cfsProcHeaders,           # section for C procs prototypes
-    cfsVars,                  # section for C variable declarations
     cfsData,                  # section for C constant data
+    cfsVars,                  # section for C variable declarations
     cfsProcs,                 # section for C procs that are not inline
     cfsInitProc,              # section for the C init proc
     cfsDatInitProc,           # section for the C datInit proc
@@ -64,16 +64,21 @@ type
     nestedExceptStmts*: int16 # how many except statements is it nested into
     frameLen*: int16
 
+  TCProcFlag* = enum
+    beforeRetNeeded,
+    threadVarAccessed,
+    hasCurFramePointer,
+    noSafePoints,
+    nimErrorFlagAccessed,
+    nimErrorFlagDeclared,
+    nimErrorFlagDisabled
+
   TCProc = object             # represents C proc that is currently generated
     prc*: PSym                # the Nim proc that this C proc belongs to
-    beforeRetNeeded*: bool    # true iff 'BeforeRet' label for proc is needed
-    threadVarAccessed*: bool  # true if the proc already accessed some threadvar
-    hasCurFramePointer*: bool # true if _nimCurFrame var needed to recover after
-                              # exception is generated
-    noSafePoints*: bool       # the proc doesn't use safe points in exception handling
+    flags*: set[TCProcFlag]
     lastLineInfo*: TLineInfo  # to avoid generating excessive 'nimln' statements
     currLineInfo*: TLineInfo  # AST codegen will make this superfluous
-    nestedTryStmts*: seq[tuple[fin: PNode, inExcept: bool]]
+    nestedTryStmts*: seq[tuple[fin: PNode, inExcept: bool, label: Natural]]
                               # in how many nested try statements we are
                               # (the vars must be volatile then)
                               # bool is true when are in the except part of a try block
@@ -86,14 +91,13 @@ type
     options*: TOptions        # options that should be used for code
                               # generation; this is the same as prc.options
                               # unless prc == nil
-    maxFrameLen*: int         # max length of frame descriptor
     module*: BModule          # used to prevent excessive parameter passing
     withinLoop*: int          # > 0 if we are within a loop
     splitDecls*: int          # > 0 if we are in some context for C++ that
                               # requires 'T x = T()' to become 'T x; x = T()'
                               # (yes, C++ is weird like that)
-    gcFrameId*: Natural       # for the GC stack marking
-    gcFrameType*: Rope        # the struct {} we put the GC markers into
+    withinTryWithExcept*: int # required for goto based exception handling
+    withinBlockLeaveActions*: int # complex to explain
     sigConflicts*: CountTable[string]
 
   TTypeSeq* = seq[PType]
@@ -118,6 +122,7 @@ type
     forwardedProcs*: seq[PSym] # proc:s that did not yet have a body
     generatedHeader*: BModule
     typeInfoMarker*: TypeCacheWithOwner
+    typeInfoMarkerV2*: TypeCacheWithOwner
     config*: ConfigRef
     graph*: ModuleGraph
     strVersion*, seqVersion*: int # version of the string/seq implementation to use
@@ -142,11 +147,16 @@ type
                               # without extension)
     tmpBase*: Rope            # base for temp identifier generation
     typeCache*: TypeCache     # cache the generated types
+    typeABICache*: HashSet[SigHash] # cache for ABI checks; reusing typeCache
+                              # would be ideal but for some reason enums
+                              # don't seem to get cached so it'd generate
+                              # 1 ABI check per occurence in code
     forwTypeCache*: TypeCache # cache for forward declarations of types
     declaredThings*: IntSet   # things we have declared in this .c file
     declaredProtos*: IntSet   # prototypes we have declared in this .c file
     headerFiles*: seq[string] # needed headers to include
     typeInfoMarker*: TypeCache # needed for generating type information
+    typeInfoMarkerV2*: TypeCache
     initProc*: BProc          # code for init procedure
     preInitProc*: BProc       # code executed before the init proc
     hcrCreateTypeInfosProc*: Rope # type info globals are in here when HCR=on
@@ -172,7 +182,7 @@ proc includeHeader*(this: BModule; header: string) =
 
 proc s*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # section in the current block
-  result = p.blocks[p.blocks.len-1].sections[s]
+  result = p.blocks[^1].sections[s]
 
 proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # top level proc sections
@@ -182,8 +192,8 @@ proc newProc*(prc: PSym, module: BModule): BProc =
   new(result)
   result.prc = prc
   result.module = module
-  if prc != nil: result.options = prc.options
-  else: result.options = module.config.options
+  result.options = if prc != nil: prc.options
+                   else: module.config.options
   newSeq(result.blocks, 1)
   result.nestedTryStmts = @[]
   result.finallySafePoints = @[]

compiler/cgmeth.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-## This module implements code generation for multi methods.
+## This module implements code generation for methods.
 
 import
   intsets, options, ast, msgs, idents, renderer, types, magicsys,
@@ -23,11 +23,11 @@ proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
       result = n
     elif diff < 0:
       result = newNodeIT(nkObjUpConv, n.info, d)
-      addSon(result, n)
+      result.add n
       if downcast: internalError(conf, n.info, "cgmeth.genConv: no upcast allowed")
     elif diff > 0:
       result = newNodeIT(nkObjDownConv, n.info, d)
-      addSon(result, n)
+      result.add n
       if not downcast:
         internalError(conf, n.info, "cgmeth.genConv: no downcast allowed")
     else:
@@ -44,26 +44,26 @@ proc getDispatcher*(s: PSym): PSym =
 proc methodCall*(n: PNode; conf: ConfigRef): PNode =
   result = n
   # replace ordinary method by dispatcher method:
-  let disp = getDispatcher(result.sons[0].sym)
+  let disp = getDispatcher(result[0].sym)
   if disp != nil:
-    result.sons[0].sym = disp
+    result[0].sym = disp
     # change the arguments to up/downcasts to fit the dispatcher's parameters:
-    for i in 1 ..< len(result):
-      result.sons[i] = genConv(result.sons[i], disp.typ.sons[i], true, conf)
+    for i in 1..<result.len:
+      result[i] = genConv(result[i], disp.typ[i], true, conf)
   else:
-    localError(conf, n.info, "'" & $result.sons[0] & "' lacks a dispatcher")
+    localError(conf, n.info, "'" & $result[0] & "' lacks a dispatcher")
 
 type
   MethodResult = enum No, Invalid, Yes
 
 proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
   if a.name.id != b.name.id: return
-  if len(a.typ) != len(b.typ):
+  if a.typ.len != b.typ.len:
     return
 
-  for i in 1 ..< len(a.typ):
-    var aa = a.typ.sons[i]
-    var bb = b.typ.sons[i]
+  for i in 1..<a.typ.len:
+    var aa = a.typ[i]
+    var bb = b.typ[i]
     while true:
       aa = skipTypes(aa, {tyGenericInst, tyAlias})
       bb = skipTypes(bb, {tyGenericInst, tyAlias})
@@ -72,7 +72,7 @@ proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
         bb = bb.lastSon
       else:
         break
-    if sameType(a.typ.sons[i], b.typ.sons[i]):
+    if sameType(a.typ[i], b.typ[i]):
       if aa.kind == tyObject and result != Invalid:
         result = Yes
     elif aa.kind == tyObject and bb.kind == tyObject and (i == 1 or multiMethods):
@@ -90,22 +90,22 @@ proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
       return No
   if result == Yes:
     # check for return type:
-    if not sameTypeOrNil(a.typ.sons[0], b.typ.sons[0]):
-      if b.typ.sons[0] != nil and b.typ.sons[0].kind == tyUntyped:
+    if not sameTypeOrNil(a.typ[0], b.typ[0]):
+      if b.typ[0] != nil and b.typ[0].kind == tyUntyped:
         # infer 'auto' from the base to make it consistent:
-        b.typ.sons[0] = a.typ.sons[0]
+        b.typ[0] = a.typ[0]
       else:
         return No
 
 proc attachDispatcher(s: PSym, dispatcher: PNode) =
   if dispatcherPos < s.ast.len:
     # we've added a dispatcher already, so overwrite it
-    s.ast.sons[dispatcherPos] = dispatcher
+    s.ast[dispatcherPos] = dispatcher
   else:
     setLen(s.ast.sons, dispatcherPos+1)
     if s.ast[resultPos] == nil:
       s.ast[resultPos] = newNodeI(nkEmpty, s.info)
-    s.ast.sons[dispatcherPos] = dispatcher
+    s.ast[dispatcherPos] = dispatcher
 
 proc createDispatcher(s: PSym): PSym =
   var disp = copySym(s)
@@ -113,18 +113,18 @@ proc createDispatcher(s: PSym): PSym =
   excl(disp.flags, sfExported)
   disp.typ = copyType(disp.typ, disp.typ.owner, false)
   # we can't inline the dispatcher itself (for now):
-  if disp.typ.callConv == ccInline: disp.typ.callConv = ccDefault
+  if disp.typ.callConv == ccInline: disp.typ.callConv = ccNimCall
   disp.ast = copyTree(s.ast)
-  disp.ast.sons[bodyPos] = newNodeI(nkEmpty, s.info)
+  disp.ast[bodyPos] = newNodeI(nkEmpty, s.info)
   disp.loc.r = nil
-  if s.typ.sons[0] != nil:
+  if s.typ[0] != nil:
     if disp.ast.len > resultPos:
-      disp.ast.sons[resultPos].sym = copySym(s.ast.sons[resultPos].sym)
+      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym)
     else:
       # We've encountered a method prototype without a filled-in
       # resultPos slot. We put a placeholder in there that will
       # be updated in fixupDispatcher().
-      disp.ast.addSon(newNodeI(nkEmpty, s.info))
+      disp.ast.add newNodeI(nkEmpty, s.info)
   attachDispatcher(s, newSymNode(disp))
   # attach to itself to prevent bugs:
   attachDispatcher(disp, newSymNode(disp))
@@ -137,8 +137,8 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
   # the lock level of the dispatcher needs to be updated/checked
   # against that of the method.
   if disp.ast.len > resultPos and meth.ast.len > resultPos and
-     disp.ast.sons[resultPos].kind == nkEmpty:
-    disp.ast.sons[resultPos] = copyTree(meth.ast.sons[resultPos])
+     disp.ast[resultPos].kind == nkEmpty:
+    disp.ast[resultPos] = copyTree(meth.ast[resultPos])
 
   # The following code works only with lock levels, so we disable
   # it when they're not available.
@@ -158,13 +158,12 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
         disp.typ.lockLevel = meth.typ.lockLevel
 
 proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
-  let L = len(g.methods)
   var witness: PSym
-  for i in 0 ..< L:
+  for i in 0..<g.methods.len:
     let disp = g.methods[i].dispatcher
     case sameMethodBucket(disp, s, multimethods = optMultiMethods in g.config.globalOptions)
     of Yes:
-      add(g.methods[i].methods, s)
+      g.methods[i].methods.add(s)
       attachDispatcher(s, disp.ast[dispatcherPos])
       fixupDispatcher(s, disp, g.config)
       #echo "fixup ", disp.name.s, " ", disp.id
@@ -178,7 +177,7 @@ proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
     of Invalid:
       if witness.isNil: witness = g.methods[i].methods[0]
   # create a new dispatcher:
-  add(g.methods, (methods: @[s], dispatcher: createDispatcher(s)))
+  g.methods.add((methods: @[s], dispatcher: createDispatcher(s)))
   #echo "adding ", s.info
   #if fromCache:
   #  internalError(s.info, "no method dispatcher found")
@@ -190,32 +189,32 @@ proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
 
 proc relevantCol(methods: seq[PSym], col: int): bool =
   # returns true iff the position is relevant
-  var t = methods[0].typ.sons[col].skipTypes(skipPtrs)
+  var t = methods[0].typ[col].skipTypes(skipPtrs)
   if t.kind == tyObject:
-    for i in 1 .. high(methods):
-      let t2 = skipTypes(methods[i].typ.sons[col], skipPtrs)
+    for i in 1..high(methods):
+      let t2 = skipTypes(methods[i].typ[col], skipPtrs)
       if not sameType(t2, t):
         return true
 
 proc cmpSignatures(a, b: PSym, relevantCols: IntSet): int =
-  for col in 1 ..< len(a.typ):
+  for col in 1..<a.typ.len:
     if contains(relevantCols, col):
-      var aa = skipTypes(a.typ.sons[col], skipPtrs)
-      var bb = skipTypes(b.typ.sons[col], skipPtrs)
+      var aa = skipTypes(a.typ[col], skipPtrs)
+      var bb = skipTypes(b.typ[col], skipPtrs)
       var d = inheritanceDiff(aa, bb)
       if (d != high(int)) and d != 0:
         return d
 
 proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
   # we use shellsort here; fast and simple
-  var n = len(a)
+  var n = a.len
   var h = 1
   while true:
     h = 3 * h + 1
     if h > n: break
   while true:
     h = h div 3
-    for i in h ..< n:
+    for i in h..<n:
       var v = a[i]
       var j = i
       while cmpSignatures(a[j - h], v, relevantCols) >= 0:
@@ -228,71 +227,70 @@ proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
 proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet): PSym =
   var base = methods[0].ast[dispatcherPos].sym
   result = base
-  var paramLen = len(base.typ)
+  var paramLen = base.typ.len
   var nilchecks = newNodeI(nkStmtList, base.info)
   var disp = newNodeI(nkIfStmt, base.info)
-  var ands = getSysMagic(g, unknownLineInfo(), "and", mAnd)
-  var iss = getSysMagic(g, unknownLineInfo(), "of", mOf)
-  let boolType = getSysType(g, unknownLineInfo(), tyBool)
-  for col in 1 ..< paramLen:
+  var ands = getSysMagic(g, unknownLineInfo, "and", mAnd)
+  var iss = getSysMagic(g, unknownLineInfo, "of", mOf)
+  let boolType = getSysType(g, unknownLineInfo, tyBool)
+  for col in 1..<paramLen:
     if contains(relevantCols, col):
-      let param = base.typ.n.sons[col].sym
+      let param = base.typ.n[col].sym
       if param.typ.skipTypes(abstractInst).kind in {tyRef, tyPtr}:
-        addSon(nilchecks, newTree(nkCall,
-            newSymNode(getCompilerProc(g, "chckNilDisp")), newSymNode(param)))
-  for meth in 0 .. high(methods):
+        nilchecks.add newTree(nkCall,
+            newSymNode(getCompilerProc(g, "chckNilDisp")), newSymNode(param))
+  for meth in 0..high(methods):
     var curr = methods[meth]      # generate condition:
     var cond: PNode = nil
-    for col in 1 ..< paramLen:
+    for col in 1..<paramLen:
       if contains(relevantCols, col):
         var isn = newNodeIT(nkCall, base.info, boolType)
-        addSon(isn, newSymNode(iss))
-        let param = base.typ.n.sons[col].sym
-        addSon(isn, newSymNode(param))
-        addSon(isn, newNodeIT(nkType, base.info, curr.typ.sons[col]))
+        isn.add newSymNode(iss)
+        let param = base.typ.n[col].sym
+        isn.add newSymNode(param)
+        isn.add newNodeIT(nkType, base.info, curr.typ[col])
         if cond != nil:
           var a = newNodeIT(nkCall, base.info, boolType)
-          addSon(a, newSymNode(ands))
-          addSon(a, cond)
-          addSon(a, isn)
+          a.add newSymNode(ands)
+          a.add cond
+          a.add isn
           cond = a
         else:
           cond = isn
-    let retTyp = base.typ.sons[0]
+    let retTyp = base.typ[0]
     let call = newNodeIT(nkCall, base.info, retTyp)
-    addSon(call, newSymNode(curr))
-    for col in 1 ..< paramLen:
-      addSon(call, genConv(newSymNode(base.typ.n.sons[col].sym),
-                           curr.typ.sons[col], false, g.config))
+    call.add newSymNode(curr)
+    for col in 1..<paramLen:
+      call.add genConv(newSymNode(base.typ.n[col].sym),
+                           curr.typ[col], false, g.config)
     var ret: PNode
     if retTyp != nil:
       var a = newNodeI(nkFastAsgn, base.info)
-      addSon(a, newSymNode(base.ast.sons[resultPos].sym))
-      addSon(a, call)
+      a.add newSymNode(base.ast[resultPos].sym)
+      a.add call
       ret = newNodeI(nkReturnStmt, base.info)
-      addSon(ret, a)
+      ret.add a
     else:
       ret = call
     if cond != nil:
       var a = newNodeI(nkElifBranch, base.info)
-      addSon(a, cond)
-      addSon(a, ret)
-      addSon(disp, a)
+      a.add cond
+      a.add ret
+      disp.add a
     else:
       disp = ret
   nilchecks.add disp
   nilchecks.flags.incl nfTransf # should not be further transformed
-  result.ast.sons[bodyPos] = nilchecks
+  result.ast[bodyPos] = nilchecks
 
 proc generateMethodDispatchers*(g: ModuleGraph): PNode =
   result = newNode(nkStmtList)
-  for bucket in 0 ..< len(g.methods):
+  for bucket in 0..<g.methods.len:
     var relevantCols = initIntSet()
-    for col in 1 ..< len(g.methods[bucket].methods[0].typ):
+    for col in 1..<g.methods[bucket].methods[0].typ.len:
       if relevantCol(g.methods[bucket].methods, col): incl(relevantCols, col)
       if optMultiMethods notin g.config.globalOptions:
         # if multi-methods are not enabled, we are interested only in the first field
         break
     sortBucket(g.methods[bucket].methods, relevantCols)
-    addSon(result,
-           newSymNode(genDispatcher(g, g.methods[bucket].methods, relevantCols)))
+    result.add newSymNode(genDispatcher(g, g.methods[bucket].methods, relevantCols))

compiler/closureiters.nim

@@ -121,7 +121,10 @@
 #   yield 2
 #   if :unrollFinally: # This node is created by `newEndFinallyNode`
 #     if :curExc.isNil:
-#       return :tmpResult
+#       if nearestFinally == 0:
+#         return :tmpResult
+#       else:
+#         :state = nearestFinally # bubble up
 #     else:
 #       closureIterSetupExc(nil)
 #       raise
@@ -130,7 +133,8 @@
 
 import
   ast, msgs, idents,
-  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
+  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos,
+  tables, options
 
 type
   Ctx = object
@@ -154,8 +158,8 @@ type
                     # is their finally. For finally it is parent finally. Otherwise -1
 
 const
-  nkSkip = { nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
-            nkCommentStmt } + procDefs
+  nkSkip = {nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
+            nkCommentStmt, nkMixinStmt, nkBindStmt} + procDefs
 
 proc newStateAccess(ctx: var Ctx): PNode =
   if ctx.stateVarSym.isNil:
@@ -237,7 +241,7 @@ proc toStmtList(n: PNode): PNode =
 proc addGotoOut(n: PNode, gotoOut: PNode): PNode =
   # Make sure `n` is a stmtlist, and ends with `gotoOut`
   result = toStmtList(n)
-  if result.len == 0 or result.sons[^1].kind != nkGotoState:
+  if result.len == 0 or result[^1].kind != nkGotoState:
     result.add(gotoOut)
 
 proc newTempVar(ctx: var Ctx, typ: PType): PSym =
@@ -273,7 +277,7 @@ proc transformBreaksAndContinuesInWhile(ctx: var Ctx, n: PNode, before, after: P
     if ctx.blockLevel == 0:
       result = after
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformBreaksAndContinuesInWhile(n[i], before, after)
 
 proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode =
@@ -293,7 +297,7 @@ proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode
       if label.kind == nkSym and n[0].sym == label.sym:
         result = after
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformBreaksInBlock(n[i], label, after)
 
 proc newNullifyCurExc(ctx: var Ctx, info: TLineInfo): PNode =
@@ -318,7 +322,7 @@ proc collectExceptState(ctx: var Ctx, n: PNode): PNode {.inline.} =
 
       if c.len > 1:
         var cond: PNode
-        for i in 0 .. c.len - 2:
+        for i in 0..<c.len - 1:
           assert(c[i].kind == nkType)
           let nextCond = newTree(nkCall,
             newSymNode(g.getSysMagic(c.info, "of", mOf)),
@@ -388,6 +392,10 @@ proc hasYieldsInExpressions(n: PNode): bool =
           return true
     else:
       result = n.hasYields
+  of nkCast:
+    for i in 1..<n.len:
+      if n[i].hasYieldsInExpressions:
+        return true
   else:
     for c in n:
       if c.hasYieldsInExpressions:
@@ -401,7 +409,7 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
   var n = n
   while n.kind == nkStmtListExpr:
     result.s.sons.add(n.sons)
-    result.s.sons.setLen(result.s.sons.len - 1) # delete last son
+    result.s.sons.setLen(result.s.len - 1) # delete last son
     n = n[^1]
 
   result.res = n
@@ -435,7 +443,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkYieldStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -449,7 +457,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkPar, nkObjConstr, nkTupleConstr, nkBracket:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -459,7 +467,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       if n.typ.isNil: internalError(ctx.g.config, "lowerStmtListExprs: constr typ.isNil")
       result.typ = n.typ
 
-      for i in 0 ..< n.len:
+      for i in 0..<n.len:
         case n[i].kind
         of nkExprColonExpr:
           if n[i][1].kind == nkStmtListExpr:
@@ -475,13 +483,12 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkIfStmt, nkIfExpr:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
       needsSplit = true
       var tmp: PSym
-      var s: PNode
       let isExpr = not isEmptyType(n.typ)
       if isExpr:
         tmp = ctx.newTempVar(n.typ)
@@ -541,7 +548,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkTryStmt, nkHiddenTryStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -554,7 +561,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         let tmp = ctx.newTempVar(n.typ)
 
         n[0] = ctx.convertExprBodyToAsgn(n[0], tmp)
-        for i in 1 ..< n.len:
+        for i in 1..<n.len:
           let branch = n[i]
           case branch.kind
           of nkExceptBranch:
@@ -571,7 +578,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkCaseStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -589,7 +596,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           result.add(st)
           n[0] = ex
 
-        for i in 1 ..< n.len:
+        for i in 1..<n.len:
           let branch = n[i]
           case branch.kind
           of nkOfBranch:
@@ -600,10 +607,16 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
             internalError(ctx.g.config, "lowerStmtListExpr(nkCaseStmt): " & $branch.kind)
         result.add(n)
         result.add(ctx.newEnvVarAccess(tmp))
+      elif n[0].kind == nkStmtListExpr:
+        result = newNodeI(nkStmtList, n.info)
+        let (st, ex) = exprToStmtList(n[0])
+        result.add(st)
+        n[0] = ex
+        result.add(n)
 
   of nkCallKinds, nkChckRange, nkChckRangeF, nkChckRange64:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -643,7 +656,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         result.add(ifNode)
         result.add(ctx.newEnvVarAccess(tmp))
       else:
-        for i in 0 ..< n.len:
+        for i in 0..<n.len:
           if n[i].kind == nkStmtListExpr:
             let (st, ex) = exprToStmtList(n[i])
             result.add(st)
@@ -672,7 +685,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkDiscardStmt, nkReturnStmt, nkRaiseStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -686,7 +699,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
   of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv, nkObjDownConv,
       nkDerefExpr, nkHiddenDeref:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in ord(n.kind == nkCast)..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -700,7 +713,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkAsgn, nkFastAsgn:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -738,8 +751,6 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       result.add(n)
 
   of nkWhileStmt:
-    var ns = false
-
     var condNeedsSplit = false
     n[0] = ctx.lowerStmtListExprs(n[0], condNeedsSplit)
     var bodyNeedsSplit = false
@@ -778,21 +789,24 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       result = newNodeI(nkStmtListExpr, n.info)
       result.typ = n.typ
       let (st, ex) = exprToStmtList(n[1])
-      n.kind = nkBlockStmt
+      n.transitionSonsKind(nkBlockStmt)
       n.typ = nil
       n[1] = st
       result.add(n)
       result.add(ex)
 
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], needsSplit)
 
 proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   # Generate the following code:
   #   if :unrollFinally:
   #       if :curExc.isNil:
-  #         return :tmpResult
+  #         if nearestFinally == 0:
+  #           return :tmpResult
+  #         else:
+  #           :state = nearestFinally # bubble up
   #       else:
   #         raise
   let curExc = ctx.newCurExcAccess()
@@ -801,11 +815,17 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   let cmp = newTree(nkCall, newSymNode(ctx.g.getSysMagic(info, "==", mEqRef), info), curExc, nilnode)
   cmp.typ = ctx.g.getSysType(info, tyBool)
 
-  let asgn = newTree(nkFastAsgn,
-    newSymNode(getClosureIterResult(ctx.g, ctx.fn), info),
-    ctx.newTmpResultAccess())
+  let retStmt =
+    if ctx.nearestFinally == 0:
+      # last finally, we can return
+      let asgn = newTree(nkFastAsgn,
+        newSymNode(getClosureIterResult(ctx.g, ctx.fn), info),
+        ctx.newTmpResultAccess())
+      newTree(nkReturnStmt, asgn)
+    else:
+      # bubble up to next finally
+      newTree(nkGotoState, ctx.g.newIntLit(info, ctx.nearestFinally))
 
-  let retStmt = newTree(nkReturnStmt, asgn)
   let branch = newTree(nkElifBranch, cmp, retStmt)
 
   let nullifyExc = newTree(nkCall, newSymNode(ctx.g.getCompilerProc("closureIterSetupExc")), nilnode)
@@ -838,7 +858,8 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
     if n[0].kind != nkEmpty:
       let asgnTmpResult = newNodeI(nkAsgn, n.info)
       asgnTmpResult.add(ctx.newTmpResultAccess())
-      asgnTmpResult.add(n[0])
+      let x = if n[0].kind in {nkAsgn, nkFastAsgn}: n[0][1] else: n[0]
+      asgnTmpResult.add(x)
       result.add(asgnTmpResult)
 
     result.add(ctx.newNullifyCurExc(n.info))
@@ -848,8 +869,15 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
 
   of nkSkip:
     discard
+  of nkTryStmt:
+    if n.hasYields:
+      # the inner try will handle these transformations
+      discard
+    else:
+      for i in 0..<n.len:
+        n[i] = ctx.transformReturnsInTry(n[i])
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformReturnsInTry(n[i])
 
 proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode =
@@ -859,14 +887,14 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
 
   of nkStmtList, nkStmtListExpr:
     result = addGotoOut(result, gotoOut)
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       if n[i].hasYields:
         # Create a new split
         let go = newNodeI(nkGotoState, n[i].info)
         n[i] = ctx.transformClosureIteratorBody(n[i], go)
 
         let s = newNodeI(nkStmtList, n[i + 1].info)
-        for j in i + 1 ..< n.len:
+        for j in i + 1..<n.len:
           s.add(n[j])
 
         n.sons.setLen(i + 1)
@@ -889,7 +917,7 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
     result[^1] = ctx.transformClosureIteratorBody(result[^1], gotoOut)
 
   of nkIfStmt, nkCaseStmt:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
     if n[^1].kind != nkElse:
       # We don't have an else branch, but every possible branch has to end with
@@ -1001,7 +1029,7 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
     internalError(ctx.g.config, "closure iter " & $n.kind)
 
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
 
 proc stateFromGotoState(n: PNode): int =
@@ -1038,18 +1066,18 @@ proc transformStateAssignments(ctx: var Ctx, n: PNode): PNode =
       result.add(ctx.newStateAssgn(stateFromGotoState(n[1])))
 
       var retStmt = newNodeI(nkReturnStmt, n.info)
-      if n[0].sons[0].kind != nkEmpty:
-        var a = newNodeI(nkAsgn, n[0].sons[0].info)
-        var retVal = n[0].sons[0] #liftCapturedVars(n.sons[0], owner, d, c)
-        addSon(a, newSymNode(getClosureIterResult(ctx.g, ctx.fn)))
-        addSon(a, retVal)
+      if n[0][0].kind != nkEmpty:
+        var a = newNodeI(nkAsgn, n[0][0].info)
+        var retVal = n[0][0] #liftCapturedVars(n[0], owner, d, c)
+        a.add newSymNode(getClosureIterResult(ctx.g, ctx.fn))
+        a.add retVal
         retStmt.add(a)
       else:
         retStmt.add(ctx.g.emptyNode)
 
       result.add(retStmt)
     else:
-      for i in 0 ..< n.len:
+      for i in 0..<n.len:
         n[i] = ctx.transformStateAssignments(n[i])
 
   of nkSkip:
@@ -1069,7 +1097,7 @@ proc transformStateAssignments(ctx: var Ctx, n: PNode): PNode =
     result.add(breakState)
 
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformStateAssignments(n[i])
 
 proc skipStmtList(ctx: Ctx; n: PNode): PNode =
@@ -1101,7 +1129,7 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
 
   result = ctx.states[stateIdx][0].intVal.int
 
-proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
+proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
   result = n
   case n.kind
   of nkSkip:
@@ -1110,7 +1138,7 @@ proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
     result = copyTree(n)
     result[0].intVal = ctx.skipEmptyStates(result[0].intVal.int)
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.skipThroughEmptyStates(n[i])
 
 proc newArrayType(g: ModuleGraph; n: int, t: PType, owner: PSym): PType =
@@ -1281,6 +1309,101 @@ proc deleteEmptyStates(ctx: var Ctx) =
     else:
       inc i
 
+type
+  PreprocessContext = object
+    finallys: seq[PNode]
+    config: ConfigRef
+    blocks: seq[(PNode, int)]
+  FreshVarsContext = object
+    tab: Table[int, PSym]
+    config: ConfigRef
+    info: TLineInfo
+
+proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
+  case n.kind
+  of nkSym:
+    let x = c.tab.getOrDefault(n.sym.id)
+    if x == nil:
+      result = n
+    else:
+      result = newSymNode(x, n.info)
+  of nkSkip - {nkSym}:
+    result = n
+  of nkLetSection, nkVarSection:
+    result = copyNode(n)
+    for it in n:
+      if it.kind in {nkIdentDefs, nkVarTuple}:
+        let idefs = copyNode(it)
+        for v in 0..it.len-3:
+          if it[v].kind == nkSym:
+            let x = copySym(it[v].sym)
+            c.tab[it[v].sym.id] = x
+            idefs.add newSymNode(x)
+          else:
+            idefs.add it[v]
+
+        for rest in it.len-2 ..< it.len: idefs.add it[rest]
+        result.add idefs
+      else:
+        result.add it
+  of nkRaiseStmt:
+    localError(c.config, c.info, "unsupported control flow: 'finally: ... raise' duplicated because of 'break'")
+  else:
+    result = n
+    for i in 0..<n.safeLen:
+      result[i] = freshVars(n[i], c)
+
+proc preprocess(c: var PreprocessContext; n: PNode): PNode =
+  # in order to fix bug #15243 without risking regressions, we preprocess
+  # the AST so that 'break' statements inside a 'try finally' also have the
+  # finally section. We need to duplicate local variables here and also
+  # detect: 'finally: raises X' which is currently not supported. We produce
+  # an error for this case for now. All this will be done properly with Yuriy's
+  # patch.
+  result = n
+  case n.kind
+  of nkTryStmt:
+    let f = n.lastSon
+    if f.kind == nkFinally:
+      c.finallys.add f.lastSon
+
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
+    if f.kind == nkFinally:
+      discard c.finallys.pop()
+
+  of nkWhileStmt, nkBlockStmt:
+    c.blocks.add((n, c.finallys.len))
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+    discard c.blocks.pop()
+
+  of nkBreakStmt:
+    if c.blocks.len == 0:
+      discard
+    else:
+      var fin = -1
+      if n[0].kind == nkEmpty:
+        fin = c.blocks[^1][1]
+      elif n[0].kind == nkSym:
+        for i in countdown(c.blocks.high, 0):
+          if c.blocks[i][0].kind == nkBlockStmt and c.blocks[i][0][0].kind == nkSym and
+              c.blocks[i][0][0].sym == n[0].sym:
+            fin = c.blocks[i][1]
+            break
+
+      if fin >= 0:
+        result = newNodeI(nkStmtList, n.info)
+        for i in countdown(c.finallys.high, fin):
+          var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info)
+          result.add freshVars(preprocess(c, c.finallys[i]), vars)
+        result.add n
+  of nkSkip: discard
+  else:
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
 proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   var ctx: Ctx
   ctx.g = g
@@ -1293,7 +1416,10 @@ proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
     ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), fn, fn.info)
     ctx.stateVarSym.typ = g.createClosureIterStateType(fn)
   ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), fn, fn.info)
-  var n = n.toStmtList
+  var pc = PreprocessContext(finallys: @[], config: g.config)
+  var n = preprocess(pc, n.toStmtList)
+  #echo "transformed into ", n
+  #var n = n.toStmtList
 
   discard ctx.newState(n, nil)
   let gotoOut = newTree(nkGotoState, g.newIntLit(n.info, -1))

compiler/cmdlinehelper.nim

@@ -7,20 +7,33 @@
 #    distribution, for details about the copyright.
 #
 
-## Helpers for binaries that use compiler passes, eg: nim, nimsuggest, nimfix
+## Helpers for binaries that use compiler passes, e.g.: nim, nimsuggest, nimfix
 
 import
-  options, idents, nimconf, scriptconfig, extccomp, commands, msgs,
-  lineinfos, modulegraphs, condsyms, os, pathutils
+  options, idents, nimconf, extccomp, commands, msgs,
+  lineinfos, modulegraphs, condsyms, os, pathutils, parseopt
 
 from strutils import normalize
 
+proc prependCurDir*(f: AbsoluteFile): AbsoluteFile =
+  when defined(unix):
+    if os.isAbsolute(f.string): result = f
+    else: result = AbsoluteFile("./" & f.string)
+  else:
+    result = f
+
+proc addCmdPrefix*(result: var string, kind: CmdLineKind) =
+  # consider moving this to std/parseopt
+  case kind
+  of cmdLongOption: result.add "--"
+  of cmdShortOption: result.add "-"
+  of cmdArgument, cmdEnd: discard
+
 type
   NimProg* = ref object
     suggestMode*: bool
     supportsStdinFile*: bool
     processCmdLine*: proc(pass: TCmdLinePass, cmd: string; config: ConfigRef)
-    mainCommand*: proc(graph: ModuleGraph)
 
 proc initDefinesProg*(self: NimProg, conf: ConfigRef, name: string) =
   condsyms.initDefines(conf.symbols)
@@ -29,10 +42,7 @@ proc initDefinesProg*(self: NimProg, conf: ConfigRef, name: string) =
 proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   self.processCmdLine(passCmd1, "", conf)
   if self.supportsStdinFile and conf.projectName == "-":
-    conf.projectName = "stdinfile"
-    conf.projectFull = AbsoluteFile "stdinfile"
-    conf.projectPath = AbsoluteDir getCurrentDir()
-    conf.projectIsStdin = true
+    handleStdinInput(conf)
   elif conf.projectName != "":
     try:
       conf.projectFull = canonicalizePath(conf, AbsoluteFile conf.projectName)
@@ -45,46 +55,21 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   else:
     conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir())
 
-proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef): bool =
-  loadConfigs(DefaultConfig, cache, conf) # load all config files
+proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef;
+                                   graph: ModuleGraph): bool =
   if self.suggestMode:
     conf.command = "nimsuggest"
+  loadConfigs(DefaultConfig, cache, conf) # load all config files
 
-  template runNimScriptIfExists(path: AbsoluteFile) =
-    let p = path # eval once
-    if fileExists(p):
-      runNimScript(cache, p, freshDefines = false, conf)
-
-  # Caution: make sure this stays in sync with `loadConfigs`
-  if optSkipSystemConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(getSystemConfigPath(conf, DefaultConfigNims))
-
-  if optSkipUserConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(getUserConfigPath(DefaultConfigNims))
-
-  if optSkipParentConfigFiles notin conf.globalOptions:
-    for dir in parentDirs(conf.projectPath.string, fromRoot = true, inclusive = false):
-      runNimScriptIfExists(AbsoluteDir(dir) / DefaultConfigNims)
-
-  if optSkipProjConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(conf.projectPath / DefaultConfigNims)
-  block:
+  if not self.suggestMode:
     let scriptFile = conf.projectFull.changeFileExt("nims")
-    if not self.suggestMode:
-      runNimScriptIfExists(scriptFile)
-      # 'nim foo.nims' means to just run the NimScript file and do nothing more:
-      if fileExists(scriptFile) and scriptFile == conf.projectFull:
-        if conf.command == "":
-          conf.command = "e"
-          return false
-        elif conf.command.normalize == "e":
-          return false
-    else:
-      if scriptFile != conf.projectFull:
-        runNimScriptIfExists(scriptFile)
-      else:
-        # 'nimsuggest foo.nims' means to just auto-complete the NimScript file
-        discard
+    # 'nim foo.nims' means to just run the NimScript file and do nothing more:
+    if fileExists(scriptFile) and scriptFile == conf.projectFull:
+      if conf.command == "":
+        conf.command = "e"
+        return false
+      elif conf.command.normalize == "e":
+        return false
 
   # now process command line arguments again, because some options in the
   # command line can overwrite the config file's settings
@@ -93,7 +78,5 @@ proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: Confi
   if conf.command == "":
     rawMessage(conf, errGenerated, "command missing")
 
-  let graph = newModuleGraph(cache, conf)
   graph.suggestMode = self.suggestMode
-  self.mainCommand(graph)
   return true

compiler/commands.nim

@@ -18,7 +18,8 @@ template bootSwitch(name, expr, userString) =
 
 bootSwitch(usedRelease, defined(release), "-d:release")
 bootSwitch(usedDanger, defined(danger), "-d:danger")
-bootSwitch(usedGnuReadline, defined(useLinenoise), "-d:useLinenoise")
+# `useLinenoise` deprecated in favor of `nimUseLinenoise`, kept for backward compatibility
+bootSwitch(useLinenoise, defined(nimUseLinenoise) or defined(useLinenoise), "-d:nimUseLinenoise")
 bootSwitch(usedBoehm, defined(boehmgc), "--gc:boehm")
 bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
 bootSwitch(usedGenerational, defined(gcgenerational), "--gc:generational")
@@ -31,6 +32,7 @@ import
   pathutils, strtabs
 
 from incremental import nimIncremental
+from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect
 
 # but some have deps to imported modules. Yay.
 bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
@@ -51,15 +53,14 @@ const
       "Copyright (c) 2006-" & copyrightYear & " by Andreas Rumpf\n"
 
 proc genFeatureDesc[T: enum](t: typedesc[T]): string {.compileTime.} =
-  var x = ""
-  for f in low(T)..high(T):
-    if x.len > 0: x.add "|"
-    x.add $f
-  x
+  result = ""
+  for f in T:
+    if result.len > 0: result.add "|"
+    result.add $f
 
 const
-  Usage = slurp"../doc/basicopt.txt".replace(" //", " ")
-  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", " ") % [genFeatureDesc(Feature), genFeatureDesc(LegacyFeature)]
+  Usage = slurp"../doc/basicopt.txt".replace(" //", "   ")
+  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", "   ") % [genFeatureDesc(Feature), genFeatureDesc(LegacyFeature)]
 
 proc getCommandLineDesc(conf: ConfigRef): string =
   result = (HelpMessage % [VersionAsString, platform.OS[conf.target.hostOS].name,
@@ -96,12 +97,12 @@ proc writeVersionInfo(conf: ConfigRef; pass: TCmdLinePass) =
                                  CPU[conf.target.hostCPU].name, CompileDate]),
                {msgStdout})
 
-    const gitHash = gorge("git log -n 1 --format=%H").strip
+    const gitHash {.strdefine.} = gorge("git log -n 1 --format=%H").strip
     when gitHash.len == 40:
       msgWriteln(conf, "git hash: " & gitHash, {msgStdout})
 
     msgWriteln(conf, "active boot switches:" & usedRelease & usedDanger &
-      usedTinyC & usedGnuReadline & usedNativeStacktrace &
+      usedTinyC & useLinenoise & usedNativeStacktrace &
       usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC,
                {msgStdout})
     msgQuit(0)
@@ -110,7 +111,7 @@ proc writeCommandLineUsage*(conf: ConfigRef) =
   msgWriteln(conf, getCommandLineDesc(conf), {msgStdout})
 
 proc addPrefix(switch: string): string =
-  if len(switch) <= 1: result = "-" & switch
+  if switch.len <= 1: result = "-" & switch
   else: result = "--" & switch
 
 const
@@ -127,14 +128,14 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
                  info: TLineInfo) =
   cmd = ""
   var i = 0
-  if i < len(switch) and switch[i] == '-': inc(i)
-  if i < len(switch) and switch[i] == '-': inc(i)
-  while i < len(switch):
+  if i < switch.len and switch[i] == '-': inc(i)
+  if i < switch.len and switch[i] == '-': inc(i)
+  while i < switch.len:
     case switch[i]
-    of 'a'..'z', 'A'..'Z', '0'..'9', '_', '.': add(cmd, switch[i])
+    of 'a'..'z', 'A'..'Z', '0'..'9', '_', '.': cmd.add(switch[i])
     else: break
     inc(i)
-  if i >= len(switch): arg = ""
+  if i >= switch.len: arg = ""
   # cmd:arg => (cmd,arg)
   elif switch[i] in {':', '='}: arg = substr(switch, i + 1)
   # cmd[sub]:rest => (cmd,[sub]:rest)
@@ -144,24 +145,24 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
 proc processOnOffSwitch(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass,
                         info: TLineInfo) =
   case arg.normalize
-  of "","on": conf.options = conf.options + op
-  of "off": conf.options = conf.options - op
+  of "", "on": conf.options.incl op
+  of "off": conf.options.excl op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
 proc processOnOffSwitchOrList(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass,
                               info: TLineInfo): bool =
   result = false
   case arg.normalize
-  of "on": conf.options = conf.options + op
-  of "off": conf.options = conf.options - op
+  of "on": conf.options.incl op
+  of "off": conf.options.excl op
   of "list": result = true
   else: localError(conf, info, errOnOffOrListExpectedButXFound % arg)
 
 proc processOnOffSwitchG(conf: ConfigRef; op: TGlobalOptions, arg: string, pass: TCmdLinePass,
                          info: TLineInfo) =
   case arg.normalize
-  of "", "on": conf.globalOptions = conf.globalOptions + op
-  of "off": conf.globalOptions = conf.globalOptions - op
+  of "", "on": conf.globalOptions.incl op
+  of "off": conf.globalOptions.excl op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
 proc expectArg(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
@@ -174,41 +175,53 @@ proc expectNoArg(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, info:
 
 proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
                          info: TLineInfo; orig: string; conf: ConfigRef) =
-  var id = ""  # arg = key:val or [key]:val;  with val=on|off
+  var id = ""  # arg = key or [key] or key:val or [key]:val;  with val=on|off
   var i = 0
   var n = hintMin
   var isBracket = false
-  if i < len(arg) and arg[i] == '[':
+  if i < arg.len and arg[i] == '[':
     isBracket = true
     inc(i)
-  while i < len(arg) and (arg[i] notin {':', '=', ']'}):
-    add(id, arg[i])
+  while i < arg.len and (arg[i] notin {':', '=', ']'}):
+    id.add(arg[i])
     inc(i)
   if isBracket:
-    if i < len(arg) and arg[i] == ']': inc(i)
+    if i < arg.len and arg[i] == ']': inc(i)
     else: invalidCmdLineOption(conf, pass, orig, info)
 
-  if i < len(arg) and (arg[i] in {':', '='}): inc(i)
+  if i == arg.len: discard
+  elif i < arg.len and (arg[i] in {':', '='}): inc(i)
   else: invalidCmdLineOption(conf, pass, orig, info)
   if state == wHint:
-    let x = findStr(lineinfos.HintsToStr, id)
-    if x >= 0: n = TNoteKind(x + ord(hintMin))
+    let x = findStr(hintMin..hintMax, id, errUnknown)
+    if x != errUnknown: n = TNoteKind(x)
     else: localError(conf, info, "unknown hint: " & id)
   else:
-    let x = findStr(lineinfos.WarningsToStr, id)
-    if x >= 0: n = TNoteKind(x + ord(warnMin))
+    let x = findStr(warnMin..warnMax, id, errUnknown)
+    if x != errUnknown: n = TNoteKind(x)
     else: localError(conf, info, "unknown warning: " & id)
-  case substr(arg, i).normalize
-  of "on":
-    incl(conf.notes, n)
-    incl(conf.mainPackageNotes, n)
-    incl(conf.enableNotes, n)
-  of "off":
-    excl(conf.notes, n)
-    excl(conf.mainPackageNotes, n)
-    incl(conf.disableNotes, n)
-    excl(conf.foreignPackageNotes, n)
-  else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
+
+  var val = substr(arg, i).normalize
+  if val == "": val = "on"
+  if val notin ["on", "off"]:
+    localError(conf, info, errOnOrOffExpectedButXFound % arg)
+  elif n notin conf.cmdlineNotes or pass == passCmd1:
+    if pass == passCmd1: incl(conf.cmdlineNotes, n)
+    incl(conf.modifiedyNotes, n)
+    case val
+    of "on":
+      if state == wWarningAsError:
+        incl(conf.warningAsErrors, n)
+      else:
+        incl(conf.notes, n)
+        incl(conf.mainPackageNotes, n)
+    of "off":
+      if state == wWarningAsError:
+        excl(conf.warningAsErrors, n)
+      else:
+        excl(conf.notes, n)
+        excl(conf.mainPackageNotes, n)
+        excl(conf.foreignPackageNotes, n)
 
 proc processCompile(conf: ConfigRef; filename: string) =
   var found = findFile(conf, filename)
@@ -216,22 +229,25 @@ proc processCompile(conf: ConfigRef; filename: string) =
   extccomp.addExternalFileToCompile(conf, found)
 
 const
-  errNoneBoehmRefcExpectedButXFound = "'none', 'boehm' or 'refc' expected, but '$1' found"
+  errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found"
+  errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found"
 
 proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo): bool =
   case switch.normalize
   of "gc":
     case arg.normalize
-    of "boehm":        result = conf.selectedGC == gcBoehm
-    of "refc":         result = conf.selectedGC == gcRefc
-    of "v2":           result = false
+    of "boehm": result = conf.selectedGC == gcBoehm
+    of "refc": result = conf.selectedGC == gcRefc
+    of "v2": result = false
     of "markandsweep": result = conf.selectedGC == gcMarkAndSweep
     of "generational": result = false
-    of "destructors":  result = conf.selectedGC == gcDestructors
-    of "go":           result = conf.selectedGC == gcGo
-    of "none":         result = conf.selectedGC == gcNone
+    of "destructors", "arc": result = conf.selectedGC == gcArc
+    of "orc": result = conf.selectedGC == gcOrc
+    of "hooks": result = conf.selectedGC == gcHooks
+    of "go": result = conf.selectedGC == gcGo
+    of "none": result = conf.selectedGC == gcNone
     of "stack", "regions": result = conf.selectedGC == gcRegions
     else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "opt":
@@ -243,15 +259,22 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
   of "verbosity": result = $conf.verbosity == arg
   of "app":
     case arg.normalize
-    of "gui":       result = contains(conf.globalOptions, optGenGuiApp)
-    of "console":   result = not contains(conf.globalOptions, optGenGuiApp)
-    of "lib":       result = contains(conf.globalOptions, optGenDynLib) and
+    of "gui": result = contains(conf.globalOptions, optGenGuiApp)
+    of "console": result = not contains(conf.globalOptions, optGenGuiApp)
+    of "lib": result = contains(conf.globalOptions, optGenDynLib) and
                       not contains(conf.globalOptions, optGenGuiApp)
     of "staticlib": result = contains(conf.globalOptions, optGenStaticLib) and
                       not contains(conf.globalOptions, optGenGuiApp)
     else: localError(conf, info, errGuiConsoleOrLibExpectedButXFound % arg)
   of "dynliboverride":
     result = isDynlibOverride(conf, arg)
+  of "exceptions":
+    case arg.normalize
+    of "cpp": result = conf.exc == excCpp
+    of "setjmp": result = conf.exc == excSetjmp
+    of "quirky": result = conf.exc == excQuirky
+    of "goto": result = conf.exc == excGoto
+    else: localError(conf, info, errInvalidExceptionSystem % arg)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool =
@@ -265,6 +288,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "hints": result = contains(conf.options, optHints)
   of "threadanalysis": result = contains(conf.globalOptions, optThreadAnalysis)
   of "stacktrace": result = contains(conf.options, optStackTrace)
+  of "stacktracemsgs": result = contains(conf.options, optStackTraceMsgs)
   of "linetrace": result = contains(conf.options, optLineTrace)
   of "debugger": result = contains(conf.globalOptions, optCDebug)
   of "profiler": result = contains(conf.options, optProfiler)
@@ -274,13 +298,14 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
     result = conf.options * {optNaNCheck, optInfCheck} == {optNaNCheck, optInfCheck}
   of "infchecks": result = contains(conf.options, optInfCheck)
   of "nanchecks": result = contains(conf.options, optNaNCheck)
-  of "nilchecks": result = contains(conf.options, optNilCheck)
+  of "nilchecks": result = false # not a thing
   of "objchecks": result = contains(conf.options, optObjCheck)
   of "fieldchecks": result = contains(conf.options, optFieldCheck)
   of "rangechecks": result = contains(conf.options, optRangeCheck)
   of "boundchecks": result = contains(conf.options, optBoundsCheck)
   of "refchecks": result = contains(conf.options, optRefCheck)
   of "overflowchecks": result = contains(conf.options, optOverflowCheck)
+  of "staticboundchecks": result = contains(conf.options, optStaticBoundsCheck)
   of "stylechecks": result = contains(conf.options, optStyleCheck)
   of "linedir": result = contains(conf.options, optLineDir)
   of "assertions", "a": result = contains(conf.options, optAssert)
@@ -294,7 +319,6 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "patterns", "trmacros": result = contains(conf.options, optTrMacros)
   of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace)
   of "nilseqs": result = contains(conf.options, optNilSeqs)
-  of "oldast": result = contains(conf.options, optOldAst)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc processPath(conf: ConfigRef; path: string, info: TLineInfo,
@@ -359,6 +383,27 @@ proc dynlibOverride(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, in
     expectArg(conf, switch, arg, pass, info)
     options.inclDynlibOverride(conf, arg)
 
+proc handleStdinInput*(conf: ConfigRef) =
+  conf.projectName = "stdinfile"
+  conf.projectFull = conf.projectName.AbsoluteFile
+  conf.projectPath = AbsoluteDir getCurrentDir()
+  conf.projectIsStdin = true
+  if conf.outDir.isEmpty:
+    conf.outDir = getNimcacheDir(conf)
+
+proc specialDefine(conf: ConfigRef, key: string) =
+  # Keep this syncronized with the default config/nim.cfg!
+  if cmpIgnoreStyle(key, "nimQuirky") == 0:
+    conf.exc = excQuirky
+  elif cmpIgnoreStyle(key, "release") == 0 or cmpIgnoreStyle(key, "danger") == 0:
+    conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize}
+    conf.globalOptions.excl {optExcessiveStackTrace, optCDebug}
+    conf.options.incl optOptimizeSpeed
+  if cmpIgnoreStyle(key, "danger") == 0 or cmpIgnoreStyle(key, "quick") == 0:
+    conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck,
+      optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir}
+    conf.globalOptions.excl {optCDebug}
+
 proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
                     conf: ConfigRef) =
   var
@@ -366,8 +411,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   case switch.normalize
   of "path", "p":
     expectArg(conf, switch, arg, pass, info)
-    addPath(conf, if pass == passPP: processCfgPath(conf, arg, info)
-                  else: processPath(conf, arg, info), info)
+    for path in nimbleSubs(conf, arg):
+      addPath(conf, if pass == passPP: processCfgPath(conf, path, info)
+                    else: processPath(conf, path, info), info)
   of "nimblepath", "babelpath":
     # keep the old name for compat
     if pass in {passCmd2, passPP} and optNoNimblePath notin conf.globalOptions:
@@ -399,17 +445,28 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "outdir":
     expectArg(conf, switch, arg, pass, info)
     conf.outDir = processPath(conf, arg, info, notRelativeToProj=true)
+  of "usenimcache":
+    processOnOffSwitchG(conf, {optUseNimcache}, arg, pass, info)
   of "docseesrcurl":
     expectArg(conf, switch, arg, pass, info)
     conf.docSeeSrcUrl = arg
+  of "docroot":
+    conf.docRoot = if arg.len == 0: docRootDefault else: arg
+  of "backend", "b":
+    let backend = parseEnum(arg.normalize, TBackend.default)
+    if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
+    conf.backend = backend
+  of "doccmd": conf.docCmd = arg
   of "mainmodule", "m":
     discard "allow for backwards compatibility, but don't do anything"
   of "define", "d":
     expectArg(conf, switch, arg, pass, info)
     if {':', '='} in arg:
       splitSwitch(conf, arg, key, val, pass, info)
+      specialDefine(conf, key)
       defineSymbol(conf.symbols, key, val)
     else:
+      specialDefine(conf, arg)
       defineSymbol(conf.symbols, arg)
   of "undef", "u":
     expectArg(conf, switch, arg, pass, info)
@@ -437,14 +494,20 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "forcebuild", "f":
     processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
   of "project":
-    processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
+    processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info)
   of "gc":
+    if conf.backend == backendJs or conf.command == "js":
+      # for: bug #16033
+      # This might still be imperfect, in rarse corner cases
+      # (where command is reset in nimscript, maybe).
+      return
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
       case arg.normalize
       of "boehm":
         conf.selectedGC = gcBoehm
         defineSymbol(conf.symbols, "boehmgc")
+        incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS
       of "refc":
         conf.selectedGC = gcRefc
       of "v2":
@@ -452,9 +515,31 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       of "markandsweep":
         conf.selectedGC = gcMarkAndSweep
         defineSymbol(conf.symbols, "gcmarkandsweep")
-      of "destructors":
-        conf.selectedGC = gcDestructors
+      of "destructors", "arc":
+        conf.selectedGC = gcArc
         defineSymbol(conf.symbols, "gcdestructors")
+        defineSymbol(conf.symbols, "gcarc")
+        incl conf.globalOptions, optSeqDestructors
+        incl conf.globalOptions, optTinyRtti
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+          defineSymbol(conf.symbols, "nimV2")
+      of "orc":
+        conf.selectedGC = gcOrc
+        defineSymbol(conf.symbols, "gcdestructors")
+        defineSymbol(conf.symbols, "gcorc")
+        incl conf.globalOptions, optSeqDestructors
+        incl conf.globalOptions, optTinyRtti
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+          defineSymbol(conf.symbols, "nimV2")
+      of "hooks":
+        conf.selectedGC = gcHooks
+        defineSymbol(conf.symbols, "gchooks")
+        incl conf.globalOptions, optSeqDestructors
+        processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
       of "go":
         conf.selectedGC = gcGo
         defineSymbol(conf.symbols, "gogc")
@@ -462,17 +547,19 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         conf.selectedGC = gcNone
         defineSymbol(conf.symbols, "nogc")
       of "stack", "regions":
-        conf.selectedGC= gcRegions
+        conf.selectedGC = gcRegions
         defineSymbol(conf.symbols, "gcregions")
       else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "warnings", "w":
     if processOnOffSwitchOrList(conf, {optWarns}, arg, pass, info): listWarnings(conf)
   of "warning": processSpecificNote(arg, wWarning, pass, info, switch, conf)
   of "hint": processSpecificNote(arg, wHint, pass, info, switch, conf)
+  of "warningaserror": processSpecificNote(arg, wWarningAsError, pass, info, switch, conf)
   of "hints":
     if processOnOffSwitchOrList(conf, {optHints}, arg, pass, info): listHints(conf)
   of "threadanalysis": processOnOffSwitchG(conf, {optThreadAnalysis}, arg, pass, info)
   of "stacktrace": processOnOffSwitch(conf, {optStackTrace}, arg, pass, info)
+  of "stacktracemsgs": processOnOffSwitch(conf, {optStackTraceMsgs}, arg, pass, info)
   of "excessivestacktrace": processOnOffSwitchG(conf, {optExcessiveStackTrace}, arg, pass, info)
   of "linetrace": processOnOffSwitch(conf, {optLineTrace}, arg, pass, info)
   of "debugger":
@@ -509,38 +596,27 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     else:
       undefSymbol(conf.symbols, "hotcodereloading")
       undefSymbol(conf.symbols, "useNimRtl")
-  of "oldnewlines":
-    case arg.normalize
-    of "","on":
-      conf.oldNewlines = true
-      defineSymbol(conf.symbols, "nimOldNewlines")
-    of "off":
-      conf.oldNewlines = false
-      undefSymbol(conf.symbols, "nimOldNewlines")
-    else:
-      localError(conf, info, errOnOrOffExpectedButXFound % arg)
-  of "laxstrings": processOnOffSwitch(conf, {optLaxStrings}, arg, pass, info)
   of "nilseqs": processOnOffSwitch(conf, {optNilSeqs}, arg, pass, info)
-  of "oldast": processOnOffSwitch(conf, {optOldAst}, arg, pass, info)
   of "checks", "x": processOnOffSwitch(conf, ChecksOptions, arg, pass, info)
   of "floatchecks":
     processOnOffSwitch(conf, {optNaNCheck, optInfCheck}, arg, pass, info)
   of "infchecks": processOnOffSwitch(conf, {optInfCheck}, arg, pass, info)
   of "nanchecks": processOnOffSwitch(conf, {optNaNCheck}, arg, pass, info)
-  of "nilchecks": processOnOffSwitch(conf, {optNilCheck}, arg, pass, info)
+  of "nilchecks": discard "no such thing anymore"
   of "objchecks": processOnOffSwitch(conf, {optObjCheck}, arg, pass, info)
   of "fieldchecks": processOnOffSwitch(conf, {optFieldCheck}, arg, pass, info)
   of "rangechecks": processOnOffSwitch(conf, {optRangeCheck}, arg, pass, info)
   of "boundchecks": processOnOffSwitch(conf, {optBoundsCheck}, arg, pass, info)
   of "refchecks": processOnOffSwitch(conf, {optRefCheck}, arg, pass, info)
   of "overflowchecks": processOnOffSwitch(conf, {optOverflowCheck}, arg, pass, info)
+  of "staticboundchecks": processOnOffSwitch(conf, {optStaticBoundsCheck}, arg, pass, info)
   of "stylechecks": processOnOffSwitch(conf, {optStyleCheck}, arg, pass, info)
   of "linedir": processOnOffSwitch(conf, {optLineDir}, arg, pass, info)
   of "assertions", "a": processOnOffSwitch(conf, {optAssert}, arg, pass, info)
   of "deadcodeelim": discard # deprecated, dead code elim always on
   of "threads":
     processOnOffSwitchG(conf, {optThreads}, arg, pass, info)
-    #if optThreads in conf.globalOptions: incl(conf.notes, warnGcUnsafe)
+    #if optThreads in conf.globalOptions: conf.setNote(warnGcUnsafe)
   of "tlsemulation": processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info)
   of "taintmode": processOnOffSwitchG(conf, {optTaintMode}, arg, pass, info)
   of "implicitstatic":
@@ -613,6 +689,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       conf.implicitIncludes.add findModule(conf, arg, toFullPath(conf, info)).string
   of "listcmd":
     processOnOffSwitchG(conf, {optListCmd}, arg, pass, info)
+  of "asm":
+    processOnOffSwitchG(conf, {optProduceAsm}, arg, pass, info)
   of "genmapping":
     processOnOffSwitchG(conf, {optGenMapping}, arg, pass, info)
   of "os":
@@ -633,6 +711,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       setTarget(conf.target, conf.target.targetOS, cpu)
   of "run", "r":
     processOnOffSwitchG(conf, {optRun}, arg, pass, info)
+  of "maxloopiterationsvm":
+    expectArg(conf, switch, arg, pass, info)
+    conf.maxLoopIterationsVM = parseInt(arg)
   of "errormax":
     expectArg(conf, switch, arg, pass, info)
     # Note: `nim check` (etc) can overwrite this.
@@ -647,9 +728,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if verbosity notin {0..3}:
       localError(conf, info, "invalid verbosity level: '$1'" % arg)
     conf.verbosity = verbosity
-    conf.notes = NotesVerbosity[conf.verbosity]
-    incl(conf.notes, conf.enableNotes)
-    excl(conf.notes, conf.disableNotes)
+    var verb = NotesVerbosity[conf.verbosity]
+    ## We override the default `verb` by explicitly modified (set/unset) notes.
+    conf.notes = (conf.modifiedyNotes * conf.notes + verb) -
+      (conf.modifiedyNotes * verb - conf.notes)
     conf.mainPackageNotes = conf.notes
   of "parallelbuild":
     expectArg(conf, switch, arg, pass, info)
@@ -748,8 +830,15 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       localError(conf, info, "unknown obsolete feature")
   of "nocppexceptions":
     expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optNoCppExceptions)
+    conf.exc = low(ExceptionSystem)
     defineSymbol(conf.symbols, "noCppExceptions")
+  of "exceptions":
+    case arg.normalize
+    of "cpp": conf.exc = excCpp
+    of "setjmp": conf.exc = excSetjmp
+    of "quirky": conf.exc = excQuirky
+    of "goto": conf.exc = excGoto
+    else: localError(conf, info, errInvalidExceptionSystem % arg)
   of "cppdefine":
     expectArg(conf, switch, arg, pass, info)
     if conf != nil:
@@ -763,13 +852,20 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       incl(conf.globalOptions, optOwnedRefs)
       incl(conf.globalOptions, optSeqDestructors)
       defineSymbol(conf.symbols, "nimV2")
-      conf.selectedGC = gcDestructors
-      defineSymbol(conf.symbols, "gcdestructors")
+      conf.selectedGC = gcHooks
+      defineSymbol(conf.symbols, "gchooks")
+      defineSymbol(conf.symbols, "nimSeqsV2")
+      defineSymbol(conf.symbols, "nimOwnedEnabled")
+  of "seqsv2":
+    processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
+    if pass in {passCmd2, passPP}:
+      defineSymbol(conf.symbols, "nimSeqsV2")
   of "stylecheck":
     case arg.normalize
     of "off": conf.globalOptions = conf.globalOptions - {optStyleHint, optStyleError}
-    of "hint": conf.globalOptions = conf.globalOptions + {optStyleHint}
+    of "hint": conf.globalOptions = conf.globalOptions + {optStyleHint} - {optStyleError}
     of "error": conf.globalOptions = conf.globalOptions + {optStyleError}
+    of "usages": conf.globalOptions.incl optStyleUsages
     else: localError(conf, info, errOffHintsError % arg)
   of "showallmismatches":
     processOnOffSwitchG(conf, {optShowAllMismatches}, arg, pass, info)
@@ -786,23 +882,54 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "expandmacro":
     expectArg(conf, switch, arg, pass, info)
     conf.macrosToExpand[arg] = "T"
-  of "oldgensym":
-    processOnOffSwitchG(conf, {optNimV019}, arg, pass, info)
+  of "expandarc":
+    expectArg(conf, switch, arg, pass, info)
+    conf.arcToExpand[arg] = "T"
   of "useversion":
     expectArg(conf, switch, arg, pass, info)
     case arg
     of "1.0":
-      discard "the default"
+      defineSymbol(conf.symbols, "NimMajor", "1")
+      defineSymbol(conf.symbols, "NimMinor", "0")
+      # always be compatible with 1.0.100:
+      defineSymbol(conf.symbols, "NimPatch", "100")
+      # old behaviors go here:
+      defineSymbol(conf.symbols, "nimOldRelativePathBehavior")
+      undefSymbol(conf.symbols, "nimDoesntTrackDefects")
+      ast.eqTypeFlags.excl {tfGcSafe, tfNoSideEffect}
+      conf.globalOptions.incl optNimV1Emulation
+    of "1.2":
+      defineSymbol(conf.symbols, "NimMajor", "1")
+      defineSymbol(conf.symbols, "NimMinor", "2")
+      # always be compatible with 1.2.100:
+      defineSymbol(conf.symbols, "NimPatch", "100")
+      conf.globalOptions.incl optNimV12Emulation
     else:
       localError(conf, info, "unknown Nim version; currently supported values are: {1.0}")
-  of "":
-    conf.projectName = "-"
+  of "benchmarkvm":
+    processOnOffSwitchG(conf, {optBenchmarkVM}, arg, pass, info)
+  of "profilevm":
+    processOnOffSwitchG(conf, {optProfileVM}, arg, pass, info)
+  of "sinkinference":
+    processOnOffSwitch(conf, {optSinkInference}, arg, pass, info)
+  of "cursorinference":
+    # undocumented, for debugging purposes only:
+    processOnOffSwitch(conf, {optCursorInference}, arg, pass, info)
+  of "panics":
+    processOnOffSwitchG(conf, {optPanics}, arg, pass, info)
+    if optPanics in conf.globalOptions:
+      defineSymbol(conf.symbols, "nimPanics")
+  of "sourcemap":
+    conf.globalOptions.incl optSourcemap
+    conf.options.incl optLineDir
+  of "deepcopy":
+    processOnOffSwitchG(conf, {optEnableDeepCopy}, arg, pass, info)
+  of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -)
+    handleStdinInput(conf)
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)
 
-template gCmdLineInfo*(): untyped = newLineInfo(commandLineIdx, 1, 1)
-
 proc processCommand*(switch: string, pass: TCmdLinePass; config: ConfigRef) =
   var cmd, arg: string
   splitSwitch(config, switch, cmd, arg, pass, gCmdLineInfo)

compiler/condsyms.nim

@@ -13,7 +13,7 @@ import
   strtabs
 
 from options import Feature
-from lineinfos import HintsToStr, WarningsToStr
+from lineinfos import hintMin, hintMax, warnMin, warnMax
 
 proc defineSymbol*(symbols: StringTableRef; symbol: string, value: string = "true") =
   symbols[symbol] = value
@@ -48,13 +48,11 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimalias")
   defineSymbol("nimlocks")
   defineSymbol("nimnode")
-  defineSymbol("nimnomagic64")
   defineSymbol("nimvarargstyped")
   defineSymbol("nimtypedescfixed")
   defineSymbol("nimKnowsNimvm")
   defineSymbol("nimArrIdx")
   defineSymbol("nimHasalignOf")
-  defineSymbol("nimNewShiftOps")
   defineSymbol("nimDistros")
   defineSymbol("nimHasCppDefine")
   defineSymbol("nimGenericInOutFlags")
@@ -88,15 +86,41 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimMacrosSizealignof")
   defineSymbol("nimNoZeroExtendMagic")
   defineSymbol("nimMacrosGetNodeId")
-  for f in low(Feature)..high(Feature):
+  for f in Feature:
     defineSymbol("nimHas" & $f)
 
-  for s in WarningsToStr:
-    defineSymbol("nimHasWarning" & s)
-  for s in HintsToStr:
-    defineSymbol("nimHasHint" & s)
+  for s in warnMin..warnMax:
+    defineSymbol("nimHasWarning" & $s)
+  for s in hintMin..hintMax:
+    defineSymbol("nimHasHint" & $s)
 
   defineSymbol("nimFixedOwned")
   defineSymbol("nimHasStyleChecks")
   defineSymbol("nimToOpenArrayCString")
   defineSymbol("nimHasUsed")
+  defineSymbol("nimFixedForwardGeneric")
+  defineSymbol("nimnomagic64")
+  defineSymbol("nimNewShiftOps")
+  defineSymbol("nimHasCursor")
+  defineSymbol("nimAlignPragma")
+  defineSymbol("nimHasExceptionsQuery")
+  defineSymbol("nimHasIsNamedTuple")
+  defineSymbol("nimHashOrdinalFixed")
+
+  when defined(nimHasLibFFI):
+    # Renaming as we can't conflate input vs output define flags; e.g. this
+    # will report the right thing regardless of whether user adds
+    # `-d:nimHasLibFFI` in his user config.
+    defineSymbol("nimHasLibFFIEnabled")
+
+  defineSymbol("nimHasSinkInference")
+  defineSymbol("nimNewIntegerOps")
+  defineSymbol("nimHasInvariant")
+  defineSymbol("nimHasStacktraceMsgs")
+  defineSymbol("nimDoesntTrackDefects")
+  defineSymbol("nimHasLentIterators")
+  defineSymbol("nimHasDeclaredMagic")
+  defineSymbol("nimHasStacktracesModule")
+  defineSymbol("nimHasEffectTraitsModule")
+  defineSymbol("nimHasCastPragmaBlocks")
+  defineSymbol("nimHasJsBigIntBackend")

compiler/configuration.nim

@@ -1,6 +0,0 @@
-## Use the module 'lineinfos' instead!
-
-{.deprecated.}
-
-import lineinfos
-export lineinfos

compiler/depends.nim

@@ -25,7 +25,7 @@ type
     dotGraph: Rope
 
 proc addDependencyAux(b: Backend; importing, imported: string) =
-  addf(b.dotGraph, "$1 -> \"$2\";$n", [rope(importing), rope(imported)])
+  b.dotGraph.addf("$1 -> \"$2\";$n", [rope(importing), rope(imported)])
   # s1 -> s2_4[label="[0-9]"];
 
 proc addDotDependency(c: PPassContext, n: PNode): PNode =
@@ -34,14 +34,14 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode =
   let b = Backend(g.graph.backend)
   case n.kind
   of nkImportStmt:
-    for i in 0 ..< len(n):
-      var imported = getModuleName(g.config, n.sons[i])
+    for i in 0..<n.len:
+      var imported = getModuleName(g.config, n[i])
       addDependencyAux(b, g.module.name.s, imported)
   of nkFromStmt, nkImportExceptStmt:
-    var imported = getModuleName(g.config, n.sons[0])
+    var imported = getModuleName(g.config, n[0])
     addDependencyAux(b, g.module.name.s, imported)
   of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr:
-    for i in 0 ..< len(n): discard addDotDependency(c, n.sons[i])
+    for i in 0..<n.len: discard addDotDependency(c, n[i])
   else:
     discard
 
@@ -51,7 +51,10 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) =
       rope(project.splitFile.name), b.dotGraph],
             changeFileExt(project, "dot"))
 
-proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
+when not defined(nimHasSinkInference):
+  {.pragma: nosinks.}
+
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext {.nosinks.} =
   var g: PGen
   new(g)
   g.module = module

compiler/dfa.nim

@@ -29,47 +29,45 @@
 ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
 ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
 
-import ast, types, intsets, lineinfos, renderer
+import ast, types, intsets, lineinfos, renderer, asciitables
 
 from patterns import sameTrees
 
 type
   InstrKind* = enum
-    goto, fork, join, def, use
+    goto, fork, def, use
   Instr* = object
-    n*: PNode
+    n*: PNode # contains the def/use location.
     case kind*: InstrKind
-    of def, use: sym*: PSym # 'sym' can also be 'nil' and
-                            # then 'n' contains the def/use location.
-                            # This is used so that we can track object
-                            # and tuple field accesses precisely.
-    of goto, fork, join: dest*: int
+    of goto, fork: dest*: int
+    else: discard
 
   ControlFlowGraph* = seq[Instr]
 
   TPosition = distinct int
+
   TBlock = object
-    label: PSym
-    fixups: seq[TPosition]
+    case isTryBlock: bool
+    of false:
+      label: PSym
+      breakFixups: seq[(TPosition, seq[PNode])] #Contains the gotos for the breaks along with their pending finales
+    of true:
+      finale: PNode
+      raiseFixups: seq[TPosition] #Contains the gotos for the raises
 
   Con = object
     code: ControlFlowGraph
-    inCall, inTryStmt: int
+    inTryStmt: int
     blocks: seq[TBlock]
-    tryStmtFixups: seq[TPosition]
-    forks: seq[TPosition]
     owner: PSym
 
-proc debugInfo(info: TLineInfo): string =
-  result = $info.line #info.toFilename & ":" & $info.line
-
-proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
+proc codeListing(c: ControlFlowGraph, start = 0; last = -1): string =
   # for debugging purposes
   # first iteration: compute all necessary labels:
   var jumpTargets = initIntSet()
   let last = if last < 0: c.len-1 else: min(last, c.len-1)
   for i in start..last:
-    if c[i].kind in {goto, fork, join}:
+    if c[i].kind in {goto, fork}:
       jumpTargets.incl(i+c[i].dest)
   var i = start
   while i <= last:
@@ -80,27 +78,22 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
     case c[i].kind
     of def, use:
       result.add renderTree(c[i].n)
-    of goto, fork, join:
+    of goto, fork:
       result.add "L"
       result.addInt c[i].dest+i
     result.add("\t#")
-    result.add(debugInfo(c[i].n.info))
+    result.add($c[i].n.info.line)
     result.add("\n")
     inc i
   if i in jumpTargets: result.add("L" & $i & ": End\n")
-  # consider calling `asciitables.alignTable`
 
-proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
+proc echoCfg*(c: ControlFlowGraph; start = 0; last = -1) {.deprecated.} =
   ## echos the ControlFlowGraph for debugging purposes.
-  var buf = ""
-  codeListing(c, buf, start, last)
-  when declared(echo):
-    echo buf
+  echo codeListing(c, start, last).alignTable
 
 proc forkI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: fork, dest: 0)
-  c.forks.add result
 
 proc gotoI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
@@ -108,6 +101,24 @@ proc gotoI(c: var Con; n: PNode): TPosition =
 
 #[
 
+Join is no more
+===============
+Instead of generating join instructions we adapt our traversal of the CFG.
+
+When encountering a fork we split into two paths, we follow the path
+starting at "pc + 1" until it encounters the joinpoint: "pc + forkInstr.dest".
+If we encounter gotos that would jump further than the current joinpoint,
+as can happen with gotos generated by unstructured controlflow such as break, raise or return,
+we simply suspend following the current path, and follow the other path until the new joinpoint
+which is simply the instruction pointer returned to us by the now suspended path.
+If the path we are following now, also encounters a goto that exceeds the joinpoint
+we repeat the process; suspending the current path and evaluating the other one with a new joinpoint.
+If we eventually reach a common joinpoint we join the two paths.
+This new "ping-pong" approach has the obvious advantage of not requiring join instructions, as such
+cutting down on the CFG size but is also mandatory for correctly handling complicated cases
+of unstructured controlflow.
+
+
 Design of join
 ==============
 
@@ -256,33 +267,36 @@ duplicate the 'join' instructions on breaks and return exits!
 
 ]#
 
-proc joinI(c: var Con; fromFork: TPosition; n: PNode) =
-  let dist = fromFork.int - c.code.len
-  c.code.add Instr(n: n, kind: join, dest: dist)
+proc genLabel(c: Con): TPosition = TPosition(c.code.len)
 
-proc genLabel(c: Con): TPosition =
-  result = TPosition(c.code.len)
+template checkedDistance(dist): int =
+  doAssert low(int) div 2 + 1 < dist and dist < high(int) div 2
+  dist
 
 proc jmpBack(c: var Con, n: PNode, p = TPosition(0)) =
-  let dist = p.int - c.code.len
-  doAssert(low(int) div 2 + 1 < dist and dist < high(int) div 2)
-  c.code.add Instr(n: n, kind: goto, dest: dist)
+  c.code.add Instr(n: n, kind: goto, dest: checkedDistance(p.int - c.code.len))
 
 proc patch(c: var Con, p: TPosition) =
   # patch with current index
-  let p = p.int
-  let diff = c.code.len - p
-  doAssert(low(int) div 2 + 1 < diff and diff < high(int) div 2)
-  c.code[p].dest = diff
+  c.code[p.int].dest = checkedDistance(c.code.len - p.int)
+
+proc gen(c: var Con; n: PNode)
 
 proc popBlock(c: var Con; oldLen: int) =
-  for f in c.blocks[oldLen].fixups:
-    c.patch(f)
+  var exits: seq[TPosition]
+  exits.add c.gotoI(newNode(nkEmpty))
+  for f in c.blocks[oldLen].breakFixups:
+    c.patch(f[0])
+    for finale in f[1]:
+      c.gen(finale)
+    exits.add c.gotoI(newNode(nkEmpty))
+  for e in exits:
+    c.patch e
   c.blocks.setLen(oldLen)
 
-template withBlock(labl: PSym; body: untyped) {.dirty.} =
-  var oldLen {.gensym.} = c.blocks.len
-  c.blocks.add TBlock(label: labl, fixups: @[])
+template withBlock(labl: PSym; body: untyped) =
+  let oldLen = c.blocks.len
+  c.blocks.add TBlock(isTryBlock: false, label: labl)
   body
   popBlock(c, oldLen)
 
@@ -290,8 +304,6 @@ proc isTrue(n: PNode): bool =
   n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or
     n.kind == nkIntLit and n.intVal != 0
 
-proc gen(c: var Con; n: PNode) # {.noSideEffect.}
-
 when true:
   proc genWhile(c: var Con; n: PNode) =
     # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
@@ -303,12 +315,13 @@ when true:
 
     Becomes:
 
-    if cond:
-      body
+    block:
       if cond:
         body
         if cond:
           body
+          if cond:
+            body
 
     We still need to ensure 'break' resolves properly, so an AST to AST
     translation is impossible.
@@ -331,28 +344,23 @@ when true:
     L4:
       join F1
     ]#
-    if isTrue(n.sons[0]):
+    if isTrue(n[0]):
       # 'while true' is an idiom in Nim and so we produce
       # better code for it:
-      for i in 0..2:
-        withBlock(nil):
-          c.gen(n.sons[1])
+      withBlock(nil):
+        for i in 0..2:
+          c.gen(n[1])
     else:
-      let oldForksLen = c.forks.len
-      var endings: array[3, TPosition]
-      for i in 0..2:
-        withBlock(nil):
-          c.gen(n.sons[0])
+      withBlock(nil):
+        var endings: array[3, TPosition]
+        for i in 0..2:
+          c.gen(n[0])
           endings[i] = c.forkI(n)
-          c.gen(n.sons[1])
-      for i in countdown(endings.high, 0):
-        let endPos = endings[i]
-        c.patch(endPos)
-        c.joinI(c.forks.pop(), n)
-      doAssert(c.forks.len == oldForksLen)
+          c.gen(n[1])
+        for i in countdown(endings.high, 0):
+          c.patch(endings[i])
 
 else:
-
   proc genWhile(c: var Con; n: PNode) =
     # lab1:
     #   cond, tmp
@@ -360,47 +368,21 @@ else:
     #   body
     #   jmp lab1
     # lab2:
-    let oldForksLen = c.forks.len
     let lab1 = c.genLabel
     withBlock(nil):
-      if isTrue(n.sons[0]):
-        c.gen(n.sons[1])
+      if isTrue(n[0]):
+        c.gen(n[1])
         c.jmpBack(n, lab1)
       else:
-        c.gen(n.sons[0])
-        let lab2 = c.forkI(n)
-        c.gen(n.sons[1])
-        c.jmpBack(n, lab1)
-        c.patch(lab2)
-    setLen(c.forks, oldForksLen)
-
-proc genBlock(c: var Con; n: PNode) =
-  withBlock(n.sons[0].sym):
-    c.gen(n.sons[1])
-
-proc genJoins(c: var Con; n: PNode) =
-  for i in countdown(c.forks.high, 0): joinI(c, c.forks[i], n)
-
-proc genBreak(c: var Con; n: PNode) =
-  genJoins(c, n)
-  let lab1 = c.gotoI(n)
-  if n.sons[0].kind == nkSym:
-    #echo cast[int](n.sons[0].sym)
-    for i in countdown(c.blocks.len-1, 0):
-      if c.blocks[i].label == n.sons[0].sym:
-        c.blocks[i].fixups.add lab1
-        return
-    #globalError(n.info, "VM problem: cannot find 'break' target")
-  else:
-    c.blocks[c.blocks.high].fixups.add lab1
+        c.gen(n[0])
+        forkT(n):
+          c.gen(n[1])
+          c.jmpBack(n, lab1)
 
 template forkT(n, body) =
-  let oldLen = c.forks.len
   let lab1 = c.forkI(n)
   body
   c.patch(lab1)
-  c.joinI(lab1, n)
-  setLen(c.forks, oldLen)
 
 proc genIf(c: var Con, n: PNode) =
   #[
@@ -439,21 +421,16 @@ proc genIf(c: var Con, n: PNode) =
     join F1
 
   ]#
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
-  for i in 0 ..< len(n):
-    var it = n.sons[i]
-    c.gen(it.sons[0])
+  for i in 0..<n.len:
+    let it = n[i]
+    c.gen(it[0])
     if it.len == 2:
-      let elsePos = forkI(c, it[1])
-      c.gen(it.sons[1])
-      endings.add(c.gotoI(it.sons[1]))
-      c.patch(elsePos)
+      forkT(it[1]):
+        c.gen(it[1])
+        endings.add c.gotoI(it[1])
   for i in countdown(endings.high, 0):
-    let endPos = endings[i]
-    c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
+    c.patch(endings[i])
 
 proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, a
@@ -461,9 +438,9 @@ proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, b
   # lab1:
   #   join F1
-  c.gen(n.sons[1])
+  c.gen(n[1])
   forkT(n):
-    c.gen(n.sons[2])
+    c.gen(n[2])
 
 proc genCase(c: var Con; n: PNode) =
   #  if (!expr1) goto lab1;
@@ -476,94 +453,107 @@ proc genCase(c: var Con; n: PNode) =
   #  lab2:
   #    elsePart
   #  Lend:
-  let isExhaustive = skipTypes(n.sons[0].typ,
+  let isExhaustive = skipTypes(n[0].typ,
     abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
 
   var endings: seq[TPosition] = @[]
-  let oldLen = c.forks.len
-  c.gen(n.sons[0])
-  for i in 1 ..< n.len:
-    let it = n.sons[i]
-    if it.len == 1:
-      c.gen(it.sons[0])
-    elif i == n.len-1 and isExhaustive:
+  c.gen(n[0])
+  for i in 1..<n.len:
+    let it = n[i]
+    if it.len == 1 or (i == n.len-1 and isExhaustive):
       # treat the last branch as 'else' if this is an exhaustive case statement.
       c.gen(it.lastSon)
     else:
-      let elsePos = c.forkI(it.lastSon)
-      c.gen(it.lastSon)
-      endings.add(c.gotoI(it.lastSon))
-      c.patch(elsePos)
+      forkT(it.lastSon):
+        c.gen(it.lastSon)
+        endings.add c.gotoI(it.lastSon)
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
+
+proc genBlock(c: var Con; n: PNode) =
+  withBlock(n[0].sym):
+    c.gen(n[1])
+
+proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) =
+  let lab1 = c.gotoI(n)
+  if c.blocks[i].isTryBlock:
+    c.blocks[i].raiseFixups.add lab1
+  else:
+    var trailingFinales: seq[PNode]
+    if c.inTryStmt > 0: #Ok, we are in a try, lets see which (if any) try's we break out from:
+      for b in countdown(c.blocks.high, i):
+        if c.blocks[b].isTryBlock:
+          trailingFinales.add c.blocks[b].finale
+
+    c.blocks[i].breakFixups.add (lab1, trailingFinales)
+
+proc genBreak(c: var Con; n: PNode) =
+  if n[0].kind == nkSym:
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock and c.blocks[i].label == n[0].sym:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    #globalError(n.info, "VM problem: cannot find 'break' target")
+  else:
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
 
 proc genTry(c: var Con; n: PNode) =
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
+
+  let oldLen = c.blocks.len
+  c.blocks.add TBlock(isTryBlock: true, finale: if n[^1].kind == nkFinally: n[^1] else: newNode(nkEmpty))
+
   inc c.inTryStmt
-  let oldFixups = c.tryStmtFixups.len
-
-  #let elsePos = c.forkI(n)
-  c.gen(n.sons[0])
+  c.gen(n[0])
   dec c.inTryStmt
-  for i in oldFixups..c.tryStmtFixups.high:
-    let f = c.tryStmtFixups[i]
+
+  for f in c.blocks[oldLen].raiseFixups:
     c.patch(f)
-    # we also need to produce join instructions
-    # for the 'fork' that might precede the goto instruction
-    if f.int-1 >= 0 and c.code[f.int-1].kind == fork:
-      c.joinI(TPosition(f.int-1), n)
 
-  setLen(c.tryStmtFixups, oldFixups)
+  c.blocks.setLen oldLen
 
-  #c.patch(elsePos)
-  for i in 1 ..< n.len:
-    let it = n.sons[i]
+  for i in 1..<n.len:
+    let it = n[i]
     if it.kind != nkFinally:
-      var blen = len(it)
-      let endExcept = c.forkI(it)
-      c.gen(it.lastSon)
-      endings.add(c.gotoI(it))
-      c.patch(endExcept)
+      forkT(it):
+        c.gen(it.lastSon)
+        endings.add c.gotoI(it)
   for i in countdown(endings.high, 0):
-    let endPos = endings[i]
-    c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-
-  # join the 'elsePos' forkI instruction:
-  #c.joinI(c.forks.pop(), n)
+    c.patch(endings[i])
 
   let fin = lastSon(n)
   if fin.kind == nkFinally:
-    c.gen(fin.sons[0])
-  doAssert(c.forks.len == oldLen)
+    c.gen(fin[0])
 
 template genNoReturn(c: var Con; n: PNode) =
   # leave the graph
   c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 proc genRaise(c: var Con; n: PNode) =
-  genJoins(c, n)
-  gen(c, n.sons[0])
+  gen(c, n[0])
   if c.inTryStmt > 0:
-    c.tryStmtFixups.add c.gotoI(n)
+    for i in countdown(c.blocks.high, 0):
+      if c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    assert false #Unreachable
   else:
     genNoReturn(c, n)
 
 proc genImplicitReturn(c: var Con) =
   if c.owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter} and resultPos < c.owner.ast.len:
-    gen(c, c.owner.ast.sons[resultPos])
+    gen(c, c.owner.ast[resultPos])
 
 proc genReturn(c: var Con; n: PNode) =
-  genJoins(c, n)
-  if n.sons[0].kind != nkEmpty:
-    gen(c, n.sons[0])
+  if n[0].kind != nkEmpty:
+    gen(c, n[0])
   else:
     genImplicitReturn(c)
-  genNoReturn(c, n)
+  genBreakOrRaiseAux(c, 0, n)
 
 const
   InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp}
@@ -573,16 +563,6 @@ const
                 nkObjDownConv, nkObjUpConv}
   PathKinds1 = {nkHiddenStdConv, nkHiddenSubConv}
 
-proc getRoot(n: PNode): PNode =
-  result = n
-  while true:
-    case result.kind
-    of PathKinds0:
-      result = result[0]
-    of PathKinds1:
-      result = result[1]
-    else: break
-
 proc skipConvDfa*(n: PNode): PNode =
   result = n
   while true:
@@ -593,74 +573,62 @@ proc skipConvDfa*(n: PNode): PNode =
       result = result[1]
     else: break
 
-proc genUse(c: var Con; orig: PNode) =
-  let n = dfa.getRoot(orig)
-  if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: orig, kind: use, sym: if orig != n: nil else: n.sym)
-
-proc aliases(obj, field: PNode): bool =
+proc aliases*(obj, field: PNode): bool =
   var n = field
   var obj = obj
-  while obj.kind in {nkHiddenSubConv, nkHiddenStdConv, nkObjDownConv, nkObjUpConv}:
-    obj = obj[0]
+  while true:
+    case obj.kind
+    of {nkObjDownConv, nkObjUpConv, nkAddr, nkHiddenAddr, nkDerefExpr, nkHiddenDeref}:
+      obj = obj[0]
+    of PathKinds1:
+      obj = obj[1]
+    else: break
   while true:
     if sameTrees(obj, n): return true
     case n.kind
-    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-       nkObjDownConv, nkObjUpConv, nkHiddenAddr, nkAddr, nkBracketExpr,
-       nkHiddenDeref, nkDerefExpr:
+    of PathKinds0:
       n = n[0]
-    else:
-      break
-  return false
+    of PathKinds1:
+      n = n[1]
+    else: break
 
-proc useInstrTargets*(ins: Instr; loc: PNode): bool =
-  assert ins.kind == use
-  if ins.sym != nil and loc.kind == nkSym:
-    result = ins.sym == loc.sym
-  else:
-    result = ins.n == loc or sameTrees(ins.n, loc)
-  if not result:
-    # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
-    # def x.f; question: does it affect the full 'x'? No.
-    # def x; question: does it affect the 'x.f'? Yes.
-    # use x.f;  question: does it affect the full 'x'? No.
-    # use x; question does it affect 'x.f'? Yes.
-    result = aliases(ins.n, loc) or aliases(loc, ins.n)
+type InstrTargetKind* = enum
+  None, Full, Partial
 
-proc defInstrTargets*(ins: Instr; loc: PNode): bool =
-  assert ins.kind == def
-  if ins.sym != nil and loc.kind == nkSym:
-    result = ins.sym == loc.sym
-  else:
-    result = ins.n == loc or sameTrees(ins.n, loc)
-  if not result:
-    # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
-    # def x.f; question: does it affect the full 'x'? No.
-    # def x; question: does it affect the 'x.f'? Yes.
-    # use x.f;  question: does it affect the full 'x'? No.
-    # use x; question does it affect 'x.f'? Yes.
-    result = aliases(ins.n, loc)
+proc instrTargets*(insloc, loc: PNode): InstrTargetKind =
+  if sameTrees(insloc, loc) or insloc.aliases(loc):
+    Full    # x -> x; x -> x.f
+  elif loc.aliases(insloc):
+    Partial # x.f -> x
+  else: None
 
 proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
   var n = orig
   while true:
     case n.kind
-    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-       nkObjDownConv, nkObjUpConv, nkHiddenAddr, nkAddr, nkBracketExpr:
+    of PathKinds0 - {nkBracketExpr, nkHiddenDeref, nkDerefExpr}:
       n = n[0]
+    of PathKinds1:
+      n = n[1]
+    of nkBracketExpr:
+      # in a[i] the 'i' must be known
+      if n.len > 1 and n[1].kind in {nkCharLit..nkUInt64Lit}:
+        n = n[0]
+      else:
+        return false
     of nkHiddenDeref, nkDerefExpr:
       # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
       # pointer indirection.
+      # bug #14159, we cannot reason about sinkParam[].location as it can
+      # still be shared for tyRef.
       n = n[0]
-      return n.kind == nkSym and n.sym.owner == owner and (isSinkParam(n.sym) or
-          n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
-    else:
-      break
+      return n.kind == nkSym and n.sym.owner == owner and
+         (n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
+    else: break
   # XXX Allow closure deref operations here if we know
   # the owner controlled the closure allocation?
   result = n.kind == nkSym and n.sym.owner == owner and
-    owner.kind != skModule and
+    {sfGlobal, sfThread, sfCursor} * n.sym.flags == {} and
     (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar)
   # Note: There is a different move analyzer possible that checks for
   # consume(param.key); param.key = newValue  for all paths. Then code like
@@ -673,45 +641,55 @@ proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
   # free the old data and all is lost! Lesson: Don't be too smart, trust the
   # lower level C++ optimizer to specialize this code.
 
-proc genDef(c: var Con; n: PNode) =
-  if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: n, kind: def, sym: n.sym)
-  elif isAnalysableFieldAccess(n, c.owner):
-    c.code.add Instr(n: n, kind: def, sym: nil)
+proc skipTrivials(c: var Con, n: PNode): PNode =
+  result = n
+  while true:
+    case result.kind
+    of PathKinds0 - {nkBracketExpr}:
+      result = result[0]
+    of nkBracketExpr:
+      gen(c, result[1])
+      result = result[0]
+    of PathKinds1:
+      result = result[1]
+    else: break
 
-proc canRaise(fn: PNode): bool =
-  const magicsThatCanRaise = {
-    mNone, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst}
-  if fn.kind == nkSym and fn.sym.magic notin magicsThatCanRaise:
-    result = false
-  else:
-    result = true
+proc genUse(c: var Con; orig: PNode) =
+  let n = c.skipTrivials(orig)
+
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    c.code.add Instr(n: orig, kind: use)
+  elif n.kind in nkCallKinds:
+    gen(c, n)
+
+proc genDef(c: var Con; orig: PNode) =
+  let n = c.skipTrivials(orig)
+
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    c.code.add Instr(n: orig, kind: def)
 
 proc genCall(c: var Con; n: PNode) =
   gen(c, n[0])
   var t = n[0].typ
   if t != nil: t = t.skipTypes(abstractInst)
-  inc c.inCall
   for i in 1..<n.len:
     gen(c, n[i])
     when false:
-      if t != nil and i < t.len and t.sons[i].kind == tyVar:
-        # This is wrong! Pass by var is a 'might def', not a 'must def'
-        # like the other defs we emit. This is not good enough for a move
-        # optimizer.
+      if t != nil and i < t.len and t[i].kind == tyOut:
+        # Pass by 'out' is a 'must def'. Good enough for a move optimizer.
         genDef(c, n[i])
   # every call can potentially raise:
-  if c.inTryStmt > 0 and canRaise(n[0]):
+  if c.inTryStmt > 0 and canRaiseConservative(n[0]):
     # we generate the instruction sequence:
     # fork lab1
     # goto exceptionHandler (except or finally)
     # lab1:
     # join F1
-    let endGoto = c.forkI(n)
-    c.tryStmtFixups.add c.gotoI(n)
-    c.patch(endGoto)
-    c.joinI(c.forks.pop(), n)
-  dec c.inCall
+    forkT(n):
+      for i in countdown(c.blocks.high, 0):
+        if c.blocks[i].isTryBlock:
+          genBreakOrRaiseAux(c, i, n)
+          break
 
 proc genMagic(c: var Con; n: PNode; m: TMagic) =
   case m
@@ -728,23 +706,23 @@ proc genVarSection(c: var Con; n: PNode) =
       discard
     elif a.kind == nkVarTuple:
       gen(c, a.lastSon)
-      for i in 0 .. a.len-3: genDef(c, a[i])
+      for i in 0..<a.len-2: genDef(c, a[i])
     else:
       gen(c, a.lastSon)
       if a.lastSon.kind != nkEmpty:
-        genDef(c, a.sons[0])
+        genDef(c, a[0])
 
 proc gen(c: var Con; n: PNode) =
   case n.kind
   of nkSym: genUse(c, n)
   of nkCallKinds:
-    if n.sons[0].kind == nkSym:
-      let s = n.sons[0].sym
+    if n[0].kind == nkSym:
+      let s = n[0].sym
       if s.magic != mNone:
         genMagic(c, n, s.magic)
       else:
         genCall(c, n)
-      if sfNoReturn in n.sons[0].sym.flags:
+      if sfNoReturn in n[0].sym.flags:
         genNoReturn(c, n)
     else:
       genCall(c, n)
@@ -755,12 +733,12 @@ proc gen(c: var Con; n: PNode) =
     # "uses" 'i'. But we are only talking about builtin array indexing so
     # it doesn't matter and 'x = 34' is NOT a usage of 'x'.
     genDef(c, n[0])
-  of PathKinds0 - {nkHiddenStdConv, nkHiddenSubConv, nkObjDownConv, nkObjUpConv}:
+  of PathKinds0 - {nkObjDownConv, nkObjUpConv}:
     genUse(c, n)
   of nkIfStmt, nkIfExpr: genIf(c, n)
   of nkWhenStmt:
     # This is "when nimvm" node. Chose the first branch.
-    gen(c, n.sons[0].sons[1])
+    gen(c, n[0][1])
   of nkCaseStmt: genCase(c, n)
   of nkWhileStmt: genWhile(c, n)
   of nkBlockExpr, nkBlockStmt: genBlock(c, n)
@@ -772,18 +750,21 @@ proc gen(c: var Con; n: PNode) =
      nkBracket, nkCurly, nkPar, nkTupleConstr, nkClosure, nkObjConstr, nkYieldStmt:
     for x in n: gen(c, x)
   of nkPragmaBlock: gen(c, n.lastSon)
-  of nkDiscardStmt, nkObjDownConv, nkObjUpConv: gen(c, n.sons[0])
-  of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast, nkHiddenSubConv, nkHiddenStdConv:
-    gen(c, n.sons[1])
-  of nkStringToCString, nkCStringToString: gen(c, n.sons[0])
+  of nkDiscardStmt, nkObjDownConv, nkObjUpConv, nkStringToCString, nkCStringToString:
+    gen(c, n[0])
+  of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast, PathKinds1:
+    gen(c, n[1])
   of nkVarSection, nkLetSection: genVarSection(c, n)
-  of nkDefer:
-    doAssert false, "dfa construction pass requires the elimination of 'defer'"
+  of nkDefer: doAssert false, "dfa construction pass requires the elimination of 'defer'"
   else: discard
 
 proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
   ## constructs a control flow graph for ``body``.
   var c = Con(code: @[], blocks: @[], owner: s)
-  gen(c, body)
-  genImplicitReturn(c)
-  shallowCopy(result, c.code)
+  withBlock(s):
+    gen(c, body)
+    genImplicitReturn(c)
+  when defined(gcArc) or defined(gcOrc):
+    result = c.code # will move
+  else:
+    shallowCopy(result, c.code)

compiler/docgen2.nim

@@ -11,7 +11,7 @@
 # semantic checking.
 
 import
-  options, ast, msgs, idents, passes, docgen, lineinfos, pathutils
+  options, ast, msgs, passes, docgen, lineinfos, pathutils
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -22,13 +22,14 @@ type
     config: ConfigRef
   PGen = ref TGen
 
-template shouldProcess(g): bool =
-  (g.module.owner.id == g.doc.conf.mainPackageId and optWholeProject in g.doc.conf.globalOptions) or
+proc shouldProcess(g: PGen): bool =
+  (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or
       sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
 
 template closeImpl(body: untyped) {.dirty.} =
   var g = PGen(p)
   let useWarning = sfMainModule notin g.module.flags
+  let groupedToc = true
   if shouldProcess(g):
     body
     try:
@@ -38,7 +39,7 @@ template closeImpl(body: untyped) {.dirty.} =
 
 proc close(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
-    writeOutput(g.doc, useWarning)
+    writeOutput(g.doc, useWarning, groupedToc)
 
 proc closeJson(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:

compiler/enumtostr.nim

@@ -13,7 +13,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
   result.typ = newType(tyProc, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
-  addSon(result.typ.n, newNodeI(nkEffectList, info))
+  result.typ.n.add newNodeI(nkEffectList, info)
 
   result.typ.addParam dest
 
@@ -22,7 +22,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
   caseStmt.add(newSymNode dest)
 
   # copy the branches over, but replace the fields with the for loop body:
-  for i in 0 ..< t.n.len:
+  for i in 0..<t.n.len:
     assert(t.n[i].kind == nkSym)
     var field = t.n[i].sym
     let val = if field.ast == nil: field.name.s else: field.ast.strVal
@@ -33,28 +33,35 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
   body.add(caseStmt)
 
   var n = newNodeI(nkProcDef, info, bodyPos+2)
-  for i in 0 ..< n.len: n.sons[i] = newNodeI(nkEmpty, info)
-  n.sons[namePos] = newSymNode(result)
-  n.sons[paramsPos] = result.typ.n
-  n.sons[bodyPos] = body
-  n.sons[resultPos] = newSymNode(res)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = body
+  n[resultPos] = newSymNode(res)
   result.ast = n
   incl result.flags, sfFromGeneric
+  incl result.flags, sfNeverRaises
 
-proc searchObjCase(obj: PNode; field: PSym): PNode =
+proc searchObjCaseImpl(obj: PNode; field: PSym): PNode =
   case obj.kind
   of nkSym:
     result = nil
   of nkElse, nkOfBranch:
-    result = searchObjCase(obj.lastSon, field)
+    result = searchObjCaseImpl(obj.lastSon, field)
   else:
     if obj.kind == nkRecCase and obj[0].kind == nkSym and obj[0].sym == field:
       result = obj
     else:
       for x in obj:
-        result = searchObjCase(x, field)
+        result = searchObjCaseImpl(x, field)
         if result != nil: break
 
+proc searchObjCase(t: PType; field: PSym): PNode =
+  result = searchObjCaseImpl(t.n, field)
+  if result == nil and t.len > 0:
+    result = searchObjCase(t[0].skipTypes({tyAlias, tyGenericInst, tyRef, tyPtr}), field)
+  doAssert result != nil
+
 proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph): PSym =
   result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), t.owner, info)
 
@@ -67,7 +74,7 @@ proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGra
   result.typ = newType(tyProc, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
-  addSon(result.typ.n, newNodeI(nkEffectList, info))
+  result.typ.n.add newNodeI(nkEffectList, info)
 
   result.typ.addParam dest
 
@@ -75,12 +82,11 @@ proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGra
   var caseStmt = newNodeI(nkCaseStmt, info)
   caseStmt.add(newSymNode dest)
 
-  let subObj = searchObjCase(t.n, field)
-  doAssert subObj != nil
-  for i in 1 ..< subObj.len:
+  let subObj = searchObjCase(t, field)
+  for i in 1..<subObj.len:
     let ofBranch = subObj[i]
     var newBranch = newNodeI(ofBranch.kind, ofBranch.info)
-    for j in 0..ofBranch.len-2:
+    for j in 0..<ofBranch.len-1:
       newBranch.add ofBranch[j]
 
     newBranch.add newTree(nkStmtList, newTree(nkFastAsgn, newSymNode(res), newIntNode(nkInt8Lit, i)))
@@ -89,10 +95,11 @@ proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGra
   body.add(caseStmt)
 
   var n = newNodeI(nkProcDef, info, bodyPos+2)
-  for i in 0 ..< n.len: n.sons[i] = newNodeI(nkEmpty, info)
-  n.sons[namePos] = newSymNode(result)
-  n.sons[paramsPos] = result.typ.n
-  n.sons[bodyPos] = body
-  n.sons[resultPos] = newSymNode(res)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = body
+  n[resultPos] = newSymNode(res)
   result.ast = n
   incl result.flags, sfFromGeneric
+  incl result.flags, sfNeverRaises

compiler/evalffi.nim

@@ -9,13 +9,18 @@
 
 ## This file implements the FFI part of the evaluator for Nim code.
 
-import ast, astalgo, ropes, types, options, tables, dynlib, msgs, os, lineinfos
+import ast, types, options, tables, dynlib, msgs, lineinfos
+from os import getAppFilename
 import pkg/libffi
 
 when defined(windows):
   const libcDll = "msvcrt.dll"
 elif defined(linux):
   const libcDll = "libc.so(.6|.5|)"
+elif defined(openbsd):
+  const libcDll = "/usr/lib/libc.so(.95.1|)"
+elif defined(bsd):
+  const libcDll = "/lib/libc.so.7"
 elif defined(osx):
   const libcDll = "/usr/lib/libSystem.dylib"
 else:
@@ -27,7 +32,7 @@ var
   gDllCache = initTable[string, LibHandle]()
 
 when defined(windows):
-  var gExeHandle = loadLib(os.getAppFilename())
+  var gExeHandle = loadLib(getAppFilename())
 else:
   var gExeHandle = loadLib()
 
@@ -46,24 +51,19 @@ proc getDll(conf: ConfigRef, cache: var TDllCache; dll: string; info: TLineInfo)
 const
   nkPtrLit = nkIntLit # hopefully we can get rid of this hack soon
 
-var myerrno {.importc: "errno", header: "<errno.h>".}: cint ## error variable
-
 proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode =
-  let name = $sym.loc.r
+  let name = sym.cname # $sym.loc.r would point to internal name
   # the AST does not support untyped pointers directly, so we use an nkIntLit
   # that contains the address instead:
   result = newNodeIT(nkPtrLit, sym.info, sym.typ)
-  case name
-  of "stdin":  result.intVal = cast[ByteAddress](system.stdin)
-  of "stdout": result.intVal = cast[ByteAddress](system.stdout)
-  of "stderr": result.intVal = cast[ByteAddress](system.stderr)
-  of "vmErrnoWrapper": result.intVal = cast[ByteAddress](myerrno)
-  else:
+  when true:
+    var libPathMsg = ""
     let lib = sym.annex
     if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}:
       globalError(conf, sym.info, "dynlib needs to be a string lit")
     var theAddr: pointer
     if (lib.isNil or lib.kind == libHeader) and not gExeHandle.isNil:
+      libPathMsg = "current exe: " & getAppFilename() & " nor libc: " & libcDll
       # first try this exe itself:
       theAddr = gExeHandle.symAddr(name)
       # then try libc:
@@ -72,12 +72,14 @@ proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode =
         theAddr = dllhandle.symAddr(name)
     elif not lib.isNil:
       let dll = if lib.kind == libHeader: libcDll else: lib.path.strVal
+      libPathMsg = dll
       let dllhandle = getDll(conf, gDllCache, dll, sym.info)
       theAddr = dllhandle.symAddr(name)
-    if theAddr.isNil: globalError(conf, sym.info, "cannot import: " & sym.name.s)
+    if theAddr.isNil: globalError(conf, sym.info,
+      "cannot import symbol: " & name & " from " & libPathMsg)
     result.intVal = cast[ByteAddress](theAddr)
 
-proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType =
+proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type =
   if t == nil: return addr libffi.type_void
 
   case t.kind
@@ -94,7 +96,7 @@ proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType =
      tyTyped, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
     result = addr libffi.type_pointer
   of tyDistinct, tyAlias, tySink:
-    result = mapType(conf, t.sons[0])
+    result = mapType(conf, t[0])
   else:
     result = nil
   # too risky:
@@ -102,7 +104,7 @@ proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType =
 
 proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI =
   case cc
-  of ccDefault: result = DEFAULT_ABI
+  of ccNimCall: result = DEFAULT_ABI
   of ccStdCall: result = when defined(windows) and defined(x86): STDCALL else: DEFAULT_ABI
   of ccCDecl: result = DEFAULT_ABI
   else:
@@ -120,16 +122,16 @@ proc packSize(conf: ConfigRef, v: PNode, typ: PType): int =
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     else:
-      result = sizeof(pointer) + packSize(conf, v.sons[0], typ.lastSon)
+      result = sizeof(pointer) + packSize(conf, v[0], typ.lastSon)
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    result = packSize(conf, v, typ.sons[0])
+    result = packSize(conf, v, typ[0])
   of tyArray:
     # consider: ptr array[0..1000_000, int] which is common for interfacing;
     # we use the real length here instead
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     elif v.len != 0:
-      result = v.len * packSize(conf, v.sons[0], typ.sons[1])
+      result = v.len * packSize(conf, v[0], typ[1])
   else:
     result = getSize(conf, typ).int
 
@@ -138,16 +140,16 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer)
 proc getField(conf: ConfigRef, n: PNode; position: int): PSym =
   case n.kind
   of nkRecList:
-    for i in 0 ..< len(n):
-      result = getField(conf, n.sons[i], position)
+    for i in 0..<n.len:
+      result = getField(conf, n[i], position)
       if result != nil: return
   of nkRecCase:
-    result = getField(conf, n.sons[0], position)
+    result = getField(conf, n[0], position)
     if result != nil: return
-    for i in 1 ..< len(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = getField(conf, lastSon(n.sons[i]), position)
+        result = getField(conf, lastSon(n[i]), position)
         if result != nil: return
       else: internalError(conf, n.info, "getField(record case branch)")
   of nkSym:
@@ -158,12 +160,12 @@ proc packObject(conf: ConfigRef, x: PNode, typ: PType, res: pointer) =
   internalAssert conf, x.kind in {nkObjConstr, nkPar, nkTupleConstr}
   # compute the field's offsets:
   discard getSize(conf, typ)
-  for i in ord(x.kind == nkObjConstr) ..< len(x):
-    var it = x.sons[i]
+  for i in ord(x.kind == nkObjConstr)..<x.len:
+    var it = x[i]
     if it.kind == nkExprColonExpr:
-      internalAssert conf, it.sons[0].kind == nkSym
-      let field = it.sons[0].sym
-      pack(conf, it.sons[1], field.typ, res +! field.offset)
+      internalAssert conf, it[0].kind == nkSym
+      let field = it[0].sym
+      pack(conf, it[1], field.typ, res +! field.offset)
     elif typ.n != nil:
       let field = getField(conf, typ.n, i)
       pack(conf, it, field.typ, res +! field.offset)
@@ -224,19 +226,19 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
         packRecCheck = 0
         globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ))
       inc packRecCheck
-      pack(conf, v.sons[0], typ.lastSon, res +! sizeof(pointer))
+      pack(conf, v[0], typ.lastSon, res +! sizeof(pointer))
       dec packRecCheck
       awr(pointer, res +! sizeof(pointer))
   of tyArray:
-    let baseSize = getSize(conf, typ.sons[1])
-    for i in 0 ..< v.len:
-      pack(conf, v.sons[i], typ.sons[1], res +! i * baseSize)
+    let baseSize = getSize(conf, typ[1])
+    for i in 0..<v.len:
+      pack(conf, v[i], typ[1], res +! i * baseSize)
   of tyObject, tyTuple:
     packObject(conf, v, typ, res)
   of tyNil:
     discard
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    pack(conf, v, typ.sons[0], res)
+    pack(conf, v, typ[0], res)
   else:
     globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ))
 
@@ -245,14 +247,14 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode
 proc unpackObjectAdd(conf: ConfigRef, x: pointer, n, result: PNode) =
   case n.kind
   of nkRecList:
-    for i in 0 ..< len(n):
-      unpackObjectAdd(conf, x, n.sons[i], result)
+    for i in 0..<n.len:
+      unpackObjectAdd(conf, x, n[i], result)
   of nkRecCase:
     globalError(conf, result.info, "case objects cannot be unpacked")
   of nkSym:
     var pair = newNodeI(nkExprColonExpr, result.info, 2)
-    pair.sons[0] = n
-    pair.sons[1] = unpack(conf, x +! n.sym.offset, n.sym.typ, nil)
+    pair[0] = n
+    pair[1] = unpack(conf, x +! n.sym.offset, n.sym.typ, nil)
     #echo "offset: ", n.sym.name.s, " ", n.sym.offset
     result.add pair
   else: discard
@@ -275,15 +277,15 @@ proc unpackObject(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
       globalError(conf, n.info, "cannot map value from FFI")
     if typ.n.isNil:
       globalError(conf, n.info, "cannot unpack unnamed tuple")
-    for i in ord(n.kind == nkObjConstr) ..< len(n):
-      var it = n.sons[i]
+    for i in ord(n.kind == nkObjConstr)..<n.len:
+      var it = n[i]
       if it.kind == nkExprColonExpr:
-        internalAssert conf, it.sons[0].kind == nkSym
-        let field = it.sons[0].sym
-        it.sons[1] = unpack(conf, x +! field.offset, field.typ, it.sons[1])
+        internalAssert conf, it[0].kind == nkSym
+        let field = it[0].sym
+        it[1] = unpack(conf, x +! field.offset, field.typ, it[1])
       else:
         let field = getField(conf, typ.n, i)
-        n.sons[i] = unpack(conf, x +! field.offset, field.typ, it)
+        n[i] = unpack(conf, x +! field.offset, field.typ, it)
 
 proc unpackArray(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   if n.isNil:
@@ -294,9 +296,9 @@ proc unpackArray(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
     result = n
     if result.kind != nkBracket:
       globalError(conf, n.info, "cannot map value from FFI")
-  let baseSize = getSize(conf, typ.sons[1])
-  for i in 0 ..< result.len:
-    result.sons[i] = unpack(conf, x +! i * baseSize, typ.sons[1], result.sons[i])
+  let baseSize = getSize(conf, typ[1])
+  for i in 0..<result.len:
+    result[i] = unpack(conf, x +! i * baseSize, typ[1], result[i])
 
 proc canonNodeKind(k: TNodeKind): TNodeKind =
   case k
@@ -327,7 +329,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
     else:
       reset n[]
       result = n
-      result.kind = nkNilLit
+      result[] = TNode(kind: nkNilLit)
       result.typ = typ
 
   template awi(kind, v: untyped): untyped = aw(kind, v, intVal)
@@ -376,7 +378,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
       awi(nkPtrLit, cast[ByteAddress](p))
     elif n != nil and n.len == 1:
       internalAssert(conf, n.kind == nkRefTy)
-      n.sons[0] = unpack(conf, p, typ.lastSon, n.sons[0])
+      n[0] = unpack(conf, p, typ.lastSon, n[0])
       result = n
     else:
       globalError(conf, n.info, "cannot map value from FFI " & typeToString(typ))
@@ -419,41 +421,41 @@ proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode =
     dealloc a
 
 proc callForeignFunction*(conf: ConfigRef, call: PNode): PNode =
-  internalAssert conf, call.sons[0].kind == nkPtrLit
+  internalAssert conf, call[0].kind == nkPtrLit
 
   var cif: TCif
-  var sig: TParamList
+  var sig: ParamList
   # use the arguments' types for varargs support:
-  for i in 1..call.len-1:
-    sig[i-1] = mapType(conf, call.sons[i].typ)
+  for i in 1..<call.len:
+    sig[i-1] = mapType(conf, call[i].typ)
     if sig[i-1].isNil:
       globalError(conf, call.info, "cannot map FFI type")
 
-  let typ = call.sons[0].typ
+  let typ = call[0].typ
   if prep_cif(cif, mapCallConv(conf, typ.callConv, call.info), cuint(call.len-1),
-              mapType(conf, typ.sons[0]), sig) != OK:
+              mapType(conf, typ[0]), sig) != OK:
     globalError(conf, call.info, "error in FFI call")
 
-  var args: TArgList
-  let fn = cast[pointer](call.sons[0].intVal)
-  for i in 1 .. call.len-1:
-    var t = call.sons[i].typ
-    args[i-1] = alloc0(packSize(conf, call.sons[i], t))
-    pack(conf, call.sons[i], t, args[i-1])
-  let retVal = if isEmptyType(typ.sons[0]): pointer(nil)
-               else: alloc(getSize(conf, typ.sons[0]).int)
+  var args: ArgList
+  let fn = cast[pointer](call[0].intVal)
+  for i in 1..<call.len:
+    var t = call[i].typ
+    args[i-1] = alloc0(packSize(conf, call[i], t))
+    pack(conf, call[i], t, args[i-1])
+  let retVal = if isEmptyType(typ[0]): pointer(nil)
+               else: alloc(getSize(conf, typ[0]).int)
 
   libffi.call(cif, fn, retVal, args)
 
   if retVal.isNil:
     result = newNode(nkEmpty)
   else:
-    result = unpack(conf, retVal, typ.sons[0], nil)
+    result = unpack(conf, retVal, typ[0], nil)
     result.info = call.info
 
   if retVal != nil: dealloc retVal
-  for i in 1 .. call.len-1:
-    call.sons[i] = unpack(conf, args[i-1], typ.sons[i], call[i])
+  for i in 1..<call.len:
+    call[i] = unpack(conf, args[i-1], typ[i], call[i])
     dealloc args[i-1]
 
 proc callForeignFunction*(conf: ConfigRef, fn: PNode, fntyp: PType,
@@ -462,39 +464,39 @@ proc callForeignFunction*(conf: ConfigRef, fn: PNode, fntyp: PType,
   internalAssert conf, fn.kind == nkPtrLit
 
   var cif: TCif
-  var sig: TParamList
+  var sig: ParamList
   for i in 0..len-1:
     var aTyp = args[i+start].typ
     if aTyp.isNil:
       internalAssert conf, i+1 < fntyp.len
-      aTyp = fntyp.sons[i+1]
+      aTyp = fntyp[i+1]
       args[i+start].typ = aTyp
     sig[i] = mapType(conf, aTyp)
     if sig[i].isNil: globalError(conf, info, "cannot map FFI type")
 
   if prep_cif(cif, mapCallConv(conf, fntyp.callConv, info), cuint(len),
-              mapType(conf, fntyp.sons[0]), sig) != OK:
+              mapType(conf, fntyp[0]), sig) != OK:
     globalError(conf, info, "error in FFI call")
 
-  var cargs: TArgList
+  var cargs: ArgList
   let fn = cast[pointer](fn.intVal)
-  for i in 0 .. len-1:
+  for i in 0..len-1:
     let t = args[i+start].typ
     cargs[i] = alloc0(packSize(conf, args[i+start], t))
     pack(conf, args[i+start], t, cargs[i])
-  let retVal = if isEmptyType(fntyp.sons[0]): pointer(nil)
-               else: alloc(getSize(conf, fntyp.sons[0]).int)
+  let retVal = if isEmptyType(fntyp[0]): pointer(nil)
+               else: alloc(getSize(conf, fntyp[0]).int)
 
   libffi.call(cif, fn, retVal, cargs)
 
   if retVal.isNil:
     result = newNode(nkEmpty)
   else:
-    result = unpack(conf, retVal, fntyp.sons[0], nil)
+    result = unpack(conf, retVal, fntyp[0], nil)
     result.info = info
 
   if retVal != nil: dealloc retVal
-  for i in 0 .. len-1:
+  for i in 0..len-1:
     let t = args[i+start].typ
     args[i+start] = unpack(conf, cargs[i], t, args[i+start])
     dealloc cargs[i]

compiler/evaltempl.nim

@@ -21,6 +21,7 @@ type
                       # new symbol
     config: ConfigRef
     ic: IdentCache
+    instID: int
 
 proc copyNode(ctx: TemplCtx, a, b: PNode): PNode =
   result = copyNode(a)
@@ -39,10 +40,10 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
     var s = templ.sym
     if (s.owner == nil and s.kind == skParam) or s.owner == c.owner:
       if s.kind == skParam and {sfGenSym, sfTemplateParam} * s.flags == {sfTemplateParam}:
-        handleParam actual.sons[s.position]
+        handleParam actual[s.position]
       elif (s.owner != nil) and (s.kind == skGenericParam or
            s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam):
-        handleParam actual.sons[s.owner.typ.len + s.position - 1]
+        handleParam actual[s.owner.typ.len + s.position - 1]
       else:
         internalAssert c.config, sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
@@ -53,8 +54,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
           #if x.kind == skParam and x.owner.kind == skModule:
           #  internalAssert c.config, false
           idTablePut(c.mapping, s, x)
-        if sfGenSym in s.flags and optNimV019 notin c.config.globalOptions:
-          result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $x.id),
+        if sfGenSym in s.flags:
+          result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $c.instID),
             if c.instLines: actual.info else: templ.info)
         else:
           result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
@@ -68,8 +69,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
     # "declarative" context (bug #9235).
     if c.isDeclarative:
       var res = copyNode(c, templ, actual)
-      for i in 0 ..< len(templ):
-        evalTemplateAux(templ.sons[i], actual, c, res)
+      for i in 0..<templ.len:
+        evalTemplateAux(templ[i], actual, c, res)
       result.add res
     else:
       result.add newNodeI(nkEmpty, templ.info)
@@ -82,8 +83,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
       c.isDeclarative = true
       isDeclarative = true
     var res = copyNode(c, templ, actual)
-    for i in 0 ..< len(templ):
-      evalTemplateAux(templ.sons[i], actual, c, res)
+    for i in 0..<templ.len:
+      evalTemplateAux(templ[i], actual, c, res)
     result.add res
     if isDeclarative: c.isDeclarative = false
 
@@ -122,22 +123,22 @@ proc evalTemplateArgs(n: PNode, s: PSym; conf: ConfigRef; fromHlo: bool): PNode
                 n.renderTree)
 
   result = newNodeI(nkArgList, n.info)
-  for i in 1 .. givenRegularParams:
-    result.addSon n[i]
+  for i in 1..givenRegularParams:
+    result.add n[i]
 
   # handle parameters with default values, which were
   # not supplied by the user
-  for i in givenRegularParams+1 .. expectedRegularParams:
-    let default = s.typ.n.sons[i].sym.ast
+  for i in givenRegularParams+1..expectedRegularParams:
+    let default = s.typ.n[i].sym.ast
     if default.isNil or default.kind == nkEmpty:
       localError(conf, n.info, errWrongNumberOfArguments)
-      addSon(result, newNodeI(nkEmpty, n.info))
+      result.add newNodeI(nkEmpty, n.info)
     else:
-      addSon(result, default.copyTree)
+      result.add default.copyTree
 
   # add any generic parameters
-  for i in 1 .. genericParams:
-    result.addSon n.sons[givenRegularParams + i]
+  for i in 1..genericParams:
+    result.add n[givenRegularParams + i]
 
 # to prevent endless recursion in template instantiation
 const evalTemplateLimit* = 1000
@@ -155,7 +156,7 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
       if x.kind in nkCallKinds:
         for i in 1..<x.len:
           if x[i].kind in nkCallKinds:
-            x.sons[i].info = info
+            x[i].info = info
   else:
     result = newNodeI(nkStmtListExpr, info)
     var d = newNodeI(nkComesFrom, info)
@@ -166,7 +167,7 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
 
 proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
                    conf: ConfigRef;
-                   ic: IdentCache;
+                   ic: IdentCache; instID: ref int;
                    fromHlo=false): PNode =
   inc(conf.evalTemplateCounter)
   if conf.evalTemplateCounter > evalTemplateLimit:
@@ -181,13 +182,14 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   ctx.config = conf
   ctx.ic = ic
   initIdTable(ctx.mapping)
+  ctx.instID = instID[]
 
   let body = tmpl.getBody
   #echo "instantion of ", renderTree(body, {renderIds})
   if isAtom(body):
     result = newNodeI(nkPar, body.info)
     evalTemplateAux(body, args, ctx, result)
-    if result.len == 1: result = result.sons[0]
+    if result.len == 1: result = result[0]
     else:
       localError(conf, result.info, "illformed AST: " &
                   renderTree(result, {renderNoComments}))
@@ -196,10 +198,12 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
     ctx.instLines = sfCallsite in tmpl.flags
     if ctx.instLines:
       result.info = n.info
-    for i in 0 ..< safeLen(body):
-      evalTemplateAux(body.sons[i], args, ctx, result)
+    for i in 0..<body.safeLen:
+      evalTemplateAux(body[i], args, ctx, result)
   result.flags.incl nfFromTemplate
   result = wrapInComesFrom(n.info, tmpl, result)
   #if ctx.debugActive:
   #  echo "instantion of ", renderTree(result, {renderIds})
   dec(conf.evalTemplateCounter)
+  # The instID must be unique for every template instantiation, so we increment it here
+  inc instID[]

compiler/extccomp.nim

@@ -14,7 +14,7 @@
 
 import
   ropes, os, strutils, osproc, platform, condsyms, options, msgs,
-  lineinfos, std / sha1, streams, pathutils, sequtils, times
+  lineinfos, std / sha1, streams, pathutils, sequtils, times, strtabs
 
 type
   TInfoCCProp* = enum         # properties of the C compiler:
@@ -48,6 +48,8 @@ type
                          # used on some platforms
     asmStmtFrmt: string, # format of ASM statement
     structStmtFmt: string, # Format for struct statement
+    produceAsm: string,  # Format how to produce assembler listings
+    cppXsupport: string, # what to do to enable C++X support
     props: TInfoCCProps] # properties of the C compiler
 
 
@@ -58,13 +60,16 @@ type
 template compiler(name, settings: untyped): untyped =
   proc name: TInfoCC {.compileTime.} = settings
 
+const
+  gnuAsmListing = "-Wa,-acdl=$asmfile -g -fverbose-asm -masm=intel"
+
 # GNU C and C++ Compiler
 compiler gcc:
   result = (
     name: "gcc",
     objExt: "o",
-    optSpeed: " -O3 ",
-    optSize: " -Os ",
+    optSpeed: " -O3 -fno-ident",
+    optSize: " -Os -fno-ident",
     compilerExe: "gcc",
     cppCompiler: "g++",
     compileTmpl: "-c $options $include -o $objfile $file",
@@ -80,6 +85,8 @@ compiler gcc:
     pic: "-fPIC",
     asmStmtFrmt: "asm($1);$n",
     structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name
+    produceAsm: gnuAsmListing,
+    cppXsupport: "-std=gnu++14 -funsigned-char",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm,
             hasAttribute})
 
@@ -105,6 +112,8 @@ compiler nintendoSwitchGCC:
     pic: "-fPIE",
     asmStmtFrmt: "asm($1);$n",
     structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name
+    produceAsm: gnuAsmListing,
+    cppXsupport: "-std=gnu++14 -funsigned-char",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm,
             hasAttribute})
 
@@ -134,16 +143,16 @@ compiler vcc:
   result = (
     name: "vcc",
     objExt: "obj",
-    optSpeed: " /Ogityb2 /G7 ",
-    optSize: " /O1 /G7 ",
+    optSpeed: " /Ogityb2 ",
+    optSize: " /O1 ",
     compilerExe: "cl",
     cppCompiler: "cl",
-    compileTmpl: "/c$vccplatform $options $include /Fo$objfile $file",
-    buildGui: " /link /SUBSYSTEM:WINDOWS ",
+    compileTmpl: "/c$vccplatform $options $include /nologo /Fo$objfile $file",
+    buildGui: " /SUBSYSTEM:WINDOWS user32.lib ",
     buildDll: " /LD",
     buildLib: "lib /OUT:$libfile $objfiles",
     linkerExe: "cl",
-    linkTmpl: "$options $builddll$vccplatform /Fe$exefile $objfiles $buildgui",
+    linkTmpl: "$builddll$vccplatform /Fe$exefile $objfiles $buildgui /nologo $options",
     includeCmd: " /I",
     linkDirCmd: " /LIBPATH:",
     linkLibCmd: " $1.lib",
@@ -151,6 +160,8 @@ compiler vcc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$3$n$1 $2",
+    produceAsm: "/Fa$asmfile",
+    cppXsupport: "",
     props: {hasCpp, hasAssume, hasDeclspec})
 
 compiler clangcl:
@@ -159,6 +170,7 @@ compiler clangcl:
   result.compilerExe = "clang-cl"
   result.cppCompiler = "clang-cl"
   result.linkerExe = "clang-cl"
+  result.linkTmpl = "-fuse-ld=lld " & result.linkTmpl
 
 # Intel C/C++ Compiler
 compiler icl:
@@ -174,30 +186,6 @@ compiler icc:
   result.compilerExe = "icc"
   result.linkerExe = "icc"
 
-# Local C Compiler
-compiler lcc:
-  result = (
-    name: "lcc",
-    objExt: "obj",
-    optSpeed: " -O -p6 ",
-    optSize: " -O -p6 ",
-    compilerExe: "lcc",
-    cppCompiler: "",
-    compileTmpl: "$options $include -Fo$objfile $file",
-    buildGui: " -subsystem windows",
-    buildDll: " -dll",
-    buildLib: "", # XXX: not supported yet
-    linkerExe: "lcclnk",
-    linkTmpl: "$options $buildgui $builddll -O $exefile $objfiles",
-    includeCmd: " -I",
-    linkDirCmd: "", # XXX: not supported yet
-    linkLibCmd: "", # XXX: not supported yet
-    debug: " -g5 ",
-    pic: "",
-    asmStmtFrmt: "_asm{$n$1$n}$n",
-    structStmtFmt: "$1 $2",
-    props: {})
-
 # Borland C Compiler
 compiler bcc:
   result = (
@@ -220,57 +208,11 @@ compiler bcc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
+    produceAsm: "",
+    cppXsupport: "",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard,
             hasAttribute})
 
-# Digital Mars C Compiler
-compiler dmc:
-  result = (
-    name: "dmc",
-    objExt: "obj",
-    optSpeed: " -ff -o -6 ",
-    optSize: " -ff -o -6 ",
-    compilerExe: "dmc",
-    cppCompiler: "",
-    compileTmpl: "-c $options $include -o$objfile $file",
-    buildGui: " -L/exet:nt/su:windows",
-    buildDll: " -WD",
-    buildLib: "", # XXX: not supported yet
-    linkerExe: "dmc",
-    linkTmpl: "$options $buildgui $builddll -o$exefile $objfiles",
-    includeCmd: " -I",
-    linkDirCmd: "", # XXX: not supported yet
-    linkLibCmd: "", # XXX: not supported yet
-    debug: " -g ",
-    pic: "",
-    asmStmtFrmt: "__asm{$n$1$n}$n",
-    structStmtFmt: "$3$n$1 $2",
-    props: {hasCpp})
-
-# Watcom C Compiler
-compiler wcc:
-  result = (
-    name: "wcc",
-    objExt: "obj",
-    optSpeed: " -ox -on -6 -d0 -fp6 -zW ",
-    optSize: "",
-    compilerExe: "wcl386",
-    cppCompiler: "",
-    compileTmpl: "-c $options $include -fo=$objfile $file",
-    buildGui: " -bw",
-    buildDll: " -bd",
-    buildLib: "", # XXX: not supported yet
-    linkerExe: "wcl386",
-    linkTmpl: "$options $buildgui $builddll -fe=$exefile $objfiles ",
-    includeCmd: " -i=",
-    linkDirCmd: "", # XXX: not supported yet
-    linkLibCmd: "", # XXX: not supported yet
-    debug: " -d2 ",
-    pic: "",
-    asmStmtFrmt: "__asm{$n$1$n}$n",
-    structStmtFmt: "$1 $2",
-    props: {hasCpp})
-
 # Tiny C Compiler
 compiler tcc:
   result = (
@@ -291,45 +233,22 @@ compiler tcc:
     linkLibCmd: "", # XXX: not supported yet
     debug: " -g ",
     pic: "",
-    asmStmtFrmt: "__asm{$n$1$n}$n",
+    asmStmtFrmt: "asm($1);$n",
     structStmtFmt: "$1 $2",
-    props: {hasSwitchRange, hasComputedGoto})
-
-# Pelles C Compiler
-compiler pcc:
-  # Pelles C
-  result = (
-    name: "pcc",
-    objExt: "obj",
-    optSpeed: " -Ox ",
-    optSize: " -Os ",
-    compilerExe: "cc",
-    cppCompiler: "",
-    compileTmpl: "-c $options $include -Fo$objfile $file",
-    buildGui: " -SUBSYSTEM:WINDOWS",
-    buildDll: " -DLL",
-    buildLib: "", # XXX: not supported yet
-    linkerExe: "cc",
-    linkTmpl: "$options $buildgui $builddll -OUT:$exefile $objfiles",
-    includeCmd: " -I",
-    linkDirCmd: "", # XXX: not supported yet
-    linkLibCmd: "", # XXX: not supported yet
-    debug: " -Zi ",
-    pic: "",
-    asmStmtFrmt: "__asm{$n$1$n}$n",
-    structStmtFmt: "$1 $2",
-    props: {})
+    produceAsm: gnuAsmListing,
+    cppXsupport: "",
+    props: {hasSwitchRange, hasComputedGoto, hasGnuAsm})
 
 # Your C Compiler
-compiler ucc:
+compiler envcc:
   result = (
-    name: "ucc",
+    name: "env",
     objExt: "o",
     optSpeed: " -O3 ",
     optSize: " -O1 ",
-    compilerExe: "cc",
+    compilerExe: "",
     cppCompiler: "",
-    compileTmpl: "-c $options $include -o $objfile $file",
+    compileTmpl: "-c $ccenvflags $options $include -o $objfile $file",
     buildGui: "",
     buildDll: " -shared ",
     buildLib: "", # XXX: not supported yet
@@ -342,7 +261,9 @@ compiler ucc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
-    props: {})
+    produceAsm: "",
+    cppXsupport: "",
+    props: {hasGnuAsm})
 
 const
   CC*: array[succ(low(TSystemCC))..high(TSystemCC), TInfoCC] = [
@@ -350,33 +271,28 @@ const
     nintendoSwitchGCC(),
     llvmGcc(),
     clang(),
-    lcc(),
     bcc(),
-    dmc(),
-    wcc(),
     vcc(),
     tcc(),
-    pcc(),
-    ucc(),
+    envcc(),
     icl(),
     icc(),
     clangcl()]
 
   hExt* = ".h"
 
-proc libNameTmpl(conf: ConfigRef): string {.inline.} =
-  result = if conf.target.targetOS == osWindows: "$1.lib" else: "lib$1.a"
-
 proc nameToCC*(name: string): TSystemCC =
   ## Returns the kind of compiler referred to by `name`, or ccNone
   ## if the name doesn't refer to any known compiler.
-  for i in succ(ccNone) .. high(TSystemCC):
+  for i in succ(ccNone)..high(TSystemCC):
     if cmpIgnoreStyle(name, CC[i].name) == 0:
       return i
   result = ccNone
 
-proc listCCnames(): seq[string] =
-  for i in succ(ccNone) .. high(TSystemCC):
+proc listCCnames(): string =
+  result = ""
+  for i in succ(ccNone)..high(TSystemCC):
+    if i > succ(ccNone): result.add ", "
     result.add CC[i].name
 
 proc isVSCompatible*(conf: ConfigRef): bool =
@@ -387,15 +303,14 @@ proc isVSCompatible*(conf: ConfigRef): bool =
 proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
   # use ``cpu.os.cc`` for cross compilation, unless ``--compileOnly`` is given
   # for niminst support
-  let fullSuffix =
-    if conf.cmd == cmdCompileToCpp:
-      ".cpp" & suffix
-    elif conf.cmd == cmdCompileToOC:
-      ".objc" & suffix
-    elif conf.cmd == cmdCompileToJS:
-      ".js" & suffix
-    else:
-      suffix
+  var fullSuffix = suffix
+  case conf.backend
+  of backendCpp, backendJs, backendObjc: fullSuffix = "." & $conf.backend & suffix
+  of backendC: discard
+  of backendInvalid:
+    # during parsing of cfg files; we don't know the backend yet, no point in
+    # guessing wrong thing
+    return ""
 
   if (conf.target.hostOS != conf.target.targetOS or conf.target.hostCPU != conf.target.targetCPU) and
       optCompileOnly notin conf.globalOptions:
@@ -412,17 +327,16 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
 proc setCC*(conf: ConfigRef; ccname: string; info: TLineInfo) =
   conf.cCompiler = nameToCC(ccname)
   if conf.cCompiler == ccNone:
-    let ccList = listCCnames().join(", ")
-    localError(conf, info, "unknown C compiler: '$1'. Available options are: $2" % [ccname, ccList])
+    localError(conf, info, "unknown C compiler: '$1'. Available options are: $2" % [ccname, listCCnames()])
   conf.compileOptions = getConfigVar(conf, conf.cCompiler, ".options.always")
   conf.linkOptions = ""
   conf.cCompilerPath = getConfigVar(conf, conf.cCompiler, ".path")
-  for i in low(CC) .. high(CC): undefSymbol(conf.symbols, CC[i].name)
+  for c in CC: undefSymbol(conf.symbols, c.name)
   defineSymbol(conf.symbols, CC[conf.cCompiler].name)
 
 proc addOpt(dest: var string, src: string) =
-  if len(dest) == 0 or dest[len(dest)-1] != ' ': add(dest, " ")
-  add(dest, src)
+  if dest.len == 0 or dest[^1] != ' ': dest.add(" ")
+  dest.add(src)
 
 proc addLinkOption*(conf: ConfigRef; option: string) =
   addOpt(conf.linkOptions, option)
@@ -439,11 +353,11 @@ proc addCompileOptionCmd*(conf: ConfigRef; option: string) =
 
 proc initVars*(conf: ConfigRef) =
   # we need to define the symbol here, because ``CC`` may have never been set!
-  for i in low(CC) .. high(CC): undefSymbol(conf.symbols, CC[i].name)
+  for c in CC: undefSymbol(conf.symbols, c.name)
   defineSymbol(conf.symbols, CC[conf.cCompiler].name)
   addCompileOption(conf, getConfigVar(conf, conf.cCompiler, ".options.always"))
   #addLinkOption(getConfigVar(cCompiler, ".options.linker"))
-  if len(conf.cCompilerPath) == 0:
+  if conf.cCompilerPath.len == 0:
     conf.cCompilerPath = getConfigVar(conf, conf.cCompiler, ".path")
 
 proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile,
@@ -457,6 +371,13 @@ proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile =
 proc addFileToCompile*(conf: ConfigRef; cf: Cfile) =
   conf.toCompile.add(cf)
 
+proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) =
+  let key = completeCfilePath(conf, withPackageName(conf, nimfile)).string
+  var value = conf.cfileSpecificOptions.getOrDefault(key)
+  if strutils.find(value, option, 0) < 0:
+    addOpt(value, option)
+    conf.cfileSpecificOptions[key] = value
+
 proc resetCompilationLists*(conf: ConfigRef) =
   conf.toCompile.setLen 0
   ## XXX: we must associate these with their originating module
@@ -469,7 +390,7 @@ proc addExternalFileToLink*(conf: ConfigRef; filename: AbsoluteFile) =
   conf.externalToLink.insert(filename.string, 0)
 
 proc execWithEcho(conf: ConfigRef; cmd: string, msg = hintExecuting): int =
-  rawMessage(conf, msg, cmd)
+  rawMessage(conf, msg, if msg == hintLinking and not(optListCmd in conf.globalOptions or conf.verbosity > 1): "" else: cmd)
   result = execCmd(cmd)
 
 proc execExternalProgram*(conf: ConfigRef; cmd: string, msg = hintExecuting) =
@@ -481,10 +402,7 @@ proc generateScript(conf: ConfigRef; script: Rope) =
   let (_, name, _) = splitFile(conf.outFile.string)
   let filename = getNimcacheDir(conf) / RelativeFile(addFileExt("compile_" & name,
                                      platform.OS[conf.target.targetOS].scriptExt))
-  if writeRope(script, filename):
-    copyFile(conf.libpath / RelativeFile"nimbase.h",
-             getNimcacheDir(conf) / RelativeFile"nimbase.h")
-  else:
+  if not writeRope(script, filename):
     rawMessage(conf, errGenerated, "could not write to file: " & filename.string)
 
 proc getOptSpeed(conf: ConfigRef; c: TSystemCC): string =
@@ -509,8 +427,9 @@ proc noAbsolutePaths(conf: ConfigRef): bool {.inline.} =
   # `optGenMapping` is included here for niminst.
   result = conf.globalOptions * {optGenScript, optGenMapping} != {}
 
-proc cFileSpecificOptions(conf: ConfigRef; nimname: string): string =
+proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string =
   result = conf.compileOptions
+
   for option in conf.compileOptionsCmd:
     if strutils.find(result, option, 0) < 0:
       addOpt(result, option)
@@ -530,21 +449,22 @@ proc cFileSpecificOptions(conf: ConfigRef; nimname: string): string =
   let key = nimname & ".always"
   if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key))
 
+  addOpt(result, conf.cfileSpecificOptions.getOrDefault(fullNimFile))
+
 proc getCompileOptions(conf: ConfigRef): string =
-  result = cFileSpecificOptions(conf, "__dummy__")
+  result = cFileSpecificOptions(conf, "__dummy__", "__dummy__")
 
 proc vccplatform(conf: ConfigRef): string =
   # VCC specific but preferable over the config hacks people
   # had to do before, see #11306
-  case conf.target.targetCPU
-  of cpuI386:
-    result = " --platform:x86"
-  of cpuArm:
-    result = " --platform:arm"
-  of cpuAmd64:
-    result = " --platform:amd64"
-  else:
-    result = ""
+  if conf.cCompiler == ccVcc:
+    let exe = getConfigVar(conf, conf.cCompiler, ".exe")
+    if "vccexe.exe" == extractFilename(exe):
+      result = case conf.target.targetCPU
+        of cpuI386: " --platform:x86"
+        of cpuArm: " --platform:arm"
+        of cpuAmd64: " --platform:amd64"
+        else: ""
 
 proc getLinkOptions(conf: ConfigRef): string =
   result = conf.linkOptions & " " & conf.linkOptionsCmd & " "
@@ -557,43 +477,96 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} =
   result = (optGenScript in conf.globalOptions and conf.target.targetOS == osWindows) or
            (conf.target.hostOS == osWindows)
 
+proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool =
+  conf.backend == backendCpp and not cfile.string.endsWith(".c")
+
+proc envFlags(conf: ConfigRef): string =
+  result = if conf.backend == backendCpp:
+            getEnv("CXXFLAGS")
+          else:
+            getEnv("CFLAGS")
+
 proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string =
-  result = if conf.cmd == cmdCompileToCpp and not cfile.string.endsWith(".c"):
-             CC[compiler].cppCompiler
-           else:
-             CC[compiler].compilerExe
+  if compiler == ccEnv:
+    result = if useCpp(conf, cfile):
+              getEnv("CXX")
+            else:
+              getEnv("CC")
+  else:
+    result = if useCpp(conf, cfile):
+              CC[compiler].cppCompiler
+            else:
+              CC[compiler].compilerExe
   if result.len == 0:
     rawMessage(conf, errGenerated,
       "Compiler '$1' doesn't support the requested target" %
       CC[compiler].name)
 
+proc ccHasSaneOverflow*(conf: ConfigRef): bool =
+  if conf.cCompiler == ccGcc:
+    result = false # assume an old or crappy GCC
+    var exe = getConfigVar(conf, conf.cCompiler, ".exe")
+    if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
+    let (s, exitCode) = try: execCmdEx(exe & " --version") except: ("", 1)
+    if exitCode == 0:
+      var i = 0
+      var j = 0
+      # the version is the last part of the first line:
+      while i < s.len and s[i] != '\n':
+        if s[i] in {' ', '\t'}: j = i+1
+        inc i
+      if j > 0:
+        var major = 0
+        while j < s.len and s[j] in {'0'..'9'}:
+          major = major * 10 + (ord(s[j]) - ord('0'))
+          inc j
+        if i < s.len and s[j] == '.': inc j
+        while j < s.len and s[j] in {'0'..'9'}:
+          inc j
+        if j+1 < s.len and s[j] == '.' and s[j+1] in {'0'..'9'}:
+          # we found a third version number, chances are high
+          # we really parsed the version:
+          result = major >= 5
+  else:
+    result = conf.cCompiler == ccCLang
+
 proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
-           elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler
+           elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler
            else: getCompilerExe(conf, compiler, AbsoluteFile"")
 
-proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): string =
-  var c = conf.cCompiler
-  var options = cFileSpecificOptions(conf, cfile.nimname)
+proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
+                         isMainFile = false; produceOutput = false): string =
+  let c = conf.cCompiler
+  # We produce files like module.nim.cpp, so the absolute Nim filename is not
+  # cfile.name but `cfile.cname.changeFileExt("")`:
+  var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string)
+  if useCpp(conf, cfile.cname):
+    # needs to be prepended so that --passc:-std=c++17 can override default.
+    # we could avoid allocation by making cFileSpecificOptions inplace
+    options = CC[c].cppXsupport & ' ' & options
+
   var exe = getConfigVar(conf, c, ".exe")
   if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname)
 
   if needsExeExt(conf): exe = addFileExt(exe, "exe")
   if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and
       ospNeedsPIC in platform.OS[conf.target.targetOS].props:
-    add(options, ' ' & CC[c].pic)
+    options.add(' ' & CC[c].pic)
 
-  var includeCmd, compilePattern: string
+  if cfile.customArgs != "":
+    options.add ' '
+    options.add cfile.customArgs
+
+  var compilePattern: string
+  # compute include paths:
+  var includeCmd = CC[c].includeCmd & quoteShell(conf.libpath)
   if not noAbsolutePaths(conf):
-    # compute include paths:
-    includeCmd = CC[c].includeCmd & quoteShell(conf.libpath)
-
     for includeDir in items(conf.cIncludes):
       includeCmd.add(join([CC[c].includeCmd, includeDir.quoteShell]))
 
     compilePattern = joinPath(conf.cCompilerPath, exe)
   else:
-    includeCmd = ""
     compilePattern = getCompilerExe(conf, c, cfile.cname)
 
   includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)]))
@@ -603,7 +576,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): str
 
   var objfile =
     if cfile.obj.isEmpty:
-      if not cfile.flags.contains(CfileFlag.External) or noAbsolutePaths(conf):
+      if CfileFlag.External notin cfile.flags or noAbsolutePaths(conf):
         toObjFile(conf, cf).string
       else:
         completeCfilePath(conf, toObjFile(conf, cf)).string
@@ -614,23 +587,36 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): str
 
   # D files are required by nintendo switch libs for
   # compilation. They are basically a list of all includes.
-  let dfile = objfile.changeFileExt(".d").quoteShell()
+  let dfile = objfile.changeFileExt(".d").quoteShell
 
-  objfile = quoteShell(objfile)
   let cfsh = quoteShell(cf)
   result = quoteShell(compilePattern % [
     "dfile", dfile,
-    "file", cfsh, "objfile", objfile, "options", options,
+    "file", cfsh, "objfile", quoteShell(objfile), "options", options,
     "include", includeCmd, "nim", getPrefixDir(conf).string,
-    "lib", conf.libpath.string])
-  add(result, ' ')
-  addf(result, CC[c].compileTmpl, [
+    "lib", conf.libpath.string,
+    "ccenvflags", envFlags(conf)])
+
+  if optProduceAsm in conf.globalOptions:
+    if CC[conf.cCompiler].produceAsm.len > 0:
+      let asmfile = objfile.changeFileExt(".asm").quoteShell
+      addOpt(result, CC[conf.cCompiler].produceAsm % ["asmfile", asmfile])
+      if produceOutput:
+        rawMessage(conf, hintUserRaw, "Produced assembler here: " & asmfile)
+    else:
+      if produceOutput:
+        rawMessage(conf, hintUserRaw, "Couldn't produce assembler listing " &
+          "for the selected C compiler: " & CC[conf.cCompiler].name)
+
+  result.add(' ')
+  result.addf(CC[c].compileTmpl, [
     "dfile", dfile,
-    "file", cfsh, "objfile", objfile,
+    "file", cfsh, "objfile", quoteShell(objfile),
     "options", options, "include", includeCmd,
     "nim", quoteShell(getPrefixDir(conf)),
     "lib", quoteShell(conf.libpath),
-    "vccplatform", vccplatform(conf)])
+    "vccplatform", vccplatform(conf),
+    "ccenvflags", envFlags(conf)])
 
 proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
   result = secureHash(
@@ -641,8 +627,7 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
     getCompileCFileCmd(conf, cfile))
 
 proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
-  if conf.cmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM}:
-    return false
+  if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good
 
   var hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1")
   var currentHash = footprint(conf, cfile)
@@ -662,6 +647,10 @@ proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) =
   if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and
       not externalFileChanged(conf, c):
     c.flags.incl CfileFlag.Cached
+  else:
+    # make sure Nim keeps recompiling the external file on reruns
+    # if compilation is not successful
+    discard tryRemoveFile(c.obj.string)
   conf.toCompile.add(c)
 
 proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) =
@@ -670,37 +659,14 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) =
     flags: {CfileFlag.External})
   addExternalFileToCompile(conf, c)
 
-proc compileCFiles(conf: ConfigRef; list: CfileList, script: var Rope, cmds: var TStringSeq,
-                  prettyCmds: var TStringSeq) =
-  var currIdx = 0
-  for it in list:
-    # call the C compiler for the .c file:
-    if it.flags.contains(CfileFlag.Cached): continue
-    var compileCmd = getCompileCFileCmd(conf, it, currIdx == list.len - 1)
-    inc currIdx
-    if optCompileOnly notin conf.globalOptions:
-      add(cmds, compileCmd)
-      let (_, name, _) = splitFile(it.cname)
-      add(prettyCmds, if hintCC in conf.notes: "CC: " & demanglePackageName(name) else: "")
-    if optGenScript in conf.globalOptions:
-      add(script, compileCmd)
-      add(script, "\n")
-
 proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
                 objfiles: string, isDllBuild: bool): string =
   if optGenStaticLib in conf.globalOptions:
-    var libname: string
-    if not conf.outFile.isEmpty:
-      libname = conf.outFile.string.expandTilde
-      if not libname.isAbsolute():
-        libname = getCurrentDir() / libname
-    else:
-      libname = (libNameTmpl(conf) % splitFile(conf.projectName).name)
-    result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(libname),
+    result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(output),
                                             "objfiles", objfiles]
   else:
     var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe")
-    if len(linkerExe) == 0: linkerExe = getLinkerExe(conf, conf.cCompiler)
+    if linkerExe.len == 0: linkerExe = getLinkerExe(conf, conf.cCompiler)
     # bug #6452: We must not use ``quoteShell`` here for ``linkerExe``
     if needsExeExt(conf): linkerExe = addFileExt(linkerExe, "exe")
     if noAbsolutePaths(conf): result = linkerExe
@@ -730,7 +696,7 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
         "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles,
         "exefile", exefile, "nim", getPrefixDir(conf).string, "lib", conf.libpath.string])
     result.add ' '
-    addf(result, linkTmpl, ["builddll", builddll,
+    result.addf(linkTmpl, ["builddll", builddll,
         "mapfile", mapfile,
         "buildgui", buildgui, "options", linkOptions,
         "objfiles", objfiles, "exefile", exefile,
@@ -793,22 +759,26 @@ template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body
         (ose.msg & " " & $ose.errorCode))
     raise
 
+proc getExtraCmds(conf: ConfigRef; output: AbsoluteFile): seq[string] =
+  when defined(macosx):
+    if optCDebug in conf.globalOptions and optGenStaticLib notin conf.globalOptions:
+      # if needed, add an option to skip or override location
+      result.add "dsymutil " & $(output).quoteShell
+
 proc execLinkCmd(conf: ConfigRef; linkCmd: string) =
   tryExceptOSErrorMessage(conf, "invocation of external linker program failed."):
-    execExternalProgram(conf, linkCmd,
-      if optListCmd in conf.globalOptions or conf.verbosity > 1: hintExecuting else: hintLinking)
+    execExternalProgram(conf, linkCmd, hintLinking)
 
 proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx: int)) =
   let runCb = proc (idx: int, p: Process) =
     let exitCode = p.peekExitCode
     if exitCode != 0:
       rawMessage(conf, errGenerated, "execution of an external compiler program '" &
-        cmds[idx] & "' failed with exit code: " & $exitCode & "\n\n" &
-        p.outputStream.readAll.strip)
+        cmds[idx] & "' failed with exit code: " & $exitCode & "\n\n")
   if conf.numberOfProcessors == 0: conf.numberOfProcessors = countProcessors()
   var res = 0
   if conf.numberOfProcessors <= 1:
-    for i in 0 .. high(cmds):
+    for i in 0..high(cmds):
       tryExceptOSErrorMessage(conf, "invocation of external compiler program failed."):
         res = execWithEcho(conf, cmds[i])
       if res != 0:
@@ -816,15 +786,8 @@ proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx:
           cmds[i])
   else:
     tryExceptOSErrorMessage(conf, "invocation of external compiler program failed."):
-      if optListCmd in conf.globalOptions or conf.verbosity > 1:
-        res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath},
-                            conf.numberOfProcessors, afterRunEvent=runCb)
-      elif conf.verbosity == 1:
-        res = execProcesses(cmds, {poStdErrToStdOut, poUsePath},
+      res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams},
                             conf.numberOfProcessors, prettyCb, afterRunEvent=runCb)
-      else:
-        res = execProcesses(cmds, {poStdErrToStdOut, poUsePath},
-                            conf.numberOfProcessors, afterRunEvent=runCb)
   if res != 0:
     if conf.numberOfProcessors <= 1:
       rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" %
@@ -866,21 +829,38 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu
                    else: platform.OS[conf.target.targetOS].dllFrmt % basename
   result = conf.getNimcacheDir / RelativeFile(targetName)
 
+proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string =
+  if conf.hasHint(hintCC):
+    if optListCmd in conf.globalOptions or conf.verbosity > 1:
+      result = MsgKindToStr[hintCC] % (demanglePackageName(path.splitFile.name) & ": " & compileCmd)
+    else:
+      result = MsgKindToStr[hintCC] % demanglePackageName(path.splitFile.name)
+
 proc callCCompiler*(conf: ConfigRef) =
   var
     linkCmd: string
+    extraCmds: seq[string]
   if conf.globalOptions * {optCompileOnly, optGenScript} == {optCompileOnly}:
     return # speed up that call if only compiling and no script shall be
            # generated
   #var c = cCompiler
   var script: Rope = nil
-  var cmds: TStringSeq = @[]
-  var prettyCmds: TStringSeq = @[]
+  var cmds: TStringSeq
+  var prettyCmds: TStringSeq
   let prettyCb = proc (idx: int) =
-    when declared(echo):
-      let cmd = prettyCmds[idx]
-      if cmd != "": echo cmd
-  compileCFiles(conf, conf.toCompile, script, cmds, prettyCmds)
+    if prettyCmds[idx].len > 0: echo prettyCmds[idx]
+
+  for idx, it in conf.toCompile:
+    # call the C compiler for the .c file:
+    if CfileFlag.Cached in it.flags: continue
+    let compileCmd = getCompileCFileCmd(conf, it, idx == conf.toCompile.len - 1, produceOutput=true)
+    if optCompileOnly notin conf.globalOptions:
+      cmds.add(compileCmd)
+      prettyCmds.add displayProgressCC(conf, $it.cname, compileCmd)
+    if optGenScript in conf.globalOptions:
+      script.add(compileCmd)
+      script.add("\n")
+
   if optCompileOnly notin conf.globalOptions:
     execCmdsInParallel(conf, cmds, prettyCb)
   if optNoLinking notin conf.globalOptions:
@@ -888,8 +868,8 @@ proc callCCompiler*(conf: ConfigRef) =
     var objfiles = ""
     for it in conf.externalToLink:
       let objFile = if noAbsolutePaths(conf): it.extractFilename else: it
-      add(objfiles, ' ')
-      add(objfiles, quoteShell(
+      objfiles.add(' ')
+      objfiles.add(quoteShell(
           addFileExt(objFile, CC[conf.cCompiler].objExt)))
 
     if conf.hcrOn: # lets assume that optCompileOnly isn't on
@@ -899,14 +879,14 @@ proc callCCompiler*(conf: ConfigRef) =
         # don't relink each of the many binaries (one for each source file) if the nim code is
         # cached because that would take too much time for small changes - the only downside to
         # this is that if an external-to-link file changes the final target wouldn't be relinked
-        if x.flags.contains(CfileFlag.Cached): continue
+        if CfileFlag.Cached in x.flags: continue
         # we pass each object file as if it is the project file - a .dll will be created for each such
         # object file in the nimcache directory, and only in the case of the main project file will
         # there be probably an executable (if the project is such) which will be copied out of the nimcache
         let objFile = conf.getObjFilePath(x)
         let buildDll = idx != mainFileIdx
         let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll)
-        add(cmds, getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll))
+        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll))
         # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache
         # for more info check the comment inside of getLinkCmd() where the /PDB:<filename> MSVC flag is used
         if isVSCompatible(conf):
@@ -917,7 +897,7 @@ proc callCCompiler*(conf: ConfigRef) =
       prettyCmds = map(prettyCmds, proc (curr: string): string = return curr.replace("CC", "Link"))
       execCmdsInParallel(conf, cmds, prettyCb)
       # only if not cached - copy the resulting main file from the nimcache folder to its originally intended destination
-      if not conf.toCompile[mainFileIdx].flags.contains(CfileFlag.Cached):
+      if CfileFlag.Cached notin conf.toCompile[mainFileIdx].flags:
         let mainObjFile = getObjFilePath(conf, conf.toCompile[mainFileIdx])
         var src = conf.hcrLinkTargetName(mainObjFile, true)
         var dst = conf.prepareToWriteOutput
@@ -925,11 +905,12 @@ proc callCCompiler*(conf: ConfigRef) =
     else:
       for x in conf.toCompile:
         let objFile = if noAbsolutePaths(conf): x.obj.extractFilename else: x.obj.string
-        add(objfiles, ' ')
-        add(objfiles, quoteShell(objFile))
+        objfiles.add(' ')
+        objfiles.add(quoteShell(objFile))
       let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput
                        else: AbsoluteFile(conf.projectName)
       linkCmd = getLinkCmd(conf, mainOutput, objfiles)
+      extraCmds = getExtraCmds(conf, mainOutput)
       if optCompileOnly notin conf.globalOptions:
         const MaxCmdLen = when defined(windows): 8_000 else: 32_000
         if linkCmd.len > MaxCmdLen:
@@ -939,11 +920,13 @@ proc callCCompiler*(conf: ConfigRef) =
           linkViaResponseFile(conf, linkCmd)
         else:
           execLinkCmd(conf, linkCmd)
+        for cmd in extraCmds:
+          execExternalProgram(conf, cmd, hintExecuting)
   else:
     linkCmd = ""
   if optGenScript in conf.globalOptions:
-    add(script, linkCmd)
-    add(script, "\n")
+    script.add(linkCmd)
+    script.add("\n")
     generateScript(conf, script)
 
 #from json import escapeJson
@@ -962,17 +945,16 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
       f.write escapeJson(x)
 
   proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) =
-    var i = 0
-    for it in clist:
+    var comma = false
+    for i, it in clist:
       if CfileFlag.Cached in it.flags: continue
       let compileCmd = getCompileCFileCmd(conf, it)
-      if i > 0: lit ",\L"
+      if comma: lit ",\L" else: comma = true
       lit "["
       str it.cname.string
       lit ", "
       str compileCmd
       lit "]"
-      inc i
 
   proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList;
                  llist: seq[string]) =
@@ -981,16 +963,16 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
       let objfile = if noAbsolutePaths(conf): it.extractFilename
                     else: it
       let objstr = addFileExt(objfile, CC[conf.cCompiler].objExt)
-      add(objfiles, ' ')
-      add(objfiles, objstr)
+      objfiles.add(' ')
+      objfiles.add(objstr)
       if pastStart: lit ",\L"
       str objstr
       pastStart = true
 
     for it in clist:
       let objstr = quoteShell(it.obj)
-      add(objfiles, ' ')
-      add(objfiles, objstr)
+      objfiles.add(' ')
+      objfiles.add(objstr)
       if pastStart: lit ",\L"
       str objstr
       pastStart = true
@@ -1013,10 +995,16 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
   var buf = newStringOfCap(50)
 
   let jsonFile = conf.getNimcacheDir / RelativeFile(conf.projectName & ".json")
+  conf.jsonBuildFile = jsonFile
+  let output = conf.absOutFile
 
   var f: File
   if open(f, jsonFile.string, fmWrite):
-    lit "{\"compile\":[\L"
+    lit "{\L"
+    lit "\"outputFile\": "
+    str $output
+
+    lit ",\L\"compile\":[\L"
     cfiles(conf, f, buf, conf.toCompile, false)
     lit "],\L\"link\":[\L"
     var objfiles = ""
@@ -1024,7 +1012,13 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
     linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink)
 
     lit "],\L\"linkcmd\": "
-    str getLinkCmd(conf, conf.absOutFile, objfiles)
+    str getLinkCmd(conf, output, objfiles)
+
+    lit ",\L\"extraCmds\": "
+    lit $(%* getExtraCmds(conf, conf.absOutFile))
+
+    lit ",\L\"stdinInput\": "
+    lit $(%* conf.projectIsStdin)
 
     if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"):
       lit ",\L\"cmdline\": "
@@ -1052,6 +1046,13 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; projectfile: Absol
       return true
     if hashNimExe() != data["nimexe"].getStr:
       return true
+    if not data.hasKey("stdinInput"): return true
+    let stdinInput = data["stdinInput"].getBool
+    if conf.projectIsStdin or stdinInput:
+      # could optimize by returning false if stdin input was the same,
+      # but I'm not sure how to get full stding input
+      return true
+
     let depfilesPairs = data["depfiles"]
     doAssert depfilesPairs.kind == JArray
     for p in depfilesPairs:
@@ -1071,50 +1072,68 @@ proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) =
   let jsonFile = toGeneratedFile(conf, projectfile, "json")
   try:
     let data = json.parseFile(jsonFile.string)
+
+    let output = data["outputFile"].getStr
+    createDir output.parentDir
+    let outputCurrent = $conf.absOutFile
+    if output != outputCurrent:
+      # previously, any specified output file would be silently ignored;
+      # simply copying won't work in some cases, for example with `extraCmds`,
+      # so we just make it an error, user should use same command for jsonscript
+      # as was used with --compileOnly.
+      globalError(conf, gCmdLineInfo, "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % [outputCurrent, output, jsonFile.string])
+
     let toCompile = data["compile"]
     doAssert toCompile.kind == JArray
-    var cmds: TStringSeq = @[]
-    var prettyCmds: TStringSeq = @[]
+    var cmds: TStringSeq
+    var prettyCmds: TStringSeq
+    let prettyCb = proc (idx: int) =
+      if prettyCmds[idx].len > 0: echo prettyCmds[idx]
+
     for c in toCompile:
       doAssert c.kind == JArray
       doAssert c.len >= 2
 
-      add(cmds, c[1].getStr)
-      let (_, name, _) = splitFile(c[0].getStr)
-      add(prettyCmds, "CC: " & demanglePackageName(name))
+      cmds.add(c[1].getStr)
+      prettyCmds.add displayProgressCC(conf, c[0].getStr, c[1].getStr)
 
-    let prettyCb = proc (idx: int) =
-      when declared(echo):
-        echo prettyCmds[idx]
     execCmdsInParallel(conf, cmds, prettyCb)
 
     let linkCmd = data["linkcmd"]
     doAssert linkCmd.kind == JString
     execLinkCmd(conf, linkCmd.getStr)
+    if data.hasKey("extraCmds"):
+      let extraCmds = data["extraCmds"]
+      doAssert extraCmds.kind == JArray
+      for cmd in extraCmds:
+        doAssert cmd.kind == JString, $cmd.kind
+        let cmd2 = cmd.getStr
+        execExternalProgram(conf, cmd2, hintExecuting)
+
   except:
-    when declared(echo):
-      echo getCurrentException().getStackTrace()
-    quit "error evaluating JSON file: " & jsonFile.string
+    let e = getCurrentException()
+    quit "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() &
+         "error evaluating JSON file: " & jsonFile.string
 
 proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope =
   for it in list:
-    addf(result, "--file:r\"$1\"$N", [rope(it.cname.string)])
+    result.addf("--file:r\"$1\"$N", [rope(it.cname.string)])
 
 proc writeMapping*(conf: ConfigRef; symbolMapping: Rope) =
   if optGenMapping notin conf.globalOptions: return
   var code = rope("[C_Files]\n")
-  add(code, genMappingFiles(conf, conf.toCompile))
-  add(code, "\n[C_Compiler]\nFlags=")
-  add(code, strutils.escape(getCompileOptions(conf)))
+  code.add(genMappingFiles(conf, conf.toCompile))
+  code.add("\n[C_Compiler]\nFlags=")
+  code.add(strutils.escape(getCompileOptions(conf)))
 
-  add(code, "\n[Linker]\nFlags=")
-  add(code, strutils.escape(getLinkOptions(conf) & " " &
+  code.add("\n[Linker]\nFlags=")
+  code.add(strutils.escape(getLinkOptions(conf) & " " &
                             getConfigVar(conf, conf.cCompiler, ".options.linker")))
 
-  add(code, "\n[Environment]\nlibpath=")
-  add(code, strutils.escape(conf.libpath.string))
+  code.add("\n[Environment]\nlibpath=")
+  code.add(strutils.escape(conf.libpath.string))
 
-  addf(code, "\n[Symbols]$n$1", [symbolMapping])
+  code.addf("\n[Symbols]$n$1", [symbolMapping])
   let filename = conf.projectPath / RelativeFile"mapping.txt"
   if not writeRope(code, filename):
     rawMessage(conf, errGenerated, "could not write to file: " & filename.string)

compiler/filter_tmpl.nim

@@ -75,7 +75,7 @@ proc parseLine(p: var TTmplParser) =
     let d = j
     var keyw = ""
     while j < len and p.x[j] in PatternChars:
-      add(keyw, p.x[j])
+      keyw.add(p.x[j])
       inc(j)
 
     scanPar(p, j)
@@ -199,8 +199,8 @@ proc parseLine(p: var TTmplParser) =
           inc(j)
     llStreamWrite(p.outp, "\\n\"")
 
-proc filterTmpl*(stdin: PLLStream, filename: AbsoluteFile,
-                 call: PNode; conf: ConfigRef): PLLStream =
+proc filterTmpl*(conf: ConfigRef, stdin: PLLStream, filename: AbsoluteFile,
+                 call: PNode): PLLStream =
   var p: TTmplParser
   p.config = conf
   p.info = newLineInfo(conf, filename, 0, 0)
@@ -214,9 +214,9 @@ proc filterTmpl*(stdin: PLLStream, filename: AbsoluteFile,
   p.x = newStringOfCap(120)
   # do not process the first line which contains the directive:
   if llStreamReadLine(p.inp, p.x):
-    p.info.line = p.info.line + 1'u16
+    inc p.info.line
   while llStreamReadLine(p.inp, p.x):
-    p.info.line = p.info.line + 1'u16
+    inc p.info.line
     parseLine(p)
   newLine(p)
   result = p.outp

compiler/filters.nim

@@ -20,13 +20,13 @@ proc invalidPragma(conf: ConfigRef; n: PNode) =
 proc getArg(conf: ConfigRef; n: PNode, name: string, pos: int): PNode =
   result = nil
   if n.kind in {nkEmpty..nkNilLit}: return
-  for i in 1 ..< len(n):
-    if n.sons[i].kind == nkExprEqExpr:
-      if n.sons[i].sons[0].kind != nkIdent: invalidPragma(conf, n)
-      if cmpIgnoreStyle(n.sons[i].sons[0].ident.s, name) == 0:
-        return n.sons[i].sons[1]
+  for i in 1..<n.len:
+    if n[i].kind == nkExprEqExpr:
+      if n[i][0].kind != nkIdent: invalidPragma(conf, n)
+      if cmpIgnoreStyle(n[i][0].ident.s, name) == 0:
+        return n[i][1]
     elif i == pos:
-      return n.sons[i]
+      return n[i]
 
 proc charArg*(conf: ConfigRef; n: PNode, name: string, pos: int, default: char): char =
   var x = getArg(conf, n, name, pos)
@@ -55,7 +55,7 @@ proc filterStrip*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, cal
   var line = newStringOfCap(80)
   while llStreamReadLine(stdin, line):
     var stripped = strip(line, leading, trailing)
-    if len(pattern) == 0 or startsWith(stripped, pattern):
+    if pattern.len == 0 or startsWith(stripped, pattern):
       llStreamWriteln(result, stripped)
     else:
       llStreamWriteln(result, line)
@@ -63,7 +63,7 @@ proc filterStrip*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, cal
 
 proc filterReplace*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, call: PNode): PLLStream =
   var sub = strArg(conf, call, "sub", 1, "")
-  if len(sub) == 0: invalidPragma(conf, call)
+  if sub.len == 0: invalidPragma(conf, call)
   var by = strArg(conf, call, "by", 2, "")
   result = llStreamOpen("")
   var line = newStringOfCap(80)

compiler/forloops.nim

@@ -1,89 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements for loop detection for better C code generation.
-
-import ast, astalgo
-
-const
-  someCmp = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
-    mEqUntracedRef, mLeI, mLeF64, mLeU, mLeU64, mLeEnum,
-    mLeCh, mLeB, mLePtr, mLtI, mLtF64, mLtU, mLtU64, mLtEnum,
-    mLtCh, mLtB, mLtPtr}
-
-proc isCounter(s: PSym): bool {.inline.} =
-  s.kind in {skResult, skVar, skLet, skTemp} and
-  {sfGlobal, sfAddrTaken} * s.flags == {}
-
-proc isCall(n: PNode): bool {.inline.} =
-  n.kind in nkCallKinds and n[0].kind == nkSym
-
-proc fromSystem(op: PSym): bool = sfSystemModule in getModule(op).flags
-
-proc getCounter(lastStmt: PNode): PSym =
-  if lastStmt.isCall:
-    let op = lastStmt.sym
-    if op.magic in {mDec, mInc} or
-        ((op.name.s == "+=" or op.name.s == "-=") and op.fromSystem):
-      if op[1].kind == nkSym and isCounter(op[1].sym):
-        result = op[1].sym
-
-proc counterInTree(n, loop: PNode; counter: PSym): bool =
-  # prune the search tree: within the loop the counter may be used:
-  if n == loop: return
-  case n.kind
-  of nkSym:
-    if n.sym == counter: return true
-  of nkVarSection, nkLetSection:
-    # definitions are fine!
-    for it in n:
-      if counterInTree(it.lastSon): return true
-  else:
-    for i in 0 ..< safeLen(n):
-      if counterInTree(n[i], loop, counter): return true
-
-proc copyExcept(n: PNode, x, dest: PNode) =
-  if x == n: return
-  if n.kind in {nkStmtList, nkStmtListExpr}:
-    for i in 0 ..< n.len: copyExcept(n[i], x, dest)
-  else:
-    dest.add n
-
-type
-  ForLoop* = object
-    counter*: PSym
-    init*, cond*, increment*, body*: PNode
-
-proc extractForLoop*(loop, fullTree: PNode): ForLoop =
-  ## returns 'counter == nil' if the while loop 'n' is not a for loop:
-  assert loop.kind == nkWhileStmt
-  let cond == loop[0]
-
-  if not cond.isCall: return
-  if cond[0].sym.magic notin someCmp: return
-
-  var lastStmt = loop[1]
-  while lastStmt.kind in {nkStmtList, nkStmtListExpr}:
-    lastStmt = lastStmt.lastSon
-
-  let counter = getCounter(lastStmt)
-  if counter.isNil or counter.ast.isNil: return
-
-  template `=~`(a, b): expr = a.kind == nkSym and a.sym == b
-
-  if cond[1] =~ counter or cond[2] =~ counter:
-    # ok, now check 'counter' is not used *after* the loop
-    if counterInTree(fullTree, loop, counter): return
-    # ok, success, fill in the fields:
-    result.counter = counter
-    result.init = counter.ast
-    result.cond = cond
-    result.increment = lastStmt
-    result.body = newNodeI(nkStmtList, loop[1].info)
-    copyExcept(loop[1], lastStmt, result.body)

compiler/guards.nim

@@ -14,18 +14,17 @@ import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents,
 
 const
   someEq = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
-    mEqUntracedRef, mEqStr, mEqSet, mEqCString}
+    mEqStr, mEqSet, mEqCString}
 
   # set excluded here as the semantics are vastly different:
-  someLe = {mLeI, mLeF64, mLeU, mLeU64, mLeEnum,
+  someLe = {mLeI, mLeF64, mLeU, mLeEnum,
             mLeCh, mLeB, mLePtr, mLeStr}
-  someLt = {mLtI, mLtF64, mLtU, mLtU64, mLtEnum,
+  someLt = {mLtI, mLtF64, mLtU, mLtEnum,
             mLtCh, mLtB, mLtPtr, mLtStr}
 
-  someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq,
-             mXLenStr, mXLenSeq}
+  someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq}
 
-  someIn = {mInRange, mInSet}
+  someIn = {mInSet}
 
   someHigh = {mHigh}
   # we don't list unsigned here because wrap around semantics suck for
@@ -47,7 +46,7 @@ proc isLet(n: PNode): bool =
     if n.sym.kind in {skLet, skTemp, skForVar}:
       result = true
     elif n.sym.kind == skParam and skipTypes(n.sym.typ,
-                                             abstractInst).kind != tyVar:
+                                             abstractInst).kind notin {tyVar}:
       result = true
 
 proc isVar(n: PNode): bool =
@@ -65,16 +64,16 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
   while true:
     case n.kind
     of nkDotExpr, nkCheckedFieldExpr, nkObjUpConv, nkObjDownConv:
-      n = n.sons[0]
+      n = n[0]
     of nkDerefExpr, nkHiddenDeref:
-      n = n.sons[0]
+      n = n[0]
       inc derefs
     of nkBracketExpr:
-      if isConstExpr(n.sons[1]) or isLet(n.sons[1]) or isConstExpr(n.sons[1].skipConv):
-        n = n.sons[0]
+      if isConstExpr(n[1]) or isLet(n[1]) or isConstExpr(n[1].skipConv):
+        n = n[0]
       else: return
     of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-      n = n.sons[1]
+      n = n[1]
     else:
       break
   result = n.isLet and derefs <= ord(isApprox)
@@ -85,8 +84,8 @@ proc interestingCaseExpr*(m: PNode): bool = isLetLocation(m, true)
 
 type
   Operators* = object
-    opNot, opContains, opLe, opLt, opAnd, opOr, opIsNil, opEq: PSym
-    opAdd, opSub, opMul, opDiv, opLen: PSym
+    opNot*, opContains*, opLe*, opLt*, opAnd*, opOr*, opIsNil*, opEq*: PSym
+    opAdd*, opSub*, opMul*, opDiv*, opLen*: PSym
 
 proc initOperators*(g: ModuleGraph): Operators =
   result.opLe = createMagic(g, "<=", mLeI)
@@ -105,34 +104,34 @@ proc initOperators*(g: ModuleGraph): Operators =
 
 proc swapArgs(fact: PNode, newOp: PSym): PNode =
   result = newNodeI(nkCall, fact.info, 3)
-  result.sons[0] = newSymNode(newOp)
-  result.sons[1] = fact.sons[2]
-  result.sons[2] = fact.sons[1]
+  result[0] = newSymNode(newOp)
+  result[1] = fact[2]
+  result[2] = fact[1]
 
 proc neg(n: PNode; o: Operators): PNode =
   if n == nil: return nil
   case n.getMagic
   of mNot:
-    result = n.sons[1]
+    result = n[1]
   of someLt:
     # not (a < b)  ==  a >= b  ==  b <= a
     result = swapArgs(n, o.opLe)
   of someLe:
     result = swapArgs(n, o.opLt)
   of mInSet:
-    if n.sons[1].kind != nkCurly: return nil
-    let t = n.sons[2].typ.skipTypes(abstractInst)
+    if n[1].kind != nkCurly: return nil
+    let t = n[2].typ.skipTypes(abstractInst)
     result = newNodeI(nkCall, n.info, 3)
-    result.sons[0] = n.sons[0]
-    result.sons[2] = n.sons[2]
+    result[0] = n[0]
+    result[2] = n[2]
     if t.kind == tyEnum:
-      var s = newNodeIT(nkCurly, n.info, n.sons[1].typ)
+      var s = newNodeIT(nkCurly, n.info, n[1].typ)
       for e in t.n:
         let eAsNode = newIntNode(nkIntLit, e.sym.position)
-        if not inSet(n.sons[1], eAsNode): s.add eAsNode
-      result.sons[1] = s
+        if not inSet(n[1], eAsNode): s.add eAsNode
+      result[1] = s
     #elif t.kind notin {tyString, tySequence} and lengthOrd(t) < 1000:
-    #  result.sons[1] = complement(n.sons[1])
+    #  result[1] = complement(n[1])
     else:
       # not ({2, 3, 4}.contains(x))   x != 2 and x != 3 and x != 4
       # XXX todo
@@ -140,13 +139,13 @@ proc neg(n: PNode; o: Operators): PNode =
   of mOr:
     # not (a or b) --> not a and not b
     let
-      a = n.sons[1].neg(o)
-      b = n.sons[2].neg(o)
+      a = n[1].neg(o)
+      b = n[2].neg(o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
     elif a != nil:
       result = a
     elif b != nil:
@@ -154,19 +153,19 @@ proc neg(n: PNode; o: Operators): PNode =
   else:
     # leave  not (a == 4)  as it is
     result = newNodeI(nkCall, n.info, 2)
-    result.sons[0] = newSymNode(o.opNot)
-    result.sons[1] = n
+    result[0] = newSymNode(o.opNot)
+    result[1] = n
 
-proc buildCall(op: PSym; a: PNode): PNode =
+proc buildCall*(op: PSym; a: PNode): PNode =
   result = newNodeI(nkCall, a.info, 2)
-  result.sons[0] = newSymNode(op)
-  result.sons[1] = a
+  result[0] = newSymNode(op)
+  result[1] = a
 
-proc buildCall(op: PSym; a, b: PNode): PNode =
+proc buildCall*(op: PSym; a, b: PNode): PNode =
   result = newNodeI(nkInfix, a.info, 3)
-  result.sons[0] = newSymNode(op)
-  result.sons[1] = a
-  result.sons[2] = b
+  result[0] = newSymNode(op)
+  result[1] = a
+  result[2] = b
 
 proc `|+|`(a, b: PNode): PNode =
   result = copyNode(a)
@@ -250,35 +249,35 @@ proc pred(n: PNode): PNode =
   else:
     result = n
 
+proc buildLe*(o: Operators; a, b: PNode): PNode =
+  result = o.opLe.buildCall(a, b)
+
 proc canon*(n: PNode; o: Operators): PNode =
-  # XXX for now only the new code in 'semparallel' uses this
   if n.safeLen >= 1:
     result = shallowCopy(n)
-    for i in 0 ..< n.len:
-      result.sons[i] = canon(n.sons[i], o)
+    for i in 0..<n.len:
+      result[i] = canon(n[i], o)
   elif n.kind == nkSym and n.sym.kind == skLet and
       n.sym.astdef.getMagic in (someEq + someAdd + someMul + someMin +
-      someMax + someHigh + {mUnaryLt} + someSub + someLen + someDiv):
+      someMax + someHigh + someSub + someLen + someDiv):
     result = n.sym.astdef.copyTree
   else:
     result = n
   case result.getMagic
   of someEq, someAdd, someMul, someMin, someMax:
     # these are symmetric; put value as last:
-    if result.sons[1].isValue and not result.sons[2].isValue:
-      result = swapArgs(result, result.sons[0].sym)
+    if result[1].isValue and not result[2].isValue:
+      result = swapArgs(result, result[0].sym)
       # (4 + foo) + 2 --> (foo + 4) + 2
   of someHigh:
     # high == len+(-1)
     result = o.opAdd.buildCall(o.opLen.buildCall(result[1]), minusOne())
-  of mUnaryLt:
-    result = buildCall(o.opAdd, result[1], minusOne())
   of someSub:
     # x - 4  -->  x + (-4)
     result = negate(result[1], result[2], result, o)
   of someLen:
-    result.sons[0] = o.opLen.newSymNode
-  of someLt:
+    result[0] = o.opLen.newSymNode
+  of someLt - {mLtF64}:
     # x < y  same as x <= y-1:
     let y = n[2].canon(o)
     let p = pred(y)
@@ -318,16 +317,16 @@ proc canon*(n: PNode; o: Operators): PNode =
         if plus != nil and not isLetLocation(x, true):
           result = buildCall(result[0].sym, plus, y[1])
       else: discard
-    elif x.isValue and y.getMagic in someAdd and y[2].isValue:
+    elif x.isValue and y.getMagic in someAdd and y[2].kind == x.kind:
       # 0 <= a.len + 3
       # -3 <= a.len
-      result.sons[1] = x |-| y[2]
-      result.sons[2] = y[1]
-    elif x.isValue and y.getMagic in someSub and y[2].isValue:
+      result[1] = x |-| y[2]
+      result[2] = y[1]
+    elif x.isValue and y.getMagic in someSub and y[2].kind == x.kind:
       # 0 <= a.len - 3
       # 3 <= a.len
-      result.sons[1] = x |+| y[2]
-      result.sons[2] = y[1]
+      result[1] = x |+| y[2]
+      result[2] = y[1]
   else: discard
 
 proc buildAdd*(a: PNode; b: BiggestInt; o: Operators): PNode =
@@ -336,41 +335,43 @@ proc buildAdd*(a: PNode; b: BiggestInt; o: Operators): PNode =
 proc usefulFact(n: PNode; o: Operators): PNode =
   case n.getMagic
   of someEq:
-    if skipConv(n.sons[2]).kind == nkNilLit and (
-        isLetLocation(n.sons[1], false) or isVar(n.sons[1])):
-      result = o.opIsNil.buildCall(n.sons[1])
+    if skipConv(n[2]).kind == nkNilLit and (
+        isLetLocation(n[1], false) or isVar(n[1])):
+      result = o.opIsNil.buildCall(n[1])
     else:
-      if isLetLocation(n.sons[1], true) or isLetLocation(n.sons[2], true):
+      if isLetLocation(n[1], true) or isLetLocation(n[2], true):
         # XXX algebraic simplifications!  'i-1 < a.len' --> 'i < a.len+1'
         result = n
+      elif n[1].getMagic in someLen or n[2].getMagic in someLen:
+        result = n
   of someLe+someLt:
-    if isLetLocation(n.sons[1], true) or isLetLocation(n.sons[2], true):
+    if isLetLocation(n[1], true) or isLetLocation(n[2], true):
       # XXX algebraic simplifications!  'i-1 < a.len' --> 'i < a.len+1'
       result = n
     elif n[1].getMagic in someLen or n[2].getMagic in someLen:
       # XXX Rethink this whole idea of 'usefulFact' for semparallel
       result = n
   of mIsNil:
-    if isLetLocation(n.sons[1], false) or isVar(n.sons[1]):
+    if isLetLocation(n[1], false) or isVar(n[1]):
       result = n
   of someIn:
-    if isLetLocation(n.sons[1], true):
+    if isLetLocation(n[1], true):
       result = n
   of mAnd:
     let
-      a = usefulFact(n.sons[1], o)
-      b = usefulFact(n.sons[2], o)
+      a = usefulFact(n[1], o)
+      b = usefulFact(n[2], o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
     elif a != nil:
       result = a
     elif b != nil:
       result = b
   of mNot:
-    let a = usefulFact(n.sons[1], o)
+    let a = usefulFact(n[1], o)
     if a != nil:
       result = a.neg(o)
   of mOr:
@@ -381,13 +382,13 @@ proc usefulFact(n: PNode; o: Operators): PNode =
     #  (x == 3) or (y == 2)  ---> not ( not (x==3) and not (y == 2))
     #  not (x != 3 and y != 2)
     let
-      a = usefulFact(n.sons[1], o).neg(o)
-      b = usefulFact(n.sons[2], o).neg(o)
+      a = usefulFact(n[1], o).neg(o)
+      b = usefulFact(n[2], o).neg(o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
       result = result.neg(o)
   elif n.kind == nkSym and n.sym.kind == skLet:
     # consider:
@@ -404,10 +405,20 @@ type
   TModel* = object
     s*: seq[PNode] # the "knowledge base"
     o*: Operators
+    beSmart*: bool
 
 proc addFact*(m: var TModel, nn: PNode) =
   let n = usefulFact(nn, m.o)
-  if n != nil: m.s.add n
+  if n != nil:
+    if not m.beSmart:
+      m.s.add n
+    else:
+      let c = canon(n, m.o)
+      if c.getMagic == mAnd:
+        addFact(m, c[1])
+        addFact(m, c[2])
+      else:
+        m.s.add c
 
 proc addFactNeg*(m: var TModel, n: PNode) =
   let n = n.neg(m.o)
@@ -442,18 +453,18 @@ proc sameTree*(a, b: PNode): bool =
     of nkType: result = a.typ == b.typ
     of nkEmpty, nkNilLit: result = true
     else:
-      if len(a) == len(b):
-        for i in 0 ..< len(a):
-          if not sameTree(a.sons[i], b.sons[i]): return
+      if a.len == b.len:
+        for i in 0..<a.len:
+          if not sameTree(a[i], b[i]): return
         result = true
 
 proc hasSubTree(n, x: PNode): bool =
   if n.sameTree(x): result = true
   else:
-    for i in 0..safeLen(n)-1:
-      if hasSubTree(n.sons[i], x): return true
+    for i in 0..n.safeLen-1:
+      if hasSubTree(n[i], x): return true
 
-proc invalidateFacts*(m: var TModel, n: PNode) =
+proc invalidateFacts*(s: var seq[PNode], n: PNode) =
   # We are able to guard local vars (as opposed to 'let' variables)!
   # 'while p != nil: f(p); p = p.next'
   # This is actually quite easy to do:
@@ -471,29 +482,32 @@ proc invalidateFacts*(m: var TModel, n: PNode) =
   # The same mechanism could be used for more complex data stored on the heap;
   # procs that 'write: []' cannot invalidate 'n.kind' for instance. In fact, we
   # could CSE these expressions then and help C's optimizer.
-  for i in 0..high(m.s):
-    if m.s[i] != nil and m.s[i].hasSubTree(n): m.s[i] = nil
+  for i in 0..high(s):
+    if s[i] != nil and s[i].hasSubTree(n): s[i] = nil
+
+proc invalidateFacts*(m: var TModel, n: PNode) =
+  invalidateFacts(m.s, n)
 
 proc valuesUnequal(a, b: PNode): bool =
   if a.isValue and b.isValue:
     result = not sameValue(a, b)
 
 proc impliesEq(fact, eq: PNode): TImplication =
-  let (loc, val) = if isLocation(eq.sons[1]): (1, 2) else: (2, 1)
+  let (loc, val) = if isLocation(eq[1]): (1, 2) else: (2, 1)
 
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], eq.sons[loc]):
+    if sameTree(fact[1], eq[loc]):
       # this is not correct; consider:  a == b;  a == 1 --> unknown!
-      if sameTree(fact.sons[2], eq.sons[val]): result = impYes
-      elif valuesUnequal(fact.sons[2], eq.sons[val]): result = impNo
-    elif sameTree(fact.sons[2], eq.sons[loc]):
-      if sameTree(fact.sons[1], eq.sons[val]): result = impYes
-      elif valuesUnequal(fact.sons[1], eq.sons[val]): result = impNo
+      if sameTree(fact[2], eq[val]): result = impYes
+      elif valuesUnequal(fact[2], eq[val]): result = impNo
+    elif sameTree(fact[2], eq[loc]):
+      if sameTree(fact[1], eq[val]): result = impYes
+      elif valuesUnequal(fact[1], eq[val]): result = impNo
   of mInSet:
     # remember: mInSet is 'contains' so the set comes first!
-    if sameTree(fact.sons[2], eq.sons[loc]) and isValue(eq.sons[val]):
-      if inSet(fact.sons[1], eq.sons[val]): result = impYes
+    if sameTree(fact[2], eq[loc]) and isValue(eq[val]):
+      if inSet(fact[1], eq[val]): result = impYes
       else: result = impNo
   of mNot, mOr, mAnd: assert(false, "impliesEq")
   else: discard
@@ -536,28 +550,28 @@ proc compareSets(a, b: PNode): TImplication =
   elif intersectSets(nil, a, b).len == 0: result = impNo
 
 proc impliesIn(fact, loc, aSet: PNode): TImplication =
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], loc):
-      if inSet(aSet, fact.sons[2]): result = impYes
+    if sameTree(fact[1], loc):
+      if inSet(aSet, fact[2]): result = impYes
       else: result = impNo
-    elif sameTree(fact.sons[2], loc):
-      if inSet(aSet, fact.sons[1]): result = impYes
+    elif sameTree(fact[2], loc):
+      if inSet(aSet, fact[1]): result = impYes
       else: result = impNo
   of mInSet:
-    if sameTree(fact.sons[2], loc):
-      result = compareSets(fact.sons[1], aSet)
+    if sameTree(fact[2], loc):
+      result = compareSets(fact[1], aSet)
   of someLe:
-    if sameTree(fact.sons[1], loc):
-      result = leImpliesIn(fact.sons[1], fact.sons[2], aSet)
-    elif sameTree(fact.sons[2], loc):
-      result = geImpliesIn(fact.sons[2], fact.sons[1], aSet)
+    if sameTree(fact[1], loc):
+      result = leImpliesIn(fact[1], fact[2], aSet)
+    elif sameTree(fact[2], loc):
+      result = geImpliesIn(fact[2], fact[1], aSet)
   of someLt:
-    if sameTree(fact.sons[1], loc):
-      result = leImpliesIn(fact.sons[1], fact.sons[2].pred, aSet)
-    elif sameTree(fact.sons[2], loc):
+    if sameTree(fact[1], loc):
+      result = leImpliesIn(fact[1], fact[2].pred, aSet)
+    elif sameTree(fact[2], loc):
       # 4 < x  -->  3 <= x
-      result = geImpliesIn(fact.sons[2], fact.sons[1].pred, aSet)
+      result = geImpliesIn(fact[2], fact[1].pred, aSet)
   of mNot, mOr, mAnd: assert(false, "impliesIn")
   else: discard
 
@@ -567,90 +581,93 @@ proc valueIsNil(n: PNode): TImplication =
   else: impUnknown
 
 proc impliesIsNil(fact, eq: PNode): TImplication =
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of mIsNil:
-    if sameTree(fact.sons[1], eq.sons[1]):
+    if sameTree(fact[1], eq[1]):
       result = impYes
   of someEq:
-    if sameTree(fact.sons[1], eq.sons[1]):
-      result = valueIsNil(fact.sons[2].skipConv)
-    elif sameTree(fact.sons[2], eq.sons[1]):
-      result = valueIsNil(fact.sons[1].skipConv)
+    if sameTree(fact[1], eq[1]):
+      result = valueIsNil(fact[2].skipConv)
+    elif sameTree(fact[2], eq[1]):
+      result = valueIsNil(fact[1].skipConv)
   of mNot, mOr, mAnd: assert(false, "impliesIsNil")
   else: discard
 
 proc impliesGe(fact, x, c: PNode): TImplication =
   assert isLocation(x)
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x = 4;  question x >= 56? --> true iff 4 >= 56
-        if leValue(c, fact.sons[2]): result = impYes
+        if leValue(c, fact[2]): result = impYes
         else: result = impNo
-    elif sameTree(fact.sons[2], x):
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impYes
+    elif sameTree(fact[2], x):
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impYes
         else: result = impNo
   of someLt:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x < 4;  question N <= x? --> false iff N <= 4
-        if leValue(fact.sons[2], c): result = impNo
+        if leValue(fact[2], c): result = impNo
         # fact:  x < 4;  question 2 <= x? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 < x; question: N-1 < x ?  --> true iff N-1 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c.pred, fact.sons[1]): result = impYes
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c.pred, fact[1]): result = impYes
   of someLe:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x <= 4;  question x >= 56? --> false iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impNo
+        if leValue(fact[2], c): result = impNo
         # fact:  x <= 4;  question x >= 2? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 <= x; question: x >= 2 ?  --> true iff 2 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impYes
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impYes
   of mNot, mOr, mAnd: assert(false, "impliesGe")
   else: discard
 
 proc impliesLe(fact, x, c: PNode): TImplication =
   if not isLocation(x):
+    if c.isValue:
+      if leValue(x, x): return impYes
+      else: return impNo
     return impliesGe(fact, c, x)
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x = 4;  question x <= 56? --> true iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impYes
+        if leValue(fact[2], c): result = impYes
         else: result = impNo
-    elif sameTree(fact.sons[2], x):
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(fact.sons[1], c): result = impYes
+    elif sameTree(fact[2], x):
+      if isValue(fact[1]) and isValue(c):
+        if leValue(fact[1], c): result = impYes
         else: result = impNo
   of someLt:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x < 4;  question x <= N? --> true iff N-1 <= 4
-        if leValue(fact.sons[2], c.pred): result = impYes
+        if leValue(fact[2], c.pred): result = impYes
         # fact:  x < 4;  question x <= 2? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 < x; question: x <= 1 ?  --> false iff 1 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impNo
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impNo
 
   of someLe:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x <= 4;  question x <= 56? --> true iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impYes
+        if leValue(fact[2], c): result = impYes
         # fact:  x <= 4;  question x <= 2? --> we don't know
 
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 <= x; question: x <= 2 ?  --> false iff 2 < 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1].pred): result = impNo
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1].pred): result = impNo
 
   of mNot, mOr, mAnd: assert(false, "impliesLe")
   else: discard
@@ -686,32 +703,32 @@ proc factImplies(fact, prop: PNode): TImplication =
 
     #  (not a) -> b  compute as  not (a -> b) ???
     #  == not a or not b == not (a and b)
-    let arg = fact.sons[1]
+    let arg = fact[1]
     case arg.getMagic
     of mIsNil, mEqRef:
       return ~factImplies(arg, prop)
     of mAnd:
       # not (a and b)  means  not a or not b:
       # a or b --> both need to imply 'prop'
-      let a = factImplies(arg.sons[1], prop)
-      let b = factImplies(arg.sons[2], prop)
+      let a = factImplies(arg[1], prop)
+      let b = factImplies(arg[2], prop)
       if a == b: return ~a
       return impUnknown
     else:
       return impUnknown
   of mAnd:
-    result = factImplies(fact.sons[1], prop)
+    result = factImplies(fact[1], prop)
     if result != impUnknown: return result
-    return factImplies(fact.sons[2], prop)
+    return factImplies(fact[2], prop)
   else: discard
 
-  case prop.sons[0].sym.magic
-  of mNot: result = ~fact.factImplies(prop.sons[1])
+  case prop[0].sym.magic
+  of mNot: result = ~fact.factImplies(prop[1])
   of mIsNil: result = impliesIsNil(fact, prop)
   of someEq: result = impliesEq(fact, prop)
-  of someLe: result = impliesLe(fact, prop.sons[1], prop.sons[2])
-  of someLt: result = impliesLt(fact, prop.sons[1], prop.sons[2])
-  of mInSet: result = impliesIn(fact, prop.sons[2], prop.sons[1])
+  of someLe: result = impliesLe(fact, prop[1], prop[2])
+  of someLt: result = impliesLt(fact, prop[1], prop[2])
+  of mInSet: result = impliesIn(fact, prop[2], prop[1])
   else: result = impUnknown
 
 proc doesImply*(facts: TModel, prop: PNode): TImplication =
@@ -802,7 +819,7 @@ proc ple(m: TModel; a, b: PNode): TImplication =
     if lastOrd(nil, a.typ) <= b.intVal: return impYes
   # 3 <= x   iff  low(x) <= 3
   if a.isValue and b.typ != nil and b.typ.isOrdinalType:
-    if firstOrd(nil, b.typ) <= a.intVal: return impYes
+    if a.intVal <= firstOrd(nil, b.typ): return impYes
 
   # x <= x
   if sameTree(a, b): return impYes
@@ -813,7 +830,11 @@ proc ple(m: TModel; a, b: PNode): TImplication =
 
   #   x <= y+c  if 0 <= c and x <= y
   #   x <= y+(-c)  if c <= 0  and y >= x
-  if b.getMagic in someAdd and zero() <=? b[2] and a <=? b[1]: return impYes
+  if b.getMagic in someAdd:
+    if zero() <=? b[2] and a <=? b[1]: return impYes
+    # x <= y-c  if x+c <= y
+    if b[2] <=? zero() and (canon(m.o.opSub.buildCall(a, b[2]), m.o) <=? b[1]):
+      return impYes
 
   #   x+c <= y  if c <= 0 and x <= y
   if a.getMagic in someAdd and a[2] <=? zero() and a[1] <=? b: return impYes
@@ -881,8 +902,8 @@ proc replaceSubTree(n, x, by: PNode): PNode =
     result = by
   elif hasSubTree(n, x):
     result = shallowCopy(n)
-    for i in 0 .. safeLen(n)-1:
-      result.sons[i] = replaceSubTree(n.sons[i], x, by)
+    for i in 0..n.safeLen-1:
+      result[i] = replaceSubTree(n[i], x, by)
   else:
     result = n
 
@@ -902,10 +923,13 @@ proc pleViaModelRec(m: var TModel; a, b: PNode): TImplication =
       # --> true  if  (len-100) <= (len-1)
       let x = fact[1]
       let y = fact[2]
-      if sameTree(x, a) and y.getMagic in someAdd and b.getMagic in someAdd and
-         sameTree(y[1], b[1]):
-        if ple(m, b[2], y[2]) == impYes:
-          return impYes
+      # x <= y.
+      # Question: x <= b? True iff y <= b.
+      if sameTree(x, a):
+        if ple(m, y, b) == impYes: return impYes
+        if y.getMagic in someAdd and b.getMagic in someAdd and sameTree(y[1], b[1]):
+          if ple(m, b[2], y[2]) == impYes:
+            return impYes
 
       # x <= y implies a <= b  if  a <= x and y <= b
       if ple(m, a, x) == impYes:
@@ -964,6 +988,10 @@ proc proveLe*(m: TModel; a, b: PNode): TImplication =
 proc addFactLe*(m: var TModel; a, b: PNode) =
   m.s.add canon(m.o.opLe.buildCall(a, b), m.o)
 
+proc addFactLt*(m: var TModel; a, b: PNode) =
+  let bb = m.o.opAdd.buildCall(b, minusOne())
+  addFactLe(m, a, bb)
+
 proc settype(n: PNode): PType =
   result = newType(tySet, n.typ.owner)
   addSonSkipIntLit(result, n.typ)
@@ -971,37 +999,37 @@ proc settype(n: PNode): PType =
 proc buildOf(it, loc: PNode; o: Operators): PNode =
   var s = newNodeI(nkCurly, it.info, it.len-1)
   s.typ = settype(loc)
-  for i in 0..it.len-2: s.sons[i] = it.sons[i]
+  for i in 0..<it.len-1: s[i] = it[i]
   result = newNodeI(nkCall, it.info, 3)
-  result.sons[0] = newSymNode(o.opContains)
-  result.sons[1] = s
-  result.sons[2] = loc
+  result[0] = newSymNode(o.opContains)
+  result[1] = s
+  result[2] = loc
 
 proc buildElse(n: PNode; o: Operators): PNode =
-  var s = newNodeIT(nkCurly, n.info, settype(n.sons[0]))
-  for i in 1..n.len-2:
-    let branch = n.sons[i]
+  var s = newNodeIT(nkCurly, n.info, settype(n[0]))
+  for i in 1..<n.len-1:
+    let branch = n[i]
     assert branch.kind != nkElse
     if branch.kind == nkOfBranch:
-      for j in 0..branch.len-2:
-        s.add(branch.sons[j])
+      for j in 0..<branch.len-1:
+        s.add(branch[j])
   result = newNodeI(nkCall, n.info, 3)
-  result.sons[0] = newSymNode(o.opContains)
-  result.sons[1] = s
-  result.sons[2] = n.sons[0]
+  result[0] = newSymNode(o.opContains)
+  result[1] = s
+  result[2] = n[0]
 
 proc addDiscriminantFact*(m: var TModel, n: PNode) =
   var fact = newNodeI(nkCall, n.info, 3)
-  fact.sons[0] = newSymNode(m.o.opEq)
-  fact.sons[1] = n.sons[0]
-  fact.sons[2] = n.sons[1]
+  fact[0] = newSymNode(m.o.opEq)
+  fact[1] = n[0]
+  fact[2] = n[1]
   m.s.add fact
 
 proc addAsgnFact*(m: var TModel, key, value: PNode) =
   var fact = newNodeI(nkCall, key.info, 3)
-  fact.sons[0] = newSymNode(m.o.opEq)
-  fact.sons[1] = key
-  fact.sons[2] = value
+  fact[0] = newSymNode(m.o.opEq)
+  fact[1] = key
+  fact[2] = value
   m.s.add fact
 
 proc sameSubexprs*(m: TModel; a, b: PNode): bool =
@@ -1015,34 +1043,34 @@ proc sameSubexprs*(m: TModel; a, b: PNode): bool =
   # However, nil checking requires exactly the same mechanism! But for now
   # we simply use sameTree and live with the unsoundness of the analysis.
   var check = newNodeI(nkCall, a.info, 3)
-  check.sons[0] = newSymNode(m.o.opEq)
-  check.sons[1] = a
-  check.sons[2] = b
+  check[0] = newSymNode(m.o.opEq)
+  check[1] = a
+  check[2] = b
   result = m.doesImply(check) == impYes
 
 proc addCaseBranchFacts*(m: var TModel, n: PNode, i: int) =
-  let branch = n.sons[i]
+  let branch = n[i]
   if branch.kind == nkOfBranch:
-    m.s.add buildOf(branch, n.sons[0], m.o)
+    m.s.add buildOf(branch, n[0], m.o)
   else:
     m.s.add n.buildElse(m.o).neg(m.o)
 
 proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
-  if check.sons[1].kind == nkCurly:
+  if check[1].kind == nkCurly:
     result = copyTree(check)
     if access.kind == nkDotExpr:
       var a = copyTree(access)
-      a.sons[1] = check.sons[2]
-      result.sons[2] = a
+      a[1] = check[2]
+      result[2] = a
       # 'access.kind != nkDotExpr' can happen for object constructors
       # which we don't check yet
   else:
     # it is some 'not'
     assert check.getMagic == mNot
-    result = buildProperFieldCheck(access, check.sons[1], o).neg(o)
+    result = buildProperFieldCheck(access, check[1], o).neg(o)
 
 proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef) =
-  for i in 1..n.len-1:
-    let check = buildProperFieldCheck(n.sons[0], n.sons[i], m.o)
+  for i in 1..<n.len:
+    let check = buildProperFieldCheck(n[0], n[i], m.o)
     if check != nil and m.doesImply(check) != impYes:
-      message(conf, n.info, warnProveField, renderTree(n.sons[0])); break
+      message(conf, n.info, warnProveField, renderTree(n[0])); break

compiler/hlo.nim

@@ -12,14 +12,14 @@
 proc hlo(c: PContext, n: PNode): PNode
 
 proc evalPattern(c: PContext, n, orig: PNode): PNode =
-  internalAssert c.config, n.kind == nkCall and n.sons[0].kind == nkSym
+  internalAssert c.config, n.kind == nkCall and n[0].kind == nkSym
   # we need to ensure that the resulting AST is semchecked. However, it's
   # awful to semcheck before macro invocation, so we don't and treat
   # templates and macros as immediate in this context.
   var rule: string
-  if optHints in c.config.options and hintPattern in c.config.notes:
+  if c.config.hasHint(hintPattern):
     rule = renderTree(n, {renderNoComments})
-  let s = n.sons[0].sym
+  let s = n[0].sym
   case s.kind
   of skMacro:
     result = semMacroExpr(c, n, orig, s)
@@ -27,7 +27,7 @@ proc evalPattern(c: PContext, n, orig: PNode): PNode =
     result = semTemplateExpr(c, n, s, {efFromHlo})
   else:
     result = semDirectOp(c, n, {})
-  if optHints in c.config.options and hintPattern in c.config.notes:
+  if c.config.hasHint(hintPattern):
     message(c.config, orig.info, hintPattern, rule & " --> '" &
       renderTree(result, {renderNoComments}) & "'")
 
@@ -50,7 +50,7 @@ proc applyPatterns(c: PContext, n: PNode): PNode =
         c.patterns[i] = nil
         if x.kind == nkStmtList:
           assert x.len == 3
-          x.sons[1] = evalPattern(c, x.sons[1], result)
+          x[1] = evalPattern(c, x[1], result)
           result = flattenStmts(x)
         else:
           result = evalPattern(c, x, result)
@@ -68,17 +68,17 @@ proc hlo(c: PContext, n: PNode): PNode =
     result = n
   else:
     if n.kind in {nkFastAsgn, nkAsgn, nkIdentDefs, nkVarTuple} and
-        n.sons[0].kind == nkSym and
-        {sfGlobal, sfPure} * n.sons[0].sym.flags == {sfGlobal, sfPure}:
+        n[0].kind == nkSym and
+        {sfGlobal, sfPure} * n[0].sym.flags == {sfGlobal, sfPure}:
       # do not optimize 'var g {.global} = re(...)' again!
       return n
     result = applyPatterns(c, n)
     if result == n:
       # no optimization applied, try subtrees:
-      for i in 0 ..< safeLen(result):
-        let a = result.sons[i]
+      for i in 0..<result.safeLen:
+        let a = result[i]
         let h = hlo(c, a)
-        if h != a: result.sons[i] = h
+        if h != a: result[i] = h
     else:
       # perform type checking, so that the replacement still fits:
       if isEmptyType(n.typ) and isEmptyType(result.typ):

compiler/idents.nim

@@ -87,7 +87,7 @@ proc getIdent*(ic: IdentCache; identifier: cstring, length: int, h: Hash): PIden
   new(result)
   result.h = h
   result.s = newString(length)
-  for i in 0 ..< length: result.s[i] = identifier[i]
+  for i in 0..<length: result.s[i] = identifier[i]
   result.next = ic.buckets[idx]
   ic.buckets[idx] = result
   if id == 0:
@@ -97,11 +97,11 @@ proc getIdent*(ic: IdentCache; identifier: cstring, length: int, h: Hash): PIden
     result.id = id
 
 proc getIdent*(ic: IdentCache; identifier: string): PIdent =
-  result = getIdent(ic, cstring(identifier), len(identifier),
+  result = getIdent(ic, cstring(identifier), identifier.len,
                     hashIgnoreStyle(identifier))
 
 proc getIdent*(ic: IdentCache; identifier: string, h: Hash): PIdent =
-  result = getIdent(ic, cstring(identifier), len(identifier), h)
+  result = getIdent(ic, cstring(identifier), identifier.len, h)
 
 proc newIdentCache*(): IdentCache =
   result = IdentCache()
@@ -110,8 +110,8 @@ proc newIdentCache*(): IdentCache =
   result.idDelegator = result.getIdent":delegator"
   result.emptyIdent = result.getIdent("")
   # initialize the keywords:
-  for s in succ(low(specialWords)) .. high(specialWords):
-    result.getIdent(specialWords[s], hashIgnoreStyle(specialWords[s])).id = ord(s)
+  for s in succ(low(TSpecialWord))..high(TSpecialWord):
+    result.getIdent($s, hashIgnoreStyle($s)).id = ord(s)
 
 proc whichKeyword*(id: PIdent): TSpecialWord =
   if id.id < 0: result = wInvalid

compiler/importer.nim

@@ -16,7 +16,7 @@ import
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
   result = initIntSet()
-  for i in 1 ..< n.len:
+  for i in 1..<n.len:
     let ident = lookups.considerQuotedIdent(c, n[i])
     result.incl(ident.id)
 
@@ -47,8 +47,8 @@ proc rawImportSymbol(c: PContext, s, origin: PSym) =
   if s.kind == skType:
     var etyp = s.typ
     if etyp.kind in {tyBool, tyEnum}:
-      for j in 0 ..< len(etyp.n):
-        var e = etyp.n.sons[j].sym
+      for j in 0..<etyp.n.len:
+        var e = etyp.n[j].sym
         if e.kind != skEnumField:
           internalError(c.config, s.info, "rawImportSymbol")
           # BUGFIX: because of aliases for enums the symbol may already
@@ -125,18 +125,18 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym) =
   of nkExportExceptStmt:
     localError(c.config, n.info, "'export except' not implemented")
   else:
-    for i in 0..safeLen(n)-1:
-      importForwarded(c, n.sons[i], exceptSet, fromMod)
+    for i in 0..n.safeLen-1:
+      importForwarded(c, n[i], exceptSet, fromMod)
 
 proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
   result = realModule
   c.unusedImports.add((realModule, n.info))
   if n.kind != nkImportAs: discard
-  elif n.len != 2 or n.sons[1].kind != nkIdent:
+  elif n.len != 2 or n[1].kind != nkIdent:
     localError(c.config, n.info, "module alias must be an identifier")
-  elif n.sons[1].ident.id != realModule.name.id:
+  elif n[1].ident.id != realModule.name.id:
     # some misguided guy will write 'import abc.foo as foo' ...
-    result = createModuleAlias(realModule, n.sons[1].ident, realModule.info,
+    result = createModuleAlias(realModule, n[1].ident, realModule.info,
                                c.config.options)
 
 proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
@@ -148,7 +148,7 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
     #echo "adding ", toFullPath(f), " at ", L+1
     if recursion >= 0:
       var err = ""
-      for i in recursion ..< L:
+      for i in recursion..<L:
         if i > recursion: err.add "\n"
         err.add toFullPath(c.config, c.graph.importStack[i]) & " imports " &
                 toFullPath(c.config, c.graph.importStack[i+1])
@@ -178,8 +178,8 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
 proc transformImportAs(c: PContext; n: PNode): PNode =
   if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
     result = newNodeI(nkImportAs, n.info)
-    result.add n.sons[1]
-    result.add n.sons[2]
+    result.add n[1]
+    result.add n[2]
   else:
     result = n
 
@@ -187,16 +187,15 @@ proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
   let it = transformImportAs(c, it)
   let m = myImportModule(c, it, importStmtResult)
   if m != nil:
-    var emptySet: IntSet
     # ``addDecl`` needs to be done before ``importAllSymbols``!
     addDecl(c, m, it.info) # add symbol to symbol table of module
-    importAllSymbolsExcept(c, m, emptySet)
+    importAllSymbols(c, m)
     #importForwarded(c, m.ast, emptySet, m)
 
 proc evalImport*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
-  for i in 0 ..< len(n):
-    let it = n.sons[i]
+  for i in 0..<n.len:
+    let it = n[i]
     if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
       let sep = it[0]
       let dir = it[1]
@@ -206,13 +205,13 @@ proc evalImport*(c: PContext, n: PNode): PNode =
       imp.add sep # dummy entry, replaced in the loop
       for x in it[2]:
         # transform `a/b/[c as d]` to `/a/b/c as d`
-        if x.kind == nkInfix and x.sons[0].ident.s == "as":
+        if x.kind == nkInfix and x[0].ident.s == "as":
           let impAs = copyTree(x)
-          imp.sons[2] = x.sons[1]
-          impAs.sons[1] = imp
+          imp[2] = x[1]
+          impAs[1] = imp
           impMod(c, imp, result)
         else:
-          imp.sons[2] = x
+          imp[2] = x
           impMod(c, imp, result)
     else:
       impMod(c, it, result)
@@ -220,22 +219,22 @@ proc evalImport*(c: PContext, n: PNode): PNode =
 proc evalFrom*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n.sons[0] = transformImportAs(c, n.sons[0])
-  var m = myImportModule(c, n.sons[0], result)
+  n[0] = transformImportAs(c, n[0])
+  var m = myImportModule(c, n[0], result)
   if m != nil:
-    n.sons[0] = newSymNode(m)
+    n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
-    for i in 1 ..< len(n):
-      if n.sons[i].kind != nkNilLit:
-        importSymbol(c, n.sons[i], m)
+    for i in 1..<n.len:
+      if n[i].kind != nkNilLit:
+        importSymbol(c, n[i], m)
 
 proc evalImportExcept*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n.sons[0] = transformImportAs(c, n.sons[0])
-  var m = myImportModule(c, n.sons[0], result)
+  n[0] = transformImportAs(c, n[0])
+  var m = myImportModule(c, n[0], result)
   if m != nil:
-    n.sons[0] = newSymNode(m)
+    n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
     importAllSymbolsExcept(c, m, readExceptSet(c, n))
     #importForwarded(c, m.ast, exceptSet, m)

compiler/index.nim

@@ -0,0 +1,18 @@
+##[
+This module only exists to generate docs for the compiler.
+
+## links
+* [main docs](../lib.html)
+* [compiler user guide](../nimc.html)
+* [Internals of the Nim Compiler](../intern.html)
+]##
+
+#[
+note: this is named `index` so that navigating to https://nim-lang.github.io/Nim/compiler/
+will work.
+
+xxx this should also import other modules, not transitively imported by `compiler/nim.nim`,
+eg `evalffi`, otherwise these aren't shown. A glob could be used at CT.
+]#
+
+import nim

compiler/installer.ini

@@ -6,12 +6,12 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv64
-  macosx: i386;amd64;powerpc64
+  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv32;riscv64
+  macosx: i386;amd64;powerpc64;arm64
   solaris: i386;amd64;sparc;sparc64
-  freebsd: i386;amd64;powerpc64
+  freebsd: i386;amd64;powerpc64;arm;arm64;riscv64;sparc64;mips;mipsel;mips64;mips64el;powerpc;powerpc64el
   netbsd: i386;amd64
-  openbsd: i386;amd64
+  openbsd: i386;amd64;arm;arm64
   dragonfly: i386;amd64
   haiku: i386;amd64
   android: i386;arm;arm64
@@ -35,9 +35,8 @@ App: Console
 License: "copying.txt"
 
 [Config]
-Files: "config/nim.cfg"
-Files: "config/nimdoc.cfg"
-Files: "config/nimdoc.tex.cfg"
+Files: "config/*.cfg"
+Files: "config/config.nims"
 
 [Documentation]
 ; Files: "doc/*.html"
@@ -77,6 +76,7 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/fusion"
 
 Files: "tests"
 
@@ -89,6 +89,7 @@ Files: "bin/vccexe.exe"
 Files: "bin/nimgrab.exe"
 Files: "bin/nimpretty.exe"
 Files: "bin/testament.exe"
+Files: "bin/nim-gdb.bat"
 
 Files: "koch.exe"
 Files: "finish.exe"
@@ -109,6 +110,7 @@ Download: r"Aporia Text Editor|dist|aporia.zip|97997|https://nim-lang.org/downlo
 Files: "bin/makelink.exe"
 Files: "bin/7zG.exe"
 Files: "bin/*.dll"
+Files: "bin/cacert.pem"
 
 [UnixBin]
 Files: "bin/nim"
@@ -117,6 +119,8 @@ Files: "bin/nim"
 [Unix]
 InstallScript: "yes"
 UninstallScript: "yes"
+Files: "bin/nim-gdb"
+Files: "bin/nim-gdb.bash"
 
 
 [InnoSetup]
@@ -139,4 +143,4 @@ licenses: "bin/nim,MIT;lib/*,MIT;"
 
 [nimble]
 pkgName: "compiler"
-pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt"
+pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt;doc/nimdoc.css"

compiler/int128.nim

@@ -29,27 +29,27 @@ template high*(t: typedesc[Int128]): Int128 = Max
 
 proc `$`*(a: Int128): string
 
-proc toInt128*[T: SomeInteger](arg: T): Int128 =
-  when T is SomeUnsignedInt:
+proc toInt128*[T: SomeInteger | bool](arg: T): Int128 =
+  when T is bool: result.sdata(0) = int32(arg)
+  elif T is SomeUnsignedInt:
     when sizeof(arg) <= 4:
       result.udata[0] = uint32(arg)
     else:
       result.udata[0] = uint32(arg and T(0xffffffff))
       result.udata[1] = uint32(arg shr 32)
+  elif sizeof(arg) <= 4:
+    result.sdata(0) = int32(arg)
+    if arg < 0: # sign extend
+      result.sdata(1) = -1
+      result.sdata(2) = -1
+      result.sdata(3) = -1
   else:
-    when sizeof(arg) <= 4:
-      result.sdata(0) = int32(arg)
-      if arg < 0: # sign extend
-        result.sdata(1) = -1
-        result.sdata(2) = -1
-        result.sdata(3) = -1
-    else:
-      let tmp = int64(arg)
-      result.udata[0] = uint32(tmp and 0xffffffff)
-      result.sdata(1) = int32(tmp shr 32)
-      if arg < 0: # sign extend
-        result.sdata(2) = -1
-        result.sdata(3) = -1
+    let tmp = int64(arg)
+    result.udata[0] = uint32(tmp and 0xffffffff)
+    result.sdata(1) = int32(tmp shr 32)
+    if arg < 0: # sign extend
+      result.sdata(2) = -1
+      result.sdata(3) = -1
 
 template isNegative(arg: Int128): bool =
   arg.sdata(3) < 0
@@ -162,7 +162,7 @@ proc castToUInt64*(arg: Int128): uint64 =
   cast[uint64](bitconcat(arg.udata[1], arg.udata[0]))
 
 proc addToHex(result: var string; arg: uint32) =
-  for i in 0 ..< 8:
+  for i in 0..<8:
     let idx = (arg shr ((7-i) * 4)) and 0xf
     result.add "0123456789abcdef"[idx]
 
@@ -250,14 +250,14 @@ proc `shr`*(a: Int128, b: int): Int128 =
     if isNegative(a):
       result.sdata(3) = -1
     result.sdata(2) = a.sdata(3) shr (b and 31)
-    result.udata[1] = cast[uint32](bitconcat(a.udata[2], a.udata[1]) shr (b and 31))
-    result.udata[0] = cast[uint32](bitconcat(a.udata[1], a.udata[0]) shr (b and 31))
+    result.udata[1] = cast[uint32](bitconcat(a.udata[3], a.udata[2]) shr (b and 31))
+    result.udata[0] = cast[uint32](bitconcat(a.udata[2], a.udata[1]) shr (b and 31))
   elif b < 96:
     if isNegative(a):
       result.sdata(3) = -1
       result.sdata(2) = -1
     result.sdata(1) = a.sdata(3) shr (b and 31)
-    result.udata[0] = cast[uint32](bitconcat(a.udata[1], a.udata[0]) shr (b and 31))
+    result.udata[0] = cast[uint32](bitconcat(a.udata[3], a.udata[2]) shr (b and 31))
   else: # b < 128
     if isNegative(a):
       result.sdata(3) = -1
@@ -373,8 +373,8 @@ proc `*`*(lhs,rhs: Int128): Int128 =
   let b00 = cast[uint64](rhs.udata[0])
 
   result = makeInt128(high64(lhs) * low64(rhs) + low64(lhs) * high64(rhs) + a32 * b32, a00 * b00)
-  result = result + toInt128(a32 * b00) shl 32
-  result = result + toInt128(a00 * b32) shl 32
+  result += toInt128(a32 * b00) shl 32
+  result += toInt128(a00 * b32) shl 32
 
 proc `*=`*(a: var Int128, b: Int128) =
   a = a * b
@@ -424,10 +424,10 @@ proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
 
   # Uses shift-subtract algorithm to divide dividend by denominator. The
   # remainder will be left in dividend.
-  for i in 0 .. shift:
+  for i in 0..shift:
     quotient = quotient shl 1
     if dividend >= denominator:
-      dividend = dividend - denominator
+      dividend -= denominator
       quotient = bitor(quotient, One)
 
     denominator = denominator shr 1
@@ -486,7 +486,7 @@ proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =
     pos += 1
 
   result = Zero
-  while pos < arg.len and arg[pos] in '0' .. '9':
+  while pos < arg.len and arg[pos] in '0'..'9':
     result = result * Ten
     result.inc(uint32(arg[pos]) - uint32('0'))
     pos += 1
@@ -592,107 +592,3 @@ proc maskBytes*(arg: Int128, numbytes: int): Int128 {.noinit.} =
     return maskUInt64(arg)
   else:
     assert(false, "masking only implemented for 1, 2, 4 and 8 bytes")
-
-
-
-
-when isMainModule:
-  let (a,b) = divMod(Ten,Ten)
-
-  doAssert $One == "1"
-  doAssert $Ten == "10"
-  doAssert $Zero == "0"
-  let c = parseDecimalInt128("12345678989876543210123456789")
-  doAssert $c == "12345678989876543210123456789"
-
-  var d : array[39, Int128]
-  d[0] =  parseDecimalInt128("1")
-  d[1] =  parseDecimalInt128("10")
-  d[2] =  parseDecimalInt128("100")
-  d[3] =  parseDecimalInt128("1000")
-  d[4] =  parseDecimalInt128("10000")
-  d[5] =  parseDecimalInt128("100000")
-  d[6] =  parseDecimalInt128("1000000")
-  d[7] =  parseDecimalInt128("10000000")
-  d[8] =  parseDecimalInt128("100000000")
-  d[9] =  parseDecimalInt128("1000000000")
-  d[10] = parseDecimalInt128("10000000000")
-  d[11] = parseDecimalInt128("100000000000")
-  d[12] = parseDecimalInt128("1000000000000")
-  d[13] = parseDecimalInt128("10000000000000")
-  d[14] = parseDecimalInt128("100000000000000")
-  d[15] = parseDecimalInt128("1000000000000000")
-  d[16] = parseDecimalInt128("10000000000000000")
-  d[17] = parseDecimalInt128("100000000000000000")
-  d[18] = parseDecimalInt128("1000000000000000000")
-  d[19] = parseDecimalInt128("10000000000000000000")
-  d[20] = parseDecimalInt128("100000000000000000000")
-  d[21] = parseDecimalInt128("1000000000000000000000")
-  d[22] = parseDecimalInt128("10000000000000000000000")
-  d[23] = parseDecimalInt128("100000000000000000000000")
-  d[24] = parseDecimalInt128("1000000000000000000000000")
-  d[25] = parseDecimalInt128("10000000000000000000000000")
-  d[26] = parseDecimalInt128("100000000000000000000000000")
-  d[27] = parseDecimalInt128("1000000000000000000000000000")
-  d[28] = parseDecimalInt128("10000000000000000000000000000")
-  d[29] = parseDecimalInt128("100000000000000000000000000000")
-  d[30] = parseDecimalInt128("1000000000000000000000000000000")
-  d[31] = parseDecimalInt128("10000000000000000000000000000000")
-  d[32] = parseDecimalInt128("100000000000000000000000000000000")
-  d[33] = parseDecimalInt128("1000000000000000000000000000000000")
-  d[34] = parseDecimalInt128("10000000000000000000000000000000000")
-  d[35] = parseDecimalInt128("100000000000000000000000000000000000")
-  d[36] = parseDecimalInt128("1000000000000000000000000000000000000")
-  d[37] = parseDecimalInt128("10000000000000000000000000000000000000")
-  d[38] = parseDecimalInt128("100000000000000000000000000000000000000")
-
-  for i in 0 ..< d.len:
-    for j in 0 ..< d.len:
-      doAssert(cmp(d[i], d[j]) == cmp(i,j))
-      if i + j < d.len:
-        doAssert d[i] * d[j] == d[i+j]
-      if i - j >= 0:
-        doAssert d[i] div d[j] == d[i-j]
-
-  var sum: Int128
-
-  for it in d:
-    sum += it
-
-  doAssert $sum == "111111111111111111111111111111111111111"
-
-  for it in d.mitems:
-    it = -it
-
-  for i in 0 ..< d.len:
-    for j in 0 ..< d.len:
-      doAssert(cmp(d[i], d[j]) == -cmp(i,j))
-      if i + j < d.len:
-        doAssert d[i] * d[j] == -d[i+j]
-      if i - j >= 0:
-        doAssert d[i] div d[j] == -d[i-j]
-
-  doAssert $high(Int128) == "170141183460469231731687303715884105727"
-  doAssert $low(Int128) == "-170141183460469231731687303715884105728"
-
-  var ma = 100'i64
-  var mb = 13
-
-  doAssert toInt128(ma) * toInt128(0) == toInt128(0)
-  doAssert toInt128(-ma) * toInt128(0) == toInt128(0)
-
-  # sign correctness
-  doAssert divMod(toInt128( ma),toInt128( mb)) == (toInt128( ma div  mb), toInt128( ma mod  mb))
-  doAssert divMod(toInt128(-ma),toInt128( mb)) == (toInt128(-ma div  mb), toInt128(-ma mod  mb))
-  doAssert divMod(toInt128( ma),toInt128(-mb)) == (toInt128( ma div -mb), toInt128( ma mod -mb))
-  doAssert divMod(toInt128(-ma),toInt128(-mb)) == (toInt128(-ma div -mb), toInt128(-ma mod -mb))
-
-  doAssert divMod(toInt128( mb),toInt128( mb)) == (toInt128( mb div  mb), toInt128( mb mod  mb))
-  doAssert divMod(toInt128(-mb),toInt128( mb)) == (toInt128(-mb div  mb), toInt128(-mb mod  mb))
-  doAssert divMod(toInt128( mb),toInt128(-mb)) == (toInt128( mb div -mb), toInt128( mb mod -mb))
-  doAssert divMod(toInt128(-mb),toInt128(-mb)) == (toInt128(-mb div -mb), toInt128(-mb mod -mb))
-
-  doAssert divMod(toInt128( mb),toInt128( ma)) == (toInt128( mb div  ma), toInt128( mb mod  ma))
-  doAssert divMod(toInt128(-mb),toInt128( ma)) == (toInt128(-mb div  ma), toInt128(-mb mod  ma))
-  doAssert divMod(toInt128( mb),toInt128(-ma)) == (toInt128( mb div -ma), toInt128( mb mod -ma))
-  doAssert divMod(toInt128(-mb),toInt128(-ma)) == (toInt128(-mb div -ma), toInt128(-mb mod -ma))

compiler/isolation_check.nim

@@ -0,0 +1,126 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Implementation of the check that `recover` needs, see
+## https://github.com/nim-lang/RFCs/issues/244 for more details.
+
+import
+  ast, types, renderer, idents, intsets
+
+proc canAlias(arg, ret: PType; marker: var IntSet): bool
+
+proc canAliasN(arg: PType; n: PNode; marker: var IntSet): bool =
+  case n.kind
+  of nkRecList:
+    for i in 0..<n.len:
+      result = canAliasN(arg, n[i], marker)
+      if result: return
+  of nkRecCase:
+    assert(n[0].kind == nkSym)
+    result = canAliasN(arg, n[0], marker)
+    if result: return
+    for i in 1..<n.len:
+      case n[i].kind
+      of nkOfBranch, nkElse:
+        result = canAliasN(arg, lastSon(n[i]), marker)
+        if result: return
+      else: discard
+  of nkSym:
+    result = canAlias(arg, n.sym.typ, marker)
+  else: discard
+
+proc canAlias(arg, ret: PType; marker: var IntSet): bool =
+  if containsOrIncl(marker, ret.id):
+    return false
+
+  if ret.kind in {tyPtr, tyPointer}:
+    # unsafe so we don't care:
+    return false
+  if compareTypes(arg, ret, dcEqIgnoreDistinct):
+    return true
+  case ret.kind
+  of tyObject:
+    if isFinal(ret):
+      result = canAliasN(arg, ret.n, marker)
+      if not result and ret.len > 0 and ret[0] != nil:
+        result = canAlias(arg, ret[0], marker)
+    else:
+      result = true
+  of tyTuple:
+    for i in 0..<ret.len:
+      result = canAlias(arg, ret[i], marker)
+      if result: break
+  of tyArray, tySequence, tyDistinct, tyGenericInst,
+     tyAlias, tyInferred, tySink, tyLent, tyOwned, tyRef:
+    result = canAlias(arg, ret.lastSon, marker)
+  of tyProc:
+    result = ret.callConv == ccClosure
+  else:
+    result = false
+
+proc isValueOnlyType(t: PType): bool = 
+  # t doesn't contain pointers and references
+  proc wrap(t: PType): bool {.nimcall.} = t.kind in {tyRef, tyPtr, tyVar, tyLent}
+  result = not types.searchTypeFor(t, wrap)
+
+proc canAlias*(arg, ret: PType): bool =
+  if isValueOnlyType(arg):
+    # can alias only with unsafeAddr(arg.x) and we don't care if it is not safe
+    result = false
+  else:
+    var marker = initIntSet()
+    result = canAlias(arg, ret, marker)
+
+proc checkIsolate*(n: PNode): bool =
+  if types.containsTyRef(n.typ):
+    # XXX Maybe require that 'n.typ' is acyclic. This is not much
+    # worse than the already exisiting inheritance and closure restrictions.
+    case n.kind
+    of nkCharLit..nkNilLit:
+      result = true
+    of nkCallKinds:
+      if n[0].typ.flags * {tfGcSafe, tfNoSideEffect} == {}:
+        return false
+      for i in 1..<n.len:
+        if checkIsolate(n[i]):
+          discard "fine, it is isolated already"
+        else:
+          let argType = n[i].typ
+          if argType != nil and not isCompileTimeOnly(argType) and containsTyRef(argType):
+            if argType.canAlias(n.typ):
+              return false
+      result = true
+    of nkIfStmt, nkIfExpr:
+      for it in n:
+        result = checkIsolate(it.lastSon)
+        if not result: break
+    of nkCaseStmt, nkObjConstr:
+      for i in 1..<n.len:
+        result = checkIsolate(n[i].lastSon)
+        if not result: break
+    of nkBracket, nkTupleConstr, nkPar:
+      for it in n:
+        result = checkIsolate(it)
+        if not result: break
+    of nkHiddenStdConv, nkHiddenSubConv, nkCast, nkConv:
+      result = checkIsolate(n[1])
+    of nkObjUpConv, nkObjDownConv, nkDotExpr:
+      result = checkIsolate(n[0])
+    of nkStmtList, nkStmtListExpr:
+      if n.len > 0:
+        result = checkIsolate(n[^1])
+      else:
+        result = false
+    else:
+      # unanalysable expression:
+      result = false
+  else:
+    # no ref, no cry:
+    result = true
+

compiler/jstypes.nim

@@ -17,22 +17,20 @@ proc genTypeInfo(p: PProc, typ: PType): Rope
 proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
   var
     s, u: Rope
-    length: int
     field: PSym
     b: PNode
   result = nil
   case n.kind
   of nkRecList:
-    length = len(n)
-    if length == 1:
-      result = genObjectFields(p, typ, n.sons[0])
+    if n.len == 1:
+      result = genObjectFields(p, typ, n[0])
     else:
       s = nil
-      for i in 0 ..< length:
-        if i > 0: add(s, ", \L")
-        add(s, genObjectFields(p, typ, n.sons[i]))
+      for i in 0..<n.len:
+        if i > 0: s.add(", \L")
+        s.add(genObjectFields(p, typ, n[i]))
       result = ("{kind: 2, len: $1, offset: 0, " &
-          "typ: null, name: null, sons: [$2]}") % [rope(length), s]
+          "typ: null, name: null, sons: [$2]}") % [rope(n.len), s]
   of nkSym:
     field = n.sym
     s = genTypeInfo(p, field.typ)
@@ -41,29 +39,28 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
                    [mangleName(p.module, field), s,
                     makeJSString(field.name.s)]
   of nkRecCase:
-    length = len(n)
-    if (n.sons[0].kind != nkSym): internalError(p.config, n.info, "genObjectFields")
-    field = n.sons[0].sym
+    if (n[0].kind != nkSym): internalError(p.config, n.info, "genObjectFields")
+    field = n[0].sym
     s = genTypeInfo(p, field.typ)
-    for i in 1 ..< length:
-      b = n.sons[i]           # branch
+    for i in 1..<n.len:
+      b = n[i]           # branch
       u = nil
       case b.kind
       of nkOfBranch:
-        if len(b) < 2:
+        if b.len < 2:
           internalError(p.config, b.info, "genObjectFields; nkOfBranch broken")
-        for j in 0 .. len(b) - 2:
-          if u != nil: add(u, ", ")
-          if b.sons[j].kind == nkRange:
-            addf(u, "[$1, $2]", [rope(getOrdValue(b.sons[j].sons[0])),
-                                 rope(getOrdValue(b.sons[j].sons[1]))])
+        for j in 0..<b.len - 1:
+          if u != nil: u.add(", ")
+          if b[j].kind == nkRange:
+            u.addf("[$1, $2]", [rope(getOrdValue(b[j][0])),
+                                 rope(getOrdValue(b[j][1]))])
           else:
-            add(u, rope(getOrdValue(b.sons[j])))
+            u.add(rope(getOrdValue(b[j])))
       of nkElse:
         u = rope(lengthOrd(p.config, field.typ))
       else: internalError(p.config, n.info, "genObjectFields(nkRecCase)")
-      if result != nil: add(result, ", \L")
-      addf(result, "[setConstr($1), $2]",
+      if result != nil: result.add(", \L")
+      result.addf("[setConstr($1), $2]",
            [u, genObjectFields(p, typ, lastSon(b))])
     result = ("{kind: 3, offset: \"$1\", len: $3, " &
         "typ: $2, name: $4, sons: [$5]}") % [
@@ -72,27 +69,27 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
   else: internalError(p.config, n.info, "genObjectFields")
 
 proc objHasTypeField(t: PType): bool {.inline.} =
-  tfInheritable in t.flags or t.sons[0] != nil
+  tfInheritable in t.flags or t[0] != nil
 
 proc genObjectInfo(p: PProc, typ: PType, name: Rope) =
   let kind = if objHasTypeField(typ): tyObject else: tyTuple
   var s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
            "finalizer: null};$n") % [name, rope(ord(kind))]
   prepend(p.g.typeInfo, s)
-  addf(p.g.typeInfo, "var NNI$1 = $2;$n",
+  p.g.typeInfo.addf("var NNI$1 = $2;$n",
        [rope(typ.id), genObjectFields(p, typ, typ.n)])
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
-  if (typ.kind == tyObject) and (typ.sons[0] != nil):
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [name, genTypeInfo(p, typ.sons[0].skipTypes(skipPtrs))])
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
+  if (typ.kind == tyObject) and (typ[0] != nil):
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [name, genTypeInfo(p, typ[0].skipTypes(skipPtrs))])
 
 proc genTupleFields(p: PProc, typ: PType): Rope =
   var s: Rope = nil
-  for i in 0 ..< typ.len:
-    if i > 0: add(s, ", \L")
+  for i in 0..<typ.len:
+    if i > 0: s.add(", \L")
     s.addf("{kind: 1, offset: \"Field$1\", len: 0, " &
            "typ: $2, name: \"Field$1\", sons: null}",
-           [i.rope, genTypeInfo(p, typ.sons[i])])
+           [i.rope, genTypeInfo(p, typ[i])])
   result = ("{kind: 2, len: $1, offset: 0, " &
             "typ: null, name: null, sons: [$2]}") % [rope(typ.len), s]
 
@@ -100,30 +97,29 @@ proc genTupleInfo(p: PProc, typ: PType, name: Rope) =
   var s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
            "finalizer: null};$n") % [name, rope(ord(typ.kind))]
   prepend(p.g.typeInfo, s)
-  addf(p.g.typeInfo, "var NNI$1 = $2;$n",
+  p.g.typeInfo.addf("var NNI$1 = $2;$n",
        [rope(typ.id), genTupleFields(p, typ)])
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
 
 proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
-  let length = len(typ.n)
   var s: Rope = nil
-  for i in 0 ..< length:
-    if (typ.n.sons[i].kind != nkSym): internalError(p.config, typ.n.info, "genEnumInfo")
-    let field = typ.n.sons[i].sym
-    if i > 0: add(s, ", \L")
+  for i in 0..<typ.n.len:
+    if (typ.n[i].kind != nkSym): internalError(p.config, typ.n.info, "genEnumInfo")
+    let field = typ.n[i].sym
+    if i > 0: s.add(", \L")
     let extName = if field.ast == nil: field.name.s else: field.ast.strVal
-    addf(s, "\"$1\": {kind: 1, offset: $1, typ: $2, name: $3, len: 0, sons: null}",
+    s.addf("\"$1\": {kind: 1, offset: $1, typ: $2, name: $3, len: 0, sons: null}",
          [rope(field.position), name, makeJSString(extName)])
   var n = ("var NNI$1 = {kind: 2, offset: 0, typ: null, " &
-      "name: null, len: $2, sons: {$3}};$n") % [rope(typ.id), rope(length), s]
+      "name: null, len: $2, sons: {$3}};$n") % [rope(typ.id), rope(typ.n.len), s]
   s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
        "finalizer: null};$n") % [name, rope(ord(typ.kind))]
   prepend(p.g.typeInfo, s)
-  add(p.g.typeInfo, n)
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
-  if typ.sons[0] != nil:
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [name, genTypeInfo(p, typ.sons[0])])
+  p.g.typeInfo.add(n)
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
+  if typ[0] != nil:
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [name, genTypeInfo(p, typ[0])])
 
 proc genTypeInfo(p: PProc, typ: PType): Rope =
   let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink, tyOwned})
@@ -131,7 +127,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
   if containsOrIncl(p.g.typeInfoGenerated, t.id): return
   case t.kind
   of tyDistinct:
-    result = genTypeInfo(p, t.sons[0])
+    result = genTypeInfo(p, t[0])
   of tyPointer, tyProc, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
@@ -142,15 +138,15 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-    addf(p.g.typeInfo, "$1.base = $2;$n",
+    p.g.typeInfo.addf("$1.base = $2;$n",
          [result, genTypeInfo(p, t.lastSon)])
   of tyArray:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [result, genTypeInfo(p, t.sons[1])])
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [result, genTypeInfo(p, t[1])])
   of tyEnum: genEnumInfo(p, t, result)
   of tyObject: genObjectInfo(p, t, result)
   of tyTuple: genTupleInfo(p, t, result)

compiler/lambdalifting.nim

@@ -128,8 +128,8 @@ proc newCall(a: PSym, b: PNode): PNode =
 
 proc createClosureIterStateType*(g: ModuleGraph; iter: PSym): PType =
   var n = newNodeI(nkRange, iter.info)
-  addSon(n, newIntNode(nkIntLit, -1))
-  addSon(n, newIntNode(nkIntLit, 0))
+  n.add newIntNode(nkIntLit, -1)
+  n.add newIntNode(nkIntLit, 0)
   result = newType(tyRange, iter)
   result.n = n
   var intType = nilOrSysInt(g)
@@ -148,27 +148,27 @@ proc createEnvObj(g: ModuleGraph; owner: PSym; info: TLineInfo): PType =
 
 proc getClosureIterResult*(g: ModuleGraph; iter: PSym): PSym =
   if resultPos < iter.ast.len:
-    result = iter.ast.sons[resultPos].sym
+    result = iter.ast[resultPos].sym
   else:
     # XXX a bit hacky:
     result = newSym(skResult, getIdent(g.cache, ":result"), iter, iter.info, {})
-    result.typ = iter.typ.sons[0]
+    result.typ = iter.typ[0]
     incl(result.flags, sfUsed)
     iter.ast.add newSymNode(result)
 
 proc addHiddenParam(routine: PSym, param: PSym) =
   assert param.kind == skParam
-  var params = routine.ast.sons[paramsPos]
+  var params = routine.ast[paramsPos]
   # -1 is correct here as param.position is 0 based but we have at position 0
   # some nkEffect node:
   param.position = routine.typ.n.len-1
-  addSon(params, newSymNode(param))
+  params.add newSymNode(param)
   #incl(routine.typ.flags, tfCapturesEnv)
   assert sfFromGeneric in param.flags
   #echo "produced environment: ", param.id, " for ", routine.id
 
 proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
-  let params = routine.ast.sons[paramsPos]
+  let params = routine.ast[paramsPos]
   let hidden = lastSon(params)
   if hidden.kind == nkSym and hidden.sym.kind == skParam and hidden.sym.name.s == paramName:
     result = hidden.sym
@@ -179,7 +179,7 @@ proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
     result = routine
 
 proc getEnvParam*(routine: PSym): PSym =
-  let params = routine.ast.sons[paramsPos]
+  let params = routine.ast[paramsPos]
   let hidden = lastSon(params)
   if hidden.kind == nkSym and hidden.sym.name.s == paramName:
     result = hidden.sym
@@ -206,8 +206,8 @@ proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode =
   # which is however the only case when we generate an assignment in the first
   # place.
   result = newNodeI(nkAsgn, info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
 
 proc makeClosure*(g: ModuleGraph; prc: PSym; env: PNode; info: TLineInfo): PNode =
   result = newNodeIT(nkClosure, info, prc.typ)
@@ -218,6 +218,10 @@ proc makeClosure*(g: ModuleGraph; prc: PSym; env: PNode; info: TLineInfo): PNode
     if env.skipConv.kind == nkClosure:
       localError(g.config, info, "internal error: taking closure of closure")
     result.add(env)
+  #if isClosureIterator(result.typ):
+  createTypeBoundOps(g, nil, result.typ, info)
+  if tfHasAsgn in result.typ.flags or optSeqDestructors in g.config.globalOptions:
+    prc.flags.incl sfInjectDestructors
 
 proc interestingIterVar(s: PSym): bool {.inline.} =
   # XXX optimization: Only lift the variable if it lives across
@@ -231,11 +235,14 @@ template isIterator*(owner: PSym): bool =
 proc liftingHarmful(conf: ConfigRef; owner: PSym): bool {.inline.} =
   ## lambda lifting can be harmful for JS-like code generators.
   let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
-  result = conf.cmd == cmdCompileToJS and not isCompileTime
+  result = conf.backend == backendJs and not isCompileTime
 
-proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo) =
-  createTypeBoundOps(g, nil, refType.lastSon, info)
-  createTypeBoundOps(g, nil, refType, info)
+proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; owner: PSym) =
+  if owner.kind != skMacro:
+    createTypeBoundOps(g, nil, refType.lastSon, info)
+    createTypeBoundOps(g, nil, refType, info)
+    if tfHasAsgn in refType.flags or optSeqDestructors in g.config.globalOptions:
+      owner.flags.incl sfInjectDestructors
 
 proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env))
@@ -249,7 +256,7 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   var env: PNode
   if owner.isIterator:
     let it = getHiddenParam(g, owner)
-    addUniqueField(it.typ.skipTypes({tyOwned}).sons[0], hp, g.cache)
+    addUniqueField(it.typ.skipTypes({tyOwned})[0], hp, g.cache)
     env = indirectAccess(newSymNode(it), hp, hp.info)
   else:
     let e = newSym(skLet, iter.name, owner, n.info)
@@ -261,8 +268,7 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
     result.add(v)
   # add 'new' statement:
   result.add newCall(getSysSym(g, n.info, "internalNew"), env)
-  if optOwnedRefs in g.config.globalOptions:
-    createTypeBoundOps(g, nil, env.typ, n.info)
+  createTypeBoundOpsLL(g, env.typ, n.info, owner)
   result.add makeClosure(g, iter, env, n.info)
 
 proc freshVarForClosureIter*(g: ModuleGraph; s, owner: PSym): PNode =
@@ -286,10 +292,11 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
   if illegalCapture(s):
     localError(g.config, n.info,
       ("'$1' is of type <$2> which cannot be captured as it would violate memory" &
-       " safety, declared here: $3") % [s.name.s, typeToString(s.typ), g.config$s.info])
-  elif owner.typ.callConv notin {ccClosure, ccDefault}:
+       " safety, declared here: $3; using '-d:nimWorkaround14447' helps in some cases") %
+      [s.name.s, typeToString(s.typ), g.config$s.info])
+  elif not (owner.typ.callConv == ccClosure or owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags):
     localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
-      [s.name.s, owner.name.s, CallingConvToStr[owner.typ.callConv]])
+      [s.name.s, owner.name.s, $owner.typ.callConv])
   incl(owner.typ.flags, tfCapturesEnv)
   owner.typ.callConv = ccClosure
 
@@ -297,7 +304,7 @@ type
   DetectionPass = object
     processed, capturedVars: IntSet
     ownerToType: Table[int, PType]
-    somethingToDo, noDestructors: bool
+    somethingToDo: bool
     graph: ModuleGraph
 
 proc initDetectionPass(g: ModuleGraph; fn: PSym): DetectionPass =
@@ -348,8 +355,8 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
   # with cycles properly, so it's better to produce a weak ref (=ptr) here.
   # This seems to be generally correct but since it's a bit risky it's disabled
   # for now.
-  let fieldType = if isDefined(c.graph.config, "nimCycleBreaker") or
-                      c.graph.config.selectedGC == gcDestructors:
+  # XXX This is wrong for the 'hamming' test, so remove this logic again.
+  let fieldType = if isDefined(c.graph.config, "nimCycleBreaker"):
                     c.getEnvTypeForOwnerUp(dep, info) #getHiddenParam(dep).typ
                   else:
                     c.getEnvTypeForOwner(dep, info)
@@ -361,9 +368,16 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
   if upField != nil:
     if upField.typ.skipTypes({tyOwned, tyRef, tyPtr}) != fieldType.skipTypes({tyOwned, tyRef, tyPtr}):
       localError(c.graph.config, dep.info, "internal error: up references do not agree")
+
+    when false:
+      if c.graph.config.selectedGC == gcDestructors and sfCursor notin upField.flags:
+        localError(c.graph.config, dep.info, "internal error: up reference is not a .cursor")
   else:
     let result = newSym(skField, upIdent, obj.owner, obj.owner.info)
     result.typ = fieldType
+    when false:
+      if c.graph.config.selectedGC == gcDestructors:
+        result.flags.incl sfCursor
     rawAddField(obj, result)
 
 discard """
@@ -420,8 +434,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     if innerProc:
       if s.isIterator: c.somethingToDo = true
       if not c.processed.containsOrIncl(s.id):
-        let body = transformBody(c.graph, s, cache = true,
-                                 noDestructors = c.noDestructors)
+        let body = transformBody(c.graph, s, cache = true)
         detectCapturedVars(body, s, c)
     let ow = s.skipGenericOwner
     if ow == owner:
@@ -482,7 +495,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
           w = up
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit,
      nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef,
-     nkConverterDef, nkMacroDef, nkFuncDef, nkCommentStmt, nkTypeOfExpr:
+     nkConverterDef, nkMacroDef, nkFuncDef, nkCommentStmt,
+     nkTypeOfExpr, nkMixinStmt, nkBindStmt:
     discard
   of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil:
@@ -512,7 +526,7 @@ proc accessViaEnvParam(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   if not envParam.isNil:
     var access = newSymNode(envParam)
     while true:
-      let obj = access.typ.sons[0]
+      let obj = access.typ[0]
       assert obj.kind == tyObject
       let field = getFieldFromObj(obj, s)
       if field != nil:
@@ -531,7 +545,7 @@ proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo): PNo
   when false:
     if owner.kind == skIterator and owner.typ.callConv == ccClosure:
       let it = getHiddenParam(owner)
-      addUniqueField(it.typ.sons[0], v)
+      addUniqueField(it.typ[0], v)
       result = indirectAccess(newSymNode(it), v, v.info)
     else:
       result = newSymNode(v)
@@ -590,7 +604,7 @@ proc rawClosureCreation(owner: PSym;
       let env2 = copyTree(env)
       env2.typ = unowned.typ
       result.add newAsgnStmt(unowned, env2, env.info)
-      createTypeBoundOpsLL(d.graph, unowned.typ, env.info)
+      createTypeBoundOpsLL(d.graph, unowned.typ, env.info, owner)
 
     # add assignment statements for captured parameters:
     for i in 1..<owner.typ.n.len:
@@ -599,6 +613,10 @@ proc rawClosureCreation(owner: PSym;
         let fieldAccess = indirectAccess(env, local, env.info)
         # add ``env.param = param``
         result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info))
+        if owner.kind != skMacro:
+          createTypeBoundOps(d.graph, nil, fieldAccess.typ, env.info)
+        if tfHasAsgn in fieldAccess.typ.flags or optSeqDestructors in d.graph.config.globalOptions:
+          owner.flags.incl sfInjectDestructors
 
   let upField = lookupInRecord(env.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
@@ -613,8 +631,7 @@ proc rawClosureCreation(owner: PSym;
       localError(d.graph.config, env.info, "internal error: cannot create up reference")
   # we are not in the sem'check phase anymore! so pass 'nil' for the PContext
   # and hope for the best:
-  if optOwnedRefs in d.graph.config.globalOptions:
-    createTypeBoundOps(d.graph, nil, env.typ, owner.info)
+  createTypeBoundOpsLL(d.graph, env.typ, owner.info, owner)
 
 proc finishClosureCreation(owner: PSym; d: DetectionPass; c: LiftingPass;
                            info: TLineInfo; res: PNode) =
@@ -623,6 +640,7 @@ proc finishClosureCreation(owner: PSym; d: DetectionPass; c: LiftingPass;
     assert unowned != nil
     let nilLit = newNodeIT(nkNilLit, info, unowned.typ)
     res.add newAsgnStmt(unowned, nilLit, info)
+    createTypeBoundOpsLL(d.graph, unowned.typ, info, owner)
 
 proc closureCreationForIter(iter: PNode;
                             d: DetectionPass; c: var LiftingPass): PNode =
@@ -642,8 +660,7 @@ proc closureCreationForIter(iter: PNode;
     addVar(vs, vnode)
     result.add(vs)
   result.add(newCall(getSysSym(d.graph, iter.info, "internalNew"), vnode))
-  if optOwnedRefs in d.graph.config.globalOptions:
-    createTypeBoundOps(d.graph, nil, vnode.typ, iter.info)
+  createTypeBoundOpsLL(d.graph, vnode.typ, iter.info, owner)
 
   let upField = lookupInRecord(v.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
@@ -669,7 +686,7 @@ proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
     result = n
 
 proc getStateField*(g: ModuleGraph; owner: PSym): PSym =
-  getHiddenParam(g, owner).typ.skipTypes({tyOwned, tyRef, tyPtr}).n.sons[0].sym
+  getHiddenParam(g, owner).typ.skipTypes({tyOwned, tyRef, tyPtr}).n[0].sym
 
 proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
                       c: var LiftingPass): PNode
@@ -713,7 +730,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         #  echo renderTree(s.getBody, {renderIds})
         let oldInContainer = c.inContainer
         c.inContainer = 0
-        var body = transformBody(d.graph, s)
+        var body = transformBody(d.graph, s, cache = false)
         body = liftCapturedVars(body, s, d, c)
         if c.envVars.getOrDefault(s.id).isNil:
           s.transformedBody = body
@@ -734,17 +751,17 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         result = accessViaEnvVar(n, owner, d, c)
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom,
      nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef, nkConverterDef,
-     nkMacroDef, nkFuncDef:
+     nkMacroDef, nkFuncDef, nkMixinStmt, nkBindStmt:
     discard
   of nkClosure:
     if n[1].kind == nkNilLit:
-      n.sons[0] = liftCapturedVars(n[0], owner, d, c)
-      let x = n.sons[0].skipConv
+      n[0] = liftCapturedVars(n[0], owner, d, c)
+      let x = n[0].skipConv
       if x.kind == nkClosure:
         #localError(n.info, "internal error: closure to closure created")
         # now we know better, so patch it:
-        n.sons[0] = x.sons[0]
-        n.sons[1] = x.sons[1]
+        n[0] = x[0]
+        n[1] = x[1]
   of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil and n[namePos].kind == nkSym:
       let oldInContainer = c.inContainer
@@ -755,29 +772,29 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
       c.inContainer = oldInContainer
   of nkHiddenStdConv:
     if n.len == 2:
-      n.sons[1] = liftCapturedVars(n[1], owner, d, c)
+      n[1] = liftCapturedVars(n[1], owner, d, c)
       if n[1].kind == nkClosure: result = n[1]
   of nkReturnStmt:
     if n[0].kind in {nkAsgn, nkFastAsgn}:
       # we have a `result = result` expression produced by the closure
       # transform, let's not touch the LHS in order to make the lifting pass
       # correct when `result` is lifted
-      n[0].sons[1] = liftCapturedVars(n[0].sons[1], owner, d, c)
+      n[0][1] = liftCapturedVars(n[0][1], owner, d, c)
     else:
-      n.sons[0] = liftCapturedVars(n[0], owner, d, c)
+      n[0] = liftCapturedVars(n[0], owner, d, c)
   of nkTypeOfExpr:
     result = n
   else:
     if owner.isIterator:
       if nfLL in n.flags:
         # special case 'when nimVm' due to bug #3636:
-        n.sons[1] = liftCapturedVars(n[1], owner, d, c)
+        n[1] = liftCapturedVars(n[1], owner, d, c)
         return
 
     let inContainer = n.kind in {nkObjConstr, nkBracket}
     if inContainer: inc c.inContainer
     for i in 0..<n.len:
-      n.sons[i] = liftCapturedVars(n[i], owner, d, c)
+      n[i] = liftCapturedVars(n[i], owner, d, c)
     if inContainer: dec c.inContainer
 
 # ------------------ old stuff -------------------------------------------
@@ -808,7 +825,8 @@ proc semCaptureSym*(s, owner: PSym) =
       var o = owner.skipGenericOwner
       while o != nil and o.kind != skModule:
         if s.owner == o:
-          if owner.typ.callConv in {ccClosure, ccDefault} or owner.kind == skIterator:
+          if owner.typ.callConv == ccClosure or owner.kind == skIterator or
+             owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags:
             owner.typ.callConv = ccClosure
             propagateClosure(owner.skipGenericOwner, s.owner)
           else:
@@ -824,24 +842,23 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType): PNo
   # pretend 'fn' is a closure iterator for the analysis:
   let oldKind = fn.kind
   let oldCC = fn.typ.callConv
-  fn.kind = skIterator
+  fn.transitionRoutineSymKind(skIterator)
   fn.typ.callConv = ccClosure
   d.ownerToType[fn.id] = ptrType
   detectCapturedVars(body, fn, d)
   result = liftCapturedVars(body, fn, d, c)
-  fn.kind = oldKind
+  fn.transitionRoutineSymKind(oldKind)
   fn.typ.callConv = oldCC
 
-proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
-                  noDestructors: bool): PNode =
-  # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
+proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PNode =
+  # XXX backend == backendJs does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
 
   # However we can do lifting for the stuff which is *only* compiletime.
   let isCompileTime = sfCompileTime in fn.flags or fn.kind == skMacro
 
   if body.kind == nkEmpty or (
-      g.config.cmd == cmdCompileToJS and not isCompileTime) or
+      g.config.backend == backendJs and not isCompileTime) or
       fn.skipGenericOwner.kind != skModule:
 
     # ignore forward declaration:
@@ -849,7 +866,6 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
     tooEarly = true
   else:
     var d = initDetectionPass(g, fn)
-    d.noDestructors = noDestructors
     detectCapturedVars(body, fn, d)
     if not d.somethingToDo and fn.isIterator:
       addClosureParam(d, fn, body.info)
@@ -903,11 +919,10 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
         ...
     """
   if liftingHarmful(g.config, owner): return body
-  var L = body.len
-  if not (body.kind == nkForStmt and body[L-2].kind in nkCallKinds):
+  if not (body.kind == nkForStmt and body[^2].kind in nkCallKinds):
     localError(g.config, body.info, "ignored invalid for loop")
     return body
-  var call = body[L-2]
+  var call = body[^2]
 
   result = newNodeI(nkStmtList, body.info)
 
@@ -928,40 +943,39 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
     result.add(v)
     # add 'new' statement:
     result.add(newCall(getSysSym(g, env.info, "internalNew"), env.newSymNode))
-    if optOwnedRefs in g.config.globalOptions:
-      createTypeBoundOps(g, nil, env.typ, body.info)
+    createTypeBoundOpsLL(g, env.typ, body.info, owner)
 
   elif op.kind == nkStmtListExpr:
     let closure = op.lastSon
     if closure.kind == nkClosure:
-      call.sons[0] = closure
-      for i in 0 .. op.len-2:
+      call[0] = closure
+      for i in 0..<op.len-1:
         result.add op[i]
 
   var loopBody = newNodeI(nkStmtList, body.info, 3)
   var whileLoop = newNodeI(nkWhileStmt, body.info, 2)
-  whileLoop.sons[0] = newIntTypeNode(1, getSysType(g, body.info, tyBool))
-  whileLoop.sons[1] = loopBody
+  whileLoop[0] = newIntTypeNode(1, getSysType(g, body.info, tyBool))
+  whileLoop[1] = loopBody
   result.add whileLoop
 
   # setup loopBody:
   # gather vars in a tuple:
   var v2 = newNodeI(nkLetSection, body.info)
-  var vpart = newNodeI(if L == 3: nkIdentDefs else: nkVarTuple, body.info)
-  for i in 0 .. L-3:
+  var vpart = newNodeI(if body.len == 3: nkIdentDefs else: nkVarTuple, body.info)
+  for i in 0..<body.len-2:
     if body[i].kind == nkSym:
-      body[i].sym.kind = skLet
-    addSon(vpart, body[i])
+      body[i].sym.transitionToLet()
+    vpart.add body[i]
 
-  addSon(vpart, newNodeI(nkEmpty, body.info)) # no explicit type
+  vpart.add newNodeI(nkEmpty, body.info) # no explicit type
   if not env.isNil:
-    call.sons[0] = makeClosure(g, call.sons[0].sym, env.newSymNode, body.info)
-  addSon(vpart, call)
-  addSon(v2, vpart)
+    call[0] = makeClosure(g, call[0].sym, env.newSymNode, body.info)
+  vpart.add call
+  v2.add vpart
 
-  loopBody.sons[0] = v2
+  loopBody[0] = v2
   var bs = newNodeI(nkBreakState, body.info)
-  bs.addSon(call.sons[0])
+  bs.add call[0]
 
   let ibs = newNodeI(nkIfStmt, body.info)
   let elifBranch = newNodeI(nkElifBranch, body.info)
@@ -973,5 +987,5 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
   elifBranch.add(br)
   ibs.add(elifBranch)
 
-  loopBody.sons[1] = ibs
-  loopBody.sons[2] = body[L-1]
+  loopBody[1] = ibs
+  loopBody[2] = body[^1]

compiler/layouter.nim

@@ -21,7 +21,7 @@ type
   SemicolonKind = enum
     detectSemicolonKind, useSemicolon, dontTouch
 
-  LayoutToken = enum
+  LayoutToken* = enum
     ltSpaces,
     ltCrucialNewline, ## a semantically crucial newline (indentation!)
     ltSplittingNewline, ## newline used for splitting up long
@@ -35,14 +35,14 @@ type
   Emitter* = object
     config: ConfigRef
     fid: FileIndex
-    lastTok: TTokType
+    lastTok: TokType
     inquote, lastTokWasTerse: bool
     semicolons: SemicolonKind
     col, lastLineNumber, lineSpan, indentLevel, indWidth*, inSection: int
     keepIndents*: int
     doIndentMore*: int
-    kinds: seq[LayoutToken]
-    tokens: seq[string]
+    kinds*: seq[LayoutToken]
+    tokens*: seq[string]
     indentStack: seq[int]
     fixedUntil: int # marks where we must not go in the content
     altSplitPos: array[SplitKind, int] # alternative split positions
@@ -146,7 +146,7 @@ proc optionalIsGood(em: var Emitter; pos, currentLen: int): bool =
 
 proc lenOfNextTokens(em: Emitter; pos: int): int =
   result = 0
-  for i in 1 ..< em.tokens.len-pos:
+  for i in 1..<em.tokens.len-pos:
     if em.kinds[pos+i] in {ltCrucialNewline, ltSplittingNewline, ltOptionalNewline}: break
     inc result, em.tokens[pos+i].len
 
@@ -160,13 +160,11 @@ proc guidingInd(em: Emitter; pos: int): int =
     inc i
   result = -1
 
-proc closeEmitter*(em: var Emitter) =
+proc renderTokens*(em: var Emitter): string =
+  ## Render Emitter tokens to a string of code
   template defaultCase() =
     content.add em.tokens[i]
     inc lineLen, em.tokens[i].len
-
-  let outFile = em.config.absOutFile
-
   var content = newStringOfCap(16_000)
   var maxLhs = 0
   var lineLen = 0
@@ -243,6 +241,11 @@ proc closeEmitter*(em: var Emitter) =
       defaultCase()
     inc i
 
+  return content
+
+proc writeOut*(em: Emitter, content: string)  =
+  ## Write to disk
+  let outFile = em.config.absOutFile
   if fileExists(outFile) and readFile(outFile.string) == content:
     discard "do nothing, see #9499"
     return
@@ -253,6 +256,11 @@ proc closeEmitter*(em: var Emitter) =
   f.llStreamWrite content
   llStreamClose(f)
 
+proc closeEmitter*(em: var Emitter) =
+  ## Renders emitter tokens and write to a file
+  let content = renderTokens(em)
+  em.writeOut(content)
+
 proc wr(em: var Emitter; x: string; lt: LayoutToken) =
   em.tokens.add x
   em.kinds.add lt
@@ -319,7 +327,7 @@ const
 
   splitters = openPars + {tkComma, tkSemiColon} # do not add 'tkColon' here!
   oprSet = {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
-            tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor}
+            tkIsnot, tkNot, tkOf, tkAs, tkFrom, tkDotDot, tkAnd, tkOr, tkXor}
 
 template goodCol(col): bool = col >= em.maxLineLen div 2
 
@@ -394,7 +402,7 @@ proc endsInAlpha(em: Emitter): bool =
   while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection}: dec(i)
   result = if i >= 0: em.tokens[i].lastChar in SymChars+{'_'} else: false
 
-proc emitComment(em: var Emitter; tok: TToken; dontIndent: bool) =
+proc emitComment(em: var Emitter; tok: Token; dontIndent: bool) =
   var col = em.col
   let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA, tok.commentOffsetB)
   em.lineSpan = countNewlines(lit)
@@ -409,7 +417,7 @@ proc emitComment(em: var Emitter; tok: TToken; dontIndent: bool) =
       inc col
     emitMultilineComment(em, lit, col, dontIndent)
 
-proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
+proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
   template wasExportMarker(em): bool =
     em.kinds.len > 0 and em.kinds[^1] == ltExportMarker
 
@@ -444,6 +452,9 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
       if tok.tokType in openPars and tok.indent > em.indentStack[^1]:
         while em.indentStack[^1] < tok.indent:
           em.indentStack.add(em.indentStack[^1] + em.indWidth)
+      while em.indentStack[^1] > tok.indent:
+        discard em.indentStack.pop()
+
       # aka: we are in an expression context:
       let alignment = max(tok.indent - em.indentStack[^1], 0)
       em.indentLevel = alignment + em.indentStack.high * em.indWidth
@@ -483,7 +494,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
       wrSpace em
 
     if not em.inquote:
-      wr(em, TokTypeToStr[tok.tokType], ltKeyword)
+      wr(em, $tok.tokType, ltKeyword)
       if tok.tokType in {tkAnd, tkOr, tkIn, tkNotin}:
         rememberSplit(splitIn)
         wrSpace em
@@ -492,28 +503,28 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
       wr(em, tok.ident.s, ltIdent)
 
   of tkColon:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     wrSpace em
   of tkSemiColon, tkComma:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     rememberSplit(splitComma)
     wrSpace em
   of openPars:
     if tok.strongSpaceA > 0 and not em.endsInWhite and
         (not em.wasExportMarker or tok.tokType == tkCurlyDotLe):
       wrSpace em
-    wr(em, TokTypeToStr[tok.tokType], ltSomeParLe)
+    wr(em, $tok.tokType, ltSomeParLe)
     rememberSplit(splitParLe)
   of closedPars:
-    wr(em, TokTypeToStr[tok.tokType], ltSomeParRi)
+    wr(em, $tok.tokType, ltSomeParRi)
   of tkColonColon:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
   of tkDot:
     lastTokWasTerse = true
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
   of tkEquals:
     if not em.inquote and not em.endsInWhite: wrSpace(em)
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     if not em.inquote: wrSpace(em)
   of tkOpr, tkDotDot:
     if em.inquote or ((tok.strongSpaceA == 0 and tok.strongSpaceB == 0) and
@@ -533,7 +544,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
         wrSpace(em)
   of tkAccent:
     if not em.inquote and endsInAlpha(em): wrSpace(em)
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     em.inquote = not em.inquote
   of tkComment:
     if not preventComment:
@@ -562,7 +573,7 @@ proc endsWith(em: Emitter; k: varargs[string]): bool =
   return true
 
 proc rfind(em: Emitter, t: string): int =
-  for i in 1 .. 5:
+  for i in 1..5:
     if em.tokens[^i] == t:
       return i
 

compiler/lexer.nim

@@ -30,41 +30,49 @@ const
 # don't forget to update the 'highlite' module if these charsets should change
 
 type
-  TTokType* = enum
-    tkInvalid, tkEof,         # order is important here!
-    tkSymbol, # keywords:
-    tkAddr, tkAnd, tkAs, tkAsm,
-    tkBind, tkBlock, tkBreak, tkCase, tkCast,
-    tkConcept, tkConst, tkContinue, tkConverter,
-    tkDefer, tkDiscard, tkDistinct, tkDiv, tkDo,
-    tkElif, tkElse, tkEnd, tkEnum, tkExcept, tkExport,
-    tkFinally, tkFor, tkFrom, tkFunc,
-    tkIf, tkImport, tkIn, tkInclude, tkInterface,
-    tkIs, tkIsnot, tkIterator,
-    tkLet,
-    tkMacro, tkMethod, tkMixin, tkMod, tkNil, tkNot, tkNotin,
-    tkObject, tkOf, tkOr, tkOut,
-    tkProc, tkPtr, tkRaise, tkRef, tkReturn,
-    tkShl, tkShr, tkStatic,
-    tkTemplate,
-    tkTry, tkTuple, tkType, tkUsing,
-    tkVar, tkWhen, tkWhile, tkXor,
-    tkYield, # end of keywords
-    tkIntLit, tkInt8Lit, tkInt16Lit, tkInt32Lit, tkInt64Lit,
-    tkUIntLit, tkUInt8Lit, tkUInt16Lit, tkUInt32Lit, tkUInt64Lit,
-    tkFloatLit, tkFloat32Lit, tkFloat64Lit, tkFloat128Lit,
-    tkStrLit, tkRStrLit, tkTripleStrLit,
-    tkGStrLit, tkGTripleStrLit, tkCharLit, tkParLe, tkParRi, tkBracketLe,
-    tkBracketRi, tkCurlyLe, tkCurlyRi,
-    tkBracketDotLe, tkBracketDotRi, # [. and  .]
-    tkCurlyDotLe, tkCurlyDotRi, # {.  and  .}
-    tkParDotLe, tkParDotRi,   # (. and .)
-    tkComma, tkSemiColon,
-    tkColon, tkColonColon, tkEquals, tkDot, tkDotDot, tkBracketLeColon,
-    tkOpr, tkComment, tkAccent,
+  TokType* = enum
+    tkInvalid = "tkInvalid", tkEof = "[EOF]", # order is important here!
+    tkSymbol = "tkSymbol", # keywords:
+    tkAddr = "addr", tkAnd = "and", tkAs = "as", tkAsm = "asm",
+    tkBind = "bind", tkBlock = "block", tkBreak = "break", tkCase = "case", tkCast = "cast",
+    tkConcept = "concept", tkConst = "const", tkContinue = "continue", tkConverter = "converter",
+    tkDefer = "defer", tkDiscard = "discard", tkDistinct = "distinct", tkDiv = "div", tkDo = "do",
+    tkElif = "elif", tkElse = "else", tkEnd = "end", tkEnum = "enum", tkExcept = "except", tkExport = "export",
+    tkFinally = "finally", tkFor = "for", tkFrom = "from", tkFunc = "func",
+    tkIf = "if", tkImport = "import", tkIn = "in", tkInclude = "include", tkInterface = "interface",
+    tkIs = "is", tkIsnot = "isnot", tkIterator = "iterator",
+    tkLet = "let",
+    tkMacro = "macro", tkMethod = "method", tkMixin = "mixin", tkMod = "mod", tkNil = "nil", tkNot = "not", tkNotin = "notin",
+    tkObject = "object", tkOf = "of", tkOr = "or", tkOut = "out",
+    tkProc = "proc", tkPtr = "ptr", tkRaise = "raise", tkRef = "ref", tkReturn = "return",
+    tkShl = "shl", tkShr = "shr", tkStatic = "static",
+    tkTemplate = "template",
+    tkTry = "try", tkTuple = "tuple", tkType = "type", tkUsing = "using",
+    tkVar = "var", tkWhen = "when", tkWhile = "while", tkXor = "xor",
+    tkYield = "yield", # end of keywords
+
+    tkIntLit = "tkIntLit", tkInt8Lit = "tkInt8Lit", tkInt16Lit = "tkInt16Lit", 
+    tkInt32Lit = "tkInt32Lit", tkInt64Lit = "tkInt64Lit",
+    tkUIntLit = "tkUIntLit", tkUInt8Lit = "tkUInt8Lit", tkUInt16Lit = "tkUInt16Lit", 
+    tkUInt32Lit = "tkUInt32Lit", tkUInt64Lit = "tkUInt64Lit",
+    tkFloatLit = "tkFloatLit", tkFloat32Lit = "tkFloat32Lit",
+    tkFloat64Lit = "tkFloat64Lit", tkFloat128Lit = "tkFloat128Lit",
+    tkStrLit = "tkStrLit", tkRStrLit = "tkRStrLit", tkTripleStrLit = "tkTripleStrLit",
+    tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit", 
+    
+    tkParLe = "(", tkParRi = ")", tkBracketLe = "[",
+    tkBracketRi = "]", tkCurlyLe = "{", tkCurlyRi = "}",
+    tkBracketDotLe = "[.", tkBracketDotRi = ".]",
+    tkCurlyDotLe = "{.", tkCurlyDotRi = ".}",
+    tkParDotLe = "(.", tkParDotRi = ".)",
+    tkComma = ",", tkSemiColon = ";",
+    tkColon = ":", tkColonColon = "::", tkEquals = "=", 
+    tkDot = ".", tkDotDot = "..", tkBracketLeColon = "[:",
+    tkOpr, tkComment, tkAccent = "`",
+    # these are fake tokens used by renderer.nim
     tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
 
-  TTokTypes* = set[TTokType]
+  TokTypes* = set[TokType]
 
 const
   weakTokens = {tkComma, tkSemiColon, tkColon,
@@ -73,53 +81,21 @@ const
     # tokens that should not be considered for previousToken
   tokKeywordLow* = succ(tkSymbol)
   tokKeywordHigh* = pred(tkIntLit)
-  TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]",
-    "tkSymbol",
-    "addr", "and", "as", "asm",
-    "bind", "block", "break", "case", "cast",
-    "concept", "const", "continue", "converter",
-    "defer", "discard", "distinct", "div", "do",
-    "elif", "else", "end", "enum", "except", "export",
-    "finally", "for", "from", "func", "if",
-    "import", "in", "include", "interface", "is", "isnot", "iterator",
-    "let",
-    "macro", "method", "mixin", "mod",
-    "nil", "not", "notin", "object", "of", "or",
-    "out", "proc", "ptr", "raise", "ref", "return",
-    "shl", "shr", "static",
-    "template",
-    "try", "tuple", "type", "using",
-    "var", "when", "while", "xor",
-    "yield",
-    "tkIntLit", "tkInt8Lit", "tkInt16Lit", "tkInt32Lit", "tkInt64Lit",
-    "tkUIntLit", "tkUInt8Lit", "tkUInt16Lit", "tkUInt32Lit", "tkUInt64Lit",
-    "tkFloatLit", "tkFloat32Lit", "tkFloat64Lit", "tkFloat128Lit",
-    "tkStrLit", "tkRStrLit",
-    "tkTripleStrLit", "tkGStrLit", "tkGTripleStrLit", "tkCharLit", "(",
-    ")", "[", "]", "{", "}", "[.", ".]", "{.", ".}", "(.", ".)",
-    ",", ";",
-    ":", "::", "=", ".", "..", "[:",
-    "tkOpr", "tkComment", "`",
-    "tkSpaces", "tkInfixOpr",
-    "tkPrefixOpr", "tkPostfixOpr"]
 
 type
-  TNumericalBase* = enum
+  NumericalBase* = enum
     base10,                   # base10 is listed as the first element,
                               # so that it is the correct default value
     base2, base8, base16
 
-  CursorPosition* {.pure.} = enum ## XXX remove this again
-    None, InToken, BeforeToken, AfterToken
-
-  TToken* = object            # a Nim token
-    tokType*: TTokType        # the type of the token
+  Token* = object             # a Nim token
+    tokType*: TokType         # the type of the token
     indent*: int              # the indentation; != -1 if the token has been
                               # preceded with indentation
     ident*: PIdent            # the parsed identifier
     iNumber*: BiggestInt      # the parsed integer literal
     fNumber*: BiggestFloat    # the parsed floating point literal
-    base*: TNumericalBase     # the numerical base; only valid for int
+    base*: NumericalBase      # the numerical base; only valid for int
                               # or float literals
     strongSpaceA*: int8       # leading spaces of an operator
     strongSpaceB*: int8       # trailing spaces of an operator
@@ -127,26 +103,25 @@ type
                               # documentation comments are here too
     line*, col*: int
     when defined(nimpretty):
-      offsetA*, offsetB*: int   # used for pretty printing so that literals
-                                # like 0b01 or  r"\L" are unaffected
+      offsetA*, offsetB*: int # used for pretty printing so that literals
+                              # like 0b01 or  r"\L" are unaffected
       commentOffsetA*, commentOffsetB*: int
 
-  TErrorHandler* = proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string)
-  TLexer* = object of TBaseLexer
+  ErrorHandler* = proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string)
+  Lexer* = object of TBaseLexer
     fileIdx*: FileIndex
     indentAhead*: int         # if > 0 an indentation has already been read
                               # this is needed because scanning comments
                               # needs so much look-ahead
     currLineIndent*: int
     strongSpaces*, allowTabs*: bool
-    cursor*: CursorPosition
-    errorHandler*: TErrorHandler
+    errorHandler*: ErrorHandler
     cache*: IdentCache
     when defined(nimsuggest):
       previousToken: TLineInfo
     config*: ConfigRef
 
-proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
+proc getLineInfo*(L: Lexer, tok: Token): TLineInfo {.inline.} =
   result = newLineInfo(L.fileIdx, tok.line, tok.col)
   when defined(nimpretty):
     result.offsetA = tok.offsetA
@@ -154,8 +129,8 @@ proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
     result.commentOffsetA = tok.commentOffsetA
     result.commentOffsetB = tok.commentOffsetB
 
-proc isKeyword*(kind: TTokType): bool =
-  result = (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)
+proc isKeyword*(kind: TokType): bool =
+  (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)
 
 template ones(n): untyped = ((1 shl n)-1) # for utf-8 conversion
 
@@ -169,28 +144,26 @@ proc isNimIdentifier*(s: string): bool =
       inc(i)
     result = true
 
-proc `$`*(tok: TToken): string =
+proc `$`*(tok: Token): string =
   case tok.tokType
-  of tkIntLit..tkInt64Lit: result = $tok.iNumber
-  of tkFloatLit..tkFloat64Lit: result = $tok.fNumber
-  of tkInvalid, tkStrLit..tkCharLit, tkComment: result = tok.literal
-  of tkParLe..tkColon, tkEof, tkAccent:
-    result = TokTypeToStr[tok.tokType]
+  of tkIntLit..tkInt64Lit: $tok.iNumber
+  of tkFloatLit..tkFloat64Lit: $tok.fNumber
+  of tkInvalid, tkStrLit..tkCharLit, tkComment: tok.literal
+  of tkParLe..tkColon, tkEof, tkAccent: $tok.tokType
   else:
     if tok.ident != nil:
-      result = tok.ident.s
+      tok.ident.s
     else:
-      result = ""
+      ""
 
-proc prettyTok*(tok: TToken): string =
-  if isKeyword(tok.tokType): result = "keyword " & tok.ident.s
-  else: result = $tok
+proc prettyTok*(tok: Token): string =
+  if isKeyword(tok.tokType): "keyword " & tok.ident.s
+  else: $tok
 
-proc printTok*(conf: ConfigRef; tok: TToken) =
-  msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" &
-      TokTypeToStr[tok.tokType] & " " & $tok)
+proc printTok*(conf: ConfigRef; tok: Token) =
+  msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" & $tok.tokType & " " & $tok)
 
-proc initToken*(L: var TToken) =
+proc initToken*(L: var Token) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
@@ -203,7 +176,7 @@ proc initToken*(L: var TToken) =
     L.commentOffsetA = 0
     L.commentOffsetB = 0
 
-proc fillToken(L: var TToken) =
+proc fillToken(L: var Token) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
@@ -216,7 +189,7 @@ proc fillToken(L: var TToken) =
     L.commentOffsetA = 0
     L.commentOffsetB = 0
 
-proc openLexer*(lex: var TLexer, fileIdx: FileIndex, inputstream: PLLStream;
+proc openLexer*(lex: var Lexer, fileIdx: FileIndex, inputstream: PLLStream;
                  cache: IdentCache; config: ConfigRef) =
   openBaseLexer(lex, inputstream)
   lex.fileIdx = fileIdx
@@ -228,36 +201,36 @@ proc openLexer*(lex: var TLexer, fileIdx: FileIndex, inputstream: PLLStream;
     lex.previousToken.fileIndex = fileIdx
   lex.config = config
 
-proc openLexer*(lex: var TLexer, filename: AbsoluteFile, inputstream: PLLStream;
+proc openLexer*(lex: var Lexer, filename: AbsoluteFile, inputstream: PLLStream;
                 cache: IdentCache; config: ConfigRef) =
   openLexer(lex, fileInfoIdx(config, filename), inputstream, cache, config)
 
-proc closeLexer*(lex: var TLexer) =
+proc closeLexer*(lex: var Lexer) =
   if lex.config != nil:
     inc(lex.config.linesCompiled, lex.lineNumber)
   closeBaseLexer(lex)
 
-proc getLineInfo(L: TLexer): TLineInfo =
+proc getLineInfo(L: Lexer): TLineInfo =
   result = newLineInfo(L.fileIdx, L.lineNumber, getColNumber(L, L.bufpos))
 
-proc dispMessage(L: TLexer; info: TLineInfo; msg: TMsgKind; arg: string) =
+proc dispMessage(L: Lexer; info: TLineInfo; msg: TMsgKind; arg: string) =
   if L.errorHandler.isNil:
     msgs.message(L.config, info, msg, arg)
   else:
     L.errorHandler(L.config, info, msg, arg)
 
-proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "") =
+proc lexMessage*(L: Lexer, msg: TMsgKind, arg = "") =
   L.dispMessage(getLineInfo(L), msg, arg)
 
-proc lexMessageTok*(L: TLexer, msg: TMsgKind, tok: TToken, arg = "") =
+proc lexMessageTok*(L: Lexer, msg: TMsgKind, tok: Token, arg = "") =
   var info = newLineInfo(L.fileIdx, tok.line, tok.col)
   L.dispMessage(info, msg, arg)
 
-proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
+proc lexMessagePos(L: var Lexer, msg: TMsgKind, pos: int, arg = "") =
   var info = newLineInfo(L.fileIdx, L.lineNumber, pos - L.lineStart)
   L.dispMessage(info, msg, arg)
 
-proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
+proc matchTwoChars(L: Lexer, first: char, second: set[char]): bool =
   result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
 
 template tokenBegin(tok, pos) {.dirty.} =
@@ -271,7 +244,6 @@ template tokenEnd(tok, pos) {.dirty.} =
     let colB = getColNumber(L, pos)+1
     if L.fileIdx == L.config.m.trackPos.fileIndex and L.config.m.trackPos.col in colA..colB and
         L.lineNumber == L.config.m.trackPos.line.int and L.config.ideCmd in {ideSug, ideCon}:
-      L.cursor = CursorPosition.InToken
       L.config.m.trackPos.col = colA.int16
     colA = 0
   when defined(nimpretty):
@@ -296,28 +268,27 @@ template tokenEndPrevious(tok, pos) =
     let colB = getColNumber(L, pos)
     if L.fileIdx == L.config.m.trackPos.fileIndex and L.config.m.trackPos.col in colA..colB and
         L.lineNumber == L.config.m.trackPos.line.int and L.config.ideCmd in {ideSug, ideCon}:
-      L.cursor = CursorPosition.BeforeToken
       L.config.m.trackPos = L.previousToken
       L.config.m.trackPosAttached = true
     colA = 0
   when defined(nimpretty):
     tok.offsetB = L.offsetBase + pos
 
-template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
-  add(t.literal, replacementChar)
+template eatChar(L: var Lexer, t: var Token, replacementChar: char) =
+  t.literal.add(replacementChar)
   inc(L.bufpos)
 
-template eatChar(L: var TLexer, t: var TToken) =
-  add(t.literal, L.buf[L.bufpos])
+template eatChar(L: var Lexer, t: var Token) =
+  t.literal.add(L.buf[L.bufpos])
   inc(L.bufpos)
 
-proc getNumber(L: var TLexer, result: var TToken) =
-  proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]): Natural =
+proc getNumber(L: var Lexer, result: var Token) =
+  proc matchUnderscoreChars(L: var Lexer, tok: var Token, chars: set[char]): Natural =
     var pos = L.bufpos              # use registers for pos, buf
     result = 0
     while true:
       if L.buf[pos] in chars:
-        add(tok.literal, L.buf[pos])
+        tok.literal.add(L.buf[pos])
         inc(pos)
         inc(result)
       else:
@@ -328,35 +299,35 @@ proc getNumber(L: var TLexer, result: var TToken) =
             "only single underscores may occur in a token and token may not " &
             "end with an underscore: e.g. '1__1' and '1_' are invalid")
           break
-        add(tok.literal, '_')
+        tok.literal.add('_')
         inc(pos)
     L.bufpos = pos
 
-  proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
+  proc matchChars(L: var Lexer, tok: var Token, chars: set[char]) =
     var pos = L.bufpos              # use registers for pos, buf
     while L.buf[pos] in chars:
-      add(tok.literal, L.buf[pos])
+      tok.literal.add(L.buf[pos])
       inc(pos)
     L.bufpos = pos
 
-  proc lexMessageLitNum(L: var TLexer, msg: string, startpos: int, msgKind = errGenerated) =
+  proc lexMessageLitNum(L: var Lexer, msg: string, startpos: int, msgKind = errGenerated) =
     # Used to get slightly human friendlier err messages.
     const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O',
       'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
     var msgPos = L.bufpos
-    var t: TToken
+    var t: Token
     t.literal = ""
     L.bufpos = startpos # Use L.bufpos as pos because of matchChars
     matchChars(L, t, literalishChars)
     # We must verify +/- specifically so that we're not past the literal
     if  L.buf[L.bufpos] in {'+', '-'} and
         L.buf[L.bufpos - 1] in {'e', 'E'}:
-      add(t.literal, L.buf[L.bufpos])
+      t.literal.add(L.buf[L.bufpos])
       inc(L.bufpos)
       matchChars(L, t, literalishChars)
     if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
       inc(L.bufpos)
-      add(t.literal, L.buf[L.bufpos])
+      t.literal.add(L.buf[L.bufpos])
       matchChars(L, t, {'0'..'9'})
     L.bufpos = msgPos
     lexMessage(L, msgKind, msg % t.literal)
@@ -571,7 +542,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
         try:
           len = parseBiggestUInt(result.literal, iNumber)
         except ValueError:
-          raise newException(OverflowError, "number out of range: " & $result.literal)
+          raise newException(OverflowDefect, "number out of range: " & $result.literal)
         if len != result.literal.len:
           raise newException(ValueError, "invalid integer: " & $result.literal)
         result.iNumber = cast[int64](iNumber)
@@ -581,7 +552,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
         try:
           len = parseBiggestInt(result.literal, iNumber)
         except ValueError:
-          raise newException(OverflowError, "number out of range: " & $result.literal)
+          raise newException(OverflowDefect, "number out of range: " & $result.literal)
         if len != result.literal.len:
           raise newException(ValueError, "invalid integer: " & $result.literal)
         result.iNumber = iNumber
@@ -607,12 +578,12 @@ proc getNumber(L: var TLexer, result: var TToken) =
 
   except ValueError:
     lexMessageLitNum(L, "invalid number: '$1'", startpos)
-  except OverflowError, RangeError:
+  except OverflowDefect, RangeDefect:
     lexMessageLitNum(L, "number out of range: '$1'", startpos)
   tokenEnd(result, postPos-1)
   L.bufpos = postPos
 
-proc handleHexChar(L: var TLexer, xi: var int; position: range[0..4]) =
+proc handleHexChar(L: var Lexer, xi: var int; position: range[0..4]) =
   template invalid() =
     lexMessage(L, errGenerated,
       "expected a hex digit, but found: " & L.buf[L.bufpos] &
@@ -637,7 +608,7 @@ proc handleHexChar(L: var TLexer, xi: var int; position: range[0..4]) =
     # Need to progress for `nim check`
     inc(L.bufpos)
 
-proc handleDecChars(L: var TLexer, xi: var int) =
+proc handleDecChars(L: var Lexer, xi: var int) =
   while L.buf[L.bufpos] in {'0'..'9'}:
     xi = (xi * 10) + (ord(L.buf[L.bufpos]) - ord('0'))
     inc(L.bufpos)
@@ -680,58 +651,53 @@ proc addUnicodeCodePoint(s: var string, i: int) =
     s[pos+4] = chr(i shr 6 and ones(6) or 0b10_0000_00)
     s[pos+5] = chr(i and ones(6) or 0b10_0000_00)
 
-proc getEscapedChar(L: var TLexer, tok: var TToken) =
+proc getEscapedChar(L: var Lexer, tok: var Token) =
   inc(L.bufpos)               # skip '\'
   case L.buf[L.bufpos]
   of 'n', 'N':
-    if L.config.oldNewlines:
-      if tok.tokType == tkCharLit:
-        lexMessage(L, errGenerated, "\\n not allowed in character literal")
-      add(tok.literal, L.config.target.tnl)
-    else:
-      add(tok.literal, '\L')
+    tok.literal.add('\L')
     inc(L.bufpos)
   of 'p', 'P':
     if tok.tokType == tkCharLit:
       lexMessage(L, errGenerated, "\\p not allowed in character literal")
-    add(tok.literal, L.config.target.tnl)
+    tok.literal.add(L.config.target.tnl)
     inc(L.bufpos)
   of 'r', 'R', 'c', 'C':
-    add(tok.literal, CR)
+    tok.literal.add(CR)
     inc(L.bufpos)
   of 'l', 'L':
-    add(tok.literal, LF)
+    tok.literal.add(LF)
     inc(L.bufpos)
   of 'f', 'F':
-    add(tok.literal, FF)
+    tok.literal.add(FF)
     inc(L.bufpos)
   of 'e', 'E':
-    add(tok.literal, ESC)
+    tok.literal.add(ESC)
     inc(L.bufpos)
   of 'a', 'A':
-    add(tok.literal, BEL)
+    tok.literal.add(BEL)
     inc(L.bufpos)
   of 'b', 'B':
-    add(tok.literal, BACKSPACE)
+    tok.literal.add(BACKSPACE)
     inc(L.bufpos)
   of 'v', 'V':
-    add(tok.literal, VT)
+    tok.literal.add(VT)
     inc(L.bufpos)
   of 't', 'T':
-    add(tok.literal, '\t')
+    tok.literal.add('\t')
     inc(L.bufpos)
   of '\'', '\"':
-    add(tok.literal, L.buf[L.bufpos])
+    tok.literal.add(L.buf[L.bufpos])
     inc(L.bufpos)
   of '\\':
-    add(tok.literal, '\\')
+    tok.literal.add('\\')
     inc(L.bufpos)
   of 'x', 'X':
     inc(L.bufpos)
     var xi = 0
     handleHexChar(L, xi, 1)
     handleHexChar(L, xi, 2)
-    add(tok.literal, chr(xi))
+    tok.literal.add(chr(xi))
   of 'u', 'U':
     if tok.tokType == tkCharLit:
       lexMessage(L, errGenerated, "\\u not allowed in character literal")
@@ -761,22 +727,17 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
       lexMessage(L, warnOctalEscape)
     var xi = 0
     handleDecChars(L, xi)
-    if (xi <= 255): add(tok.literal, chr(xi))
+    if (xi <= 255): tok.literal.add(chr(xi))
     else: lexMessage(L, errGenerated, "invalid character constant")
   else: lexMessage(L, errGenerated, "invalid character constant")
 
-proc newString(s: cstring, len: int): string =
-  ## XXX, how come there is no support for this?
-  result = newString(len)
-  for i in 0 ..< len:
-    result[i] = s[i]
-
-proc handleCRLF(L: var TLexer, pos: int): int =
+proc handleCRLF(L: var Lexer, pos: int): int =
   template registerLine =
     let col = L.getColNumber(pos)
 
-    if col > MaxLineLength:
-      lexMessagePos(L, hintLineTooLong, pos)
+    when not defined(nimpretty):
+      if col > MaxLineLength:
+        lexMessagePos(L, hintLineTooLong, pos)
 
   case L.buf[pos]
   of CR:
@@ -793,7 +754,7 @@ type
     raw,
     generalized
 
-proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
+proc getString(L: var Lexer, tok: var Token, mode: StringMode) =
   var pos = L.bufpos
   var line = L.lineNumber         # save linenumber for better error message
   tokenBegin(tok, pos - ord(mode == raw))
@@ -815,12 +776,12 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
           tokenEndIgnore(tok, pos+2)
           L.bufpos = pos + 3 # skip the three """
           break
-        add(tok.literal, '\"')
+        tok.literal.add('\"')
         inc(pos)
       of CR, LF:
         tokenEndIgnore(tok, pos)
         pos = handleCRLF(L, pos)
-        add(tok.literal, "\n")
+        tok.literal.add("\n")
       of nimlexbase.EndOfFile:
         tokenEndIgnore(tok, pos)
         var line2 = L.lineNumber
@@ -830,7 +791,7 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
         L.bufpos = pos
         break
       else:
-        add(tok.literal, L.buf[pos])
+        tok.literal.add(L.buf[pos])
         inc(pos)
   else:
     # ordinary string literal
@@ -841,7 +802,7 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
       if c == '\"':
         if mode != normal and L.buf[pos+1] == '\"':
           inc(pos, 2)
-          add(tok.literal, '"')
+          tok.literal.add('"')
         else:
           tokenEndIgnore(tok, pos)
           inc(pos) # skip '"'
@@ -855,11 +816,11 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
         getEscapedChar(L, tok)
         pos = L.bufpos
       else:
-        add(tok.literal, c)
+        tok.literal.add(c)
         inc(pos)
     L.bufpos = pos
 
-proc getCharacter(L: var TLexer, tok: var TToken) =
+proc getCharacter(L: var Lexer, tok: var Token) =
   tokenBegin(tok, L.bufpos)
   inc(L.bufpos)               # skip '
   var c = L.buf[L.bufpos]
@@ -876,7 +837,7 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
   tokenEndIgnore(tok, L.bufpos)
   inc(L.bufpos)               # skip '
 
-proc getSymbol(L: var TLexer, tok: var TToken) =
+proc getSymbol(L: var Lexer, tok: var Token) =
   var h: Hash = 0
   var pos = L.bufpos
   tokenBegin(tok, pos)
@@ -906,21 +867,21 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
       (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
     tok.tokType = tkSymbol
   else:
-    tok.tokType = TTokType(tok.ident.id + ord(tkSymbol))
+    tok.tokType = TokType(tok.ident.id + ord(tkSymbol))
     if suspicious and {optStyleHint, optStyleError} * L.config.globalOptions != {}:
       lintReport(L.config, getLineInfo(L), tok.ident.s.normalize, tok.ident.s)
   L.bufpos = pos
 
 
-proc endOperator(L: var TLexer, tok: var TToken, pos: int,
+proc endOperator(L: var Lexer, tok: var Token, pos: int,
                  hash: Hash) {.inline.} =
   var h = !$hash
   tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
   if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr
-  else: tok.tokType = TTokType(tok.ident.id - oprLow + ord(tkColon))
+  else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon))
   L.bufpos = pos
 
-proc getOperator(L: var TLexer, tok: var TToken) =
+proc getOperator(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tokenBegin(tok, pos)
   var h: Hash = 0
@@ -940,22 +901,18 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   if L.buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
-proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
+proc getPrecedence*(tok: Token): int =
   ## Calculates the precedence of the given token.
-  template considerStrongSpaces(x): untyped =
-    x + (if strongSpaces: 100 - tok.strongSpaceA.int*10 else: 0)
-
   case tok.tokType
   of tkOpr:
-    let L = tok.ident.s.len
     let relevantChar = tok.ident.s[0]
 
     # arrow like?
-    if L > 1 and tok.ident.s[L-1] == '>' and
-      tok.ident.s[L-2] in {'-', '~', '='}: return considerStrongSpaces(1)
+    if tok.ident.s.len > 1 and tok.ident.s[^1] == '>' and
+      tok.ident.s[^2] in {'-', '~', '='}: return 1
 
     template considerAsgn(value: untyped) =
-      result = if tok.ident.s[L-1] == '=': 1 else: value
+      result = if tok.ident.s[^1] == '=': 1 else: value
 
     case relevantChar
     of '$', '^': considerAsgn(10)
@@ -968,15 +925,13 @@ proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
     of '?': result = 2
     else: considerAsgn(2)
   of tkDiv, tkMod, tkShl, tkShr: result = 9
-  of tkIn, tkNotin, tkIs, tkIsnot, tkOf, tkAs: result = 5
   of tkDotDot: result = 6
+  of tkIn, tkNotin, tkIs, tkIsnot, tkOf, tkAs, tkFrom: result = 5
   of tkAnd: result = 4
   of tkOr, tkXor, tkPtr, tkRef: result = 3
   else: return -10
-  result = considerStrongSpaces(result)
 
-
-proc newlineFollows*(L: TLexer): bool =
+proc newlineFollows*(L: Lexer): bool =
   var pos = L.bufpos
   while true:
     case L.buf[pos]
@@ -992,7 +947,7 @@ proc newlineFollows*(L: TLexer): bool =
     else:
       break
 
-proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
+proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
                           isDoc: bool) =
   var pos = start
   var toStrip = 0
@@ -1057,14 +1012,14 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
   when defined(nimpretty):
     tok.commentOffsetB = L.offsetBase + pos - 1
 
-proc scanComment(L: var TLexer, tok: var TToken) =
+proc scanComment(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tok.tokType = tkComment
   # iNumber contains the number of '\n' in the token
   tok.iNumber = 0
   assert L.buf[pos+1] == '#'
   when defined(nimpretty):
-    tok.commentOffsetA = L.offsetBase + pos - 1
+    tok.commentOffsetA = L.offsetBase + pos
 
   if L.buf[pos+2] == '[':
     skipMultiLineComment(L, tok, pos+3, true)
@@ -1081,7 +1036,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
     var lastBackslash = -1
     while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
       if L.buf[pos] == '\\': lastBackslash = pos+1
-      add(tok.literal, L.buf[pos])
+      tok.literal.add(L.buf[pos])
       inc(pos)
     tokenEndIgnore(tok, pos)
     pos = handleCRLF(L, pos)
@@ -1107,7 +1062,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
   when defined(nimpretty):
     tok.commentOffsetB = L.offsetBase + pos - 1
 
-proc skip(L: var TLexer, tok: var TToken) =
+proc skip(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tokenBegin(tok, pos)
   tok.strongSpaceA = 0
@@ -1179,7 +1134,7 @@ proc skip(L: var TLexer, tok: var TToken) =
       tok.tokType = tkComment
       tok.indent = commentIndent
 
-proc rawGetTok*(L: var TLexer, tok: var TToken) =
+proc rawGetTok*(L: var Lexer, tok: var Token) =
   template atTokenEnd() {.dirty.} =
     when defined(nimsuggest):
       # we attach the cursor to the last *strong* token
@@ -1187,8 +1142,6 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
         L.previousToken.line = tok.line.uint16
         L.previousToken.col = tok.col.int16
 
-  when defined(nimsuggest):
-    L.cursor = CursorPosition.None
   fillToken(tok)
   if L.indentAhead >= 0:
     tok.indent = L.indentAhead
@@ -1259,7 +1212,6 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
         if L.fileIdx == L.config.m.trackPos.fileIndex and tok.col+1 == L.config.m.trackPos.col and
             tok.line == L.config.m.trackPos.line.int and L.config.ideCmd == ideSug:
           tok.tokType = tkDot
-          L.cursor = CursorPosition.InToken
           L.config.m.trackPos.col = tok.col.int16
           inc(L.bufpos)
           atTokenEnd()
@@ -1332,8 +1284,8 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
 
 proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
                      cache: IdentCache; config: ConfigRef): int =
-  var lex: TLexer
-  var tok: TToken
+  var lex: Lexer
+  var tok: Token
   initToken(tok)
   openLexer(lex, fileIdx, inputstream, cache, config)
   var prevToken = tkEof
@@ -1347,11 +1299,11 @@ proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
 
 proc getPrecedence*(ident: PIdent): int =
   ## assumes ident is binary operator already
-  var tok: TToken
+  var tok: Token
   initToken(tok)
   tok.ident = ident
   tok.tokType =
-    if tok.ident.id in ord(tokKeywordLow) - ord(tkSymbol) .. ord(tokKeywordHigh) - ord(tkSymbol):
-      TTokType(tok.ident.id + ord(tkSymbol))
+    if tok.ident.id in ord(tokKeywordLow) - ord(tkSymbol)..ord(tokKeywordHigh) - ord(tkSymbol):
+      TokType(tok.ident.id + ord(tkSymbol))
     else: tkOpr
-  getPrecedence(tok, false)
+  getPrecedence(tok)

compiler/liftlocals.nim

@@ -30,10 +30,10 @@ proc lookupOrAdd(c: var Ctx; s: PSym; info: TLineInfo): PNode =
   let field = addUniqueField(c.objType, s, c.cache)
   var deref = newNodeI(nkHiddenDeref, info)
   deref.typ = c.objType
-  add(deref, newSymNode(c.partialParam, info))
+  deref.add(newSymNode(c.partialParam, info))
   result = newNodeI(nkDotExpr, info)
-  add(result, deref)
-  add(result, newSymNode(field))
+  result.add(deref)
+  result.add(newSymNode(field))
   result.typ = field.typ
 
 proc liftLocals(n: PNode; i: int; c: var Ctx) =
@@ -42,14 +42,14 @@ proc liftLocals(n: PNode; i: int; c: var Ctx) =
   of nkSym:
     if interestingVar(it.sym):
       n[i] = lookupOrAdd(c, it.sym, it.info)
-  of procDefs, nkTypeSection: discard
+  of procDefs, nkTypeSection, nkMixinStmt, nkBindStmt: discard
   else:
-    for i in 0 ..< it.safeLen:
+    for i in 0..<it.safeLen:
       liftLocals(it, i, c)
 
 proc lookupParam(params, dest: PNode): PSym =
   if dest.kind != nkIdent: return nil
-  for i in 1 ..< params.len:
+  for i in 1..<params.len:
     if params[i].kind == nkSym and params[i].sym.name.id == dest.ident.id:
       return params[i].sym
 

compiler/lineinfos.nim

@@ -10,10 +10,20 @@
 ## This module contains the ``TMsgKind`` enum as well as the
 ## ``TLineInfo`` object.
 
-import ropes, tables, pathutils
+import ropes, tables, pathutils, hashes
 
 const
-  explanationsBaseUrl* = "https://nim-lang.org/docs"
+  explanationsBaseUrl* = "https://nim-lang.github.io/Nim"
+    # was: "https://nim-lang.org/docs" but we're now usually showing devel docs
+    # instead of latest release docs.
+
+proc createDocLink*(urlSuffix: string): string =
+  # os.`/` is not appropriate for urls.
+  result = explanationsBaseUrl
+  if urlSuffix.len > 0 and urlSuffix[0] == '/':
+    result.add urlSuffix
+  else:
+    result.add "/" & urlSuffix
 
 type
   TMsgKind* = enum
@@ -23,32 +33,43 @@ type
     errGeneralParseError,
     errNewSectionExpected,
     errInvalidDirectiveX,
+    errProveInit, # deadcode
     errGenerated,
     errUser,
-    warnCannotOpenFile,
-    warnOctalEscape, warnXIsNeverRead, warnXmightNotBeenInit,
-    warnDeprecated, warnConfigDeprecated,
-    warnSmallLshouldNotBeUsed, warnUnknownMagic, warnRedefinitionOfLabel,
-    warnUnknownSubstitutionX, warnLanguageXNotSupported,
-    warnFieldXNotSupported, warnCommentXIgnored,
-    warnTypelessParam,
-    warnUseBase, warnWriteToForeignHeap, warnUnsafeCode,
-    warnUnusedImportX,
-    warnEachIdentIsTuple,
-    warnProveInit, warnProveField, warnProveIndex, warnGcUnsafe, warnGcUnsafe2,
-    warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed,
-    warnInconsistentSpacing, warnCaseTransition, warnUser,
-    hintSuccess, hintSuccessX, hintCC,
-    hintLineTooLong, hintXDeclaredButNotUsed,
-    hintConvToBaseNotNeeded,
-    hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled,
-    hintProcessing, hintCodeBegin, hintCodeEnd, hintConf, hintPath,
-    hintConditionAlwaysTrue, hintConditionAlwaysFalse, hintName, hintPattern,
-    hintExecuting, hintLinking, hintDependency,
-    hintSource, hintPerformance, hintStackTrace, hintGCStats,
-    hintGlobalVar, hintExpandMacro,
-    hintUser, hintUserRaw,
-    hintExtendedContext
+
+    warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape",
+    warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit",
+    warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated",
+    warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic",
+    warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX",
+    warnLanguageXNotSupported = "LanguageXNotSupported", warnFieldXNotSupported = "FieldXNotSupported",
+    warnCommentXIgnored = "CommentXIgnored", warnTypelessParam = "TypelessParam",
+    warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap",
+    warnUnsafeCode = "UnsafeCode", warnUnusedImportX = "UnusedImport",
+    warnInheritFromException = "InheritFromException", warnEachIdentIsTuple = "EachIdentIsTuple",
+    warnUnsafeSetLen = "UnsafeSetLen", warnUnsafeDefault = "UnsafeDefault",
+    warnProveInit = "ProveInit", warnProveField = "ProveField", warnProveIndex = "ProveIndex",
+    warnUnreachableElse = "UnreachableElse", warnUnreachableCode = "UnreachableCode",
+    warnStaticIndexCheck = "IndexCheck", warnGcUnsafe = "GcUnsafe", warnGcUnsafe2 = "GcUnsafe2",
+    warnUninit = "Uninit", warnGcMem = "GcMem", warnDestructor = "Destructor",
+    warnLockLevel = "LockLevel", warnResultShadowed = "ResultShadowed",
+    warnInconsistentSpacing = "Spacing",  warnCaseTransition = "CaseTransition",
+    warnCycleCreated = "CycleCreated", warnObservableStores = "ObservableStores",
+    warnResultUsed = "ResultUsed",
+    warnUser = "User",
+
+    hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC",
+    hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed",
+    hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded",
+    hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX",
+    hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintCodeBegin = "CodeBegin",
+    hintCodeEnd = "CodeEnd", hintConf = "Conf", hintPath = "Path",
+    hintConditionAlwaysTrue = "CondTrue", hintConditionAlwaysFalse = "CondFalse", hintName = "Name",
+    hintPattern = "Pattern", hintExecuting = "Exec", hintLinking = "Link", hintDependency = "Dependency",
+    hintSource = "Source", hintPerformance = "Performance", hintStackTrace = "StackTrace",
+    hintGCStats = "GCStats", hintGlobalVar = "GlobalVar", hintExpandMacro = "ExpandMacro",
+    hintUser = "User", hintUserRaw = "UserRaw", hintExtendedContext = "ExtendedContext",
+    hintMsgOrigin = "MsgOrigin", # since 1.3.5
 
 const
   MsgKindToStr*: array[TMsgKind, string] = [
@@ -61,6 +82,7 @@ const
     errGeneralParseError: "general parse error",
     errNewSectionExpected: "new section expected",
     errInvalidDirectiveX: "invalid directive: '$1'",
+    errProveInit: "Cannot prove that '$1' is initialized.",  # deadcode
     errGenerated: "$1",
     errUser: "$1",
     warnCannotOpenFile: "cannot open '$1'",
@@ -81,25 +103,38 @@ const
     warnWriteToForeignHeap: "write to foreign heap",
     warnUnsafeCode: "unsafe code: '$1'",
     warnUnusedImportX: "imported and not used: '$1'",
+    warnInheritFromException: "inherit from a more precise exception type like ValueError, " &
+      "IOError or OSError. If these don't suit, inherit from CatchableError or Defect.",
     warnEachIdentIsTuple: "each identifier is a tuple",
+    warnUnsafeSetLen: "setLen can potentially expand the sequence, " &
+                      "but the element type '$1' doesn't have a valid default value",
+    warnUnsafeDefault: "The '$1' type doesn't have a valid default value",
     warnProveInit: "Cannot prove that '$1' is initialized. This will become a compile time error in the future.",
     warnProveField: "cannot prove that field '$1' is accessible",
     warnProveIndex: "cannot prove index '$1' is valid",
+    warnUnreachableElse: "unreachable else, all cases are already covered",
+    warnUnreachableCode: "unreachable code after 'return' statement or '{.noReturn.}' proc",
+    warnStaticIndexCheck: "$1",
     warnGcUnsafe: "not GC-safe: '$1'",
     warnGcUnsafe2: "$1",
-    warnUninit: "'$1' might not have been initialized",
+    warnUninit: "use explicit initialization of '$1' for clarity",
     warnGcMem: "'$1' uses GC'ed memory",
     warnDestructor: "usage of a type with a destructor in a non destructible context. This will become a compile time error in the future.",
     warnLockLevel: "$1",
     warnResultShadowed: "Special variable 'result' is shadowed.",
     warnInconsistentSpacing: "Number of spaces around '$#' is not consistent",
     warnCaseTransition: "Potential object case transition, instantiate new object instead",
+    warnCycleCreated: "$1",
+    warnObservableStores: "observable stores to '$1'",
+    warnResultUsed: "used 'result' variable",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
-    hintSuccessX: "operation successful ($# lines compiled; $# sec total; $#; $#)",
-    hintCC: "CC: \'$1\'", # unused
+    # keep in sync with `testament.isSuccess`
+    hintSuccessX: "${loc} lines; ${sec}s; $mem; $build build; proj: $project; out: $output",
+    hintCC: "CC: $1",
     hintLineTooLong: "line too long",
     hintXDeclaredButNotUsed: "'$1' is declared but not used",
+    hintXCannotRaiseY: "$1",
     hintConvToBaseNotNeeded: "conversion to base object is not needed",
     hintConvFromXtoItselfNotNeeded: "conversion from $1 to itself is pointless",
     hintExprAlwaysX: "expression evaluates always to '$1'",
@@ -114,7 +149,7 @@ const
     hintName: "$1",
     hintPattern: "$1",
     hintExecuting: "$1",
-    hintLinking: "",
+    hintLinking: "$1",
     hintDependency: "$1",
     hintSource: "$1",
     hintPerformance: "$1",
@@ -125,30 +160,7 @@ const
     hintUser: "$1",
     hintUserRaw: "$1",
     hintExtendedContext: "$1",
-  ]
-
-const
-  WarningsToStr* = ["CannotOpenFile", "OctalEscape",
-    "XIsNeverRead", "XmightNotBeenInit",
-    "Deprecated", "ConfigDeprecated",
-    "SmallLshouldNotBeUsed", "UnknownMagic",
-    "RedefinitionOfLabel", "UnknownSubstitutionX",
-    "LanguageXNotSupported", "FieldXNotSupported",
-    "CommentXIgnored",
-    "TypelessParam", "UseBase", "WriteToForeignHeap",
-    "UnsafeCode", "UnusedImport", "EachIdentIsTuple",
-    "ProveInit", "ProveField", "ProveIndex", "GcUnsafe", "GcUnsafe2", "Uninit",
-    "GcMem", "Destructor", "LockLevel", "ResultShadowed",
-    "Spacing", "CaseTransition", "User"]
-
-  HintsToStr* = [
-    "Success", "SuccessX", "CC", "LineTooLong",
-    "XDeclaredButNotUsed",
-    "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded",
-    "ExprAlwaysX", "QuitCalled", "Processing", "CodeBegin", "CodeEnd", "Conf",
-    "Path", "CondTrue", "CondFalse", "Name", "Pattern", "Exec", "Link", "Dependency",
-    "Source", "Performance", "StackTrace", "GCStats", "GlobalVar", "ExpandMacro",
-    "User", "UserRaw", "ExtendedContext",
+    hintMsgOrigin: "$1",
   ]
 
 const
@@ -161,20 +173,16 @@ const
   hintMin* = hintSuccess
   hintMax* = high(TMsgKind)
 
-static:
-  doAssert HintsToStr.len == ord(hintMax) - ord(hintMin) + 1
-  doAssert WarningsToStr.len == ord(warnMax) - ord(warnMin) + 1
-
 type
   TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints
   TNoteKinds* = set[TNoteKind]
 
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
-  result[3] = {low(TNoteKind)..high(TNoteKind)} - {}
+  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed}
   result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext}
   result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
-    hintSource, hintGlobalVar, hintGCStats}
+    hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin}
   result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf,
     hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC}
 
@@ -225,20 +233,19 @@ type
   TErrorOutputs* = set[TErrorOutput]
 
   ERecoverableError* = object of ValueError
-  ESuggestDone* = object of Exception
+  ESuggestDone* = object of ValueError
 
 proc `==`*(a, b: FileIndex): bool {.borrow.}
 
+proc hash*(i: TLineInfo): Hash =
+  hash (i.line.int, i.col.int, i.fileIndex.int)
+
 proc raiseRecoverableError*(msg: string) {.noinline.} =
   raise newException(ERecoverableError, msg)
 
 const
   InvalidFileIdx* = FileIndex(-1)
-
-proc unknownLineInfo*(): TLineInfo =
-  result.line = uint16(0)
-  result.col = int16(-1)
-  result.fileIndex = InvalidFileIdx
+  unknownLineInfo* = TLineInfo(line: 0, col: -1, fileIndex: InvalidFileIdx)
 
 type
   Severity* {.pure.} = enum ## VS Code only supports these three
@@ -265,7 +272,7 @@ type
 
 proc initMsgConfig*(): MsgConfig =
   result.msgContext = @[]
-  result.lastError = unknownLineInfo()
+  result.lastError = unknownLineInfo
   result.filenameToIndexTbl = initTable[string, FileIndex]()
   result.fileInfos = @[]
   result.errorOutputs = {eStdOut, eStdErr}

compiler/linter.nim

@@ -37,7 +37,7 @@ proc beautifyName(s: string, k: TSymKind): string =
     # Types should start with a capital unless builtins like 'int' etc.:
     if s =~ ["int", "uint", "cint", "cuint", "clong", "cstring", "string",
              "char", "byte", "bool", "openArray", "seq", "array", "void",
-             "pointer", "float", "csize", "cdouble", "cchar", "cschar",
+             "pointer", "float", "csize", "csize_t", "cdouble", "cchar", "cschar",
              "cshort", "cu", "nil", "typedesc", "auto", "any",
              "range", "openarray", "varargs", "set", "cfloat", "ref", "ptr",
              "untyped", "typed", "static", "sink", "lent", "type", "owned"]:
@@ -71,13 +71,9 @@ proc beautifyName(s: string, k: TSymKind): string =
 
 proc differ*(line: string, a, b: int, x: string): string =
   proc substrEq(s: string, pos, last: int, substr: string): bool =
-    var i = 0
-    var length = substr.len
-    while i < length and pos+i <= last and s[pos+i] == substr[i]:
-      inc i
-    return i == length
-
-  let last = min(b, line.len)
+    result = true
+    for i in 0..<substr.len:
+      if pos+i > last or s[pos+i] != substr[i]: return false
 
   result = ""
   if not substrEq(line, a, b, x):
@@ -85,11 +81,6 @@ proc differ*(line: string, a, b: int, x: string): string =
     if cmpIgnoreStyle(y, x) == 0:
       result = y
 
-proc checkStyle(conf: ConfigRef; cache: IdentCache; info: TLineInfo, s: string, k: TSymKind; sym: PSym) =
-  let beau = beautifyName(s, k)
-  if s != beau:
-    lintReport(conf, info, beau, s)
-
 proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
   # operators stay as they are:
   if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
@@ -102,7 +93,7 @@ proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
     lintReport(conf, info, beau, s.name.s)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
-  if {optStyleHint, optStyleError} * conf.globalOptions != {}:
+  if {optStyleHint, optStyleError} * conf.globalOptions != {} and optStyleUsages notin conf.globalOptions:
     nep1CheckDefImpl(conf, info, s, k)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) =
@@ -137,6 +128,6 @@ proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) =
     lintReport(conf, info, newName, oldName)
 
 proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
-  let wanted = canonPragmaSpelling(w)
+  let wanted = $w
   if pragmaName != wanted:
     lintReport(conf, info, wanted, pragmaName)

compiler/llstream.nim

@@ -12,9 +12,11 @@
 import
   pathutils
 
-# support '-d:useGnuReadline' for backwards compatibility:
-when not defined(windows) and (defined(useGnuReadline) or defined(useLinenoise)):
-  import rdstdin
+# support `useGnuReadline`, `useLinenoise` for backwards compatibility
+const hasRstdin = (defined(nimUseLinenoise) or defined(useLinenoise) or defined(useGnuReadline)) and
+  not defined(windows)
+
+when hasRstdin: import rdstdin
 
 type
   TLLRepl* = proc (s: PLLStream, buf: pointer, bufLen: int): int
@@ -109,12 +111,12 @@ proc llReadFromStdin(s: PLLStream, buf: pointer, bufLen: int): int =
   var line = newStringOfCap(120)
   var triples = 0
   while readLineFromStdin(if s.s.len == 0: ">>> " else: "... ", line):
-    add(s.s, line)
-    add(s.s, "\n")
+    s.s.add(line)
+    s.s.add("\n")
     inc triples, countTriples(line)
     if not continueLine(line, (triples and 1) == 1): break
   inc(s.lineOffset)
-  result = min(bufLen, len(s.s) - s.rd)
+  result = min(bufLen, s.s.len - s.rd)
   if result > 0:
     copyMem(buf, addr(s.s[s.rd]), result)
     inc(s.rd, result)
@@ -124,7 +126,7 @@ proc llStreamRead*(s: PLLStream, buf: pointer, bufLen: int): int =
   of llsNone:
     result = 0
   of llsString:
-    result = min(bufLen, len(s.s) - s.rd)
+    result = min(bufLen, s.s.len - s.rd)
     if result > 0:
       copyMem(buf, addr(s.s[0 + s.rd]), result)
       inc(s.rd, result)
@@ -139,7 +141,7 @@ proc llStreamReadLine*(s: PLLStream, line: var string): bool =
   of llsNone:
     result = true
   of llsString:
-    while s.rd < len(s.s):
+    while s.rd < s.s.len:
       case s.s[s.rd]
       of '\x0D':
         inc(s.rd)
@@ -149,9 +151,9 @@ proc llStreamReadLine*(s: PLLStream, line: var string): bool =
         inc(s.rd)
         break
       else:
-        add(line, s.s[s.rd])
+        line.add(s.s[s.rd])
         inc(s.rd)
-    result = line.len > 0 or s.rd < len(s.s)
+    result = line.len > 0 or s.rd < s.s.len
   of llsFile:
     result = readLine(s.f, line)
   of llsStdIn:
@@ -162,8 +164,8 @@ proc llStreamWrite*(s: PLLStream, data: string) =
   of llsNone, llsStdIn:
     discard
   of llsString:
-    add(s.s, data)
-    inc(s.wr, len(data))
+    s.s.add(data)
+    inc(s.wr, data.len)
   of llsFile:
     write(s.f, data)
 
@@ -177,7 +179,7 @@ proc llStreamWrite*(s: PLLStream, data: char) =
   of llsNone, llsStdIn:
     discard
   of llsString:
-    add(s.s, data)
+    s.s.add(data)
     inc(s.wr)
   of llsFile:
     c = data
@@ -189,7 +191,7 @@ proc llStreamWrite*(s: PLLStream, buf: pointer, buflen: int) =
     discard
   of llsString:
     if buflen > 0:
-      setLen(s.s, len(s.s) + buflen)
+      setLen(s.s, s.s.len + buflen)
       copyMem(addr(s.s[0 + s.wr]), buf, buflen)
       inc(s.wr, buflen)
   of llsFile:
@@ -204,7 +206,7 @@ proc llStreamReadAll*(s: PLLStream): string =
   of llsString:
     if s.rd == 0: result = s.s
     else: result = substr(s.s, s.rd)
-    s.rd = len(s.s)
+    s.rd = s.s.len
   of llsFile:
     result = newString(bufSize)
     var bytes = readBuffer(s.f, addr(result[0]), bufSize)

compiler/lookups.nim

@@ -36,11 +36,11 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
   of nkAccQuoted:
     case n.len
     of 0: handleError(n, origin)
-    of 1: result = considerQuotedIdent(c, n.sons[0], origin)
+    of 1: result = considerQuotedIdent(c, n[0], origin)
     else:
       var id = ""
       for i in 0..<n.len:
-        let x = n.sons[i]
+        let x = n[i]
         case x.kind
         of nkIdent: id.add(x.ident.s)
         of nkSym: id.add(x.sym.name.s)
@@ -49,7 +49,7 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
       result = getIdent(c.cache, id)
   of nkOpenSymChoice, nkClosedSymChoice:
     if n[0].kind == nkSym:
-      result = n.sons[0].sym.name
+      result = n[0].sym.name
     else:
       handleError(n, origin)
   else:
@@ -91,8 +91,15 @@ proc skipAlias*(s: PSym; n: PNode; conf: ConfigRef): PSym =
       message(conf, n.info, warnDeprecated, "use " & result.name.s & " instead; " &
               s.name.s & " is deprecated")
 
+proc isShadowScope*(s: PScope): bool {.inline.} = s.parent != nil and s.parent.depthLevel == s.depthLevel
+
 proc localSearchInScope*(c: PContext, s: PIdent): PSym =
-  result = strTableGet(c.currentScope.symbols, s)
+  var scope = c.currentScope
+  result = strTableGet(scope.symbols, s)
+  while result == nil and scope.isShadowScope:
+    # We are in a shadow scope, check in the parent too
+    scope = scope.parent
+    result = strTableGet(scope.symbols, s)
 
 proc searchInScopes*(c: PContext, s: PIdent): PSym =
   for scope in walkScopes(c.currentScope):
@@ -100,16 +107,15 @@ proc searchInScopes*(c: PContext, s: PIdent): PSym =
     if result != nil: return
   result = nil
 
-when declared(echo):
-  proc debugScopes*(c: PContext; limit=0) {.deprecated.} =
-    var i = 0
-    for scope in walkScopes(c.currentScope):
-      echo "scope ", i
-      for h in 0 .. high(scope.symbols.data):
-        if scope.symbols.data[h] != nil:
-          echo scope.symbols.data[h].name.s
-      if i == limit: break
-      inc i
+proc debugScopes*(c: PContext; limit=0) {.deprecated.} =
+  var i = 0
+  for scope in walkScopes(c.currentScope):
+    echo "scope ", i
+    for h in 0..high(scope.symbols.data):
+      if scope.symbols.data[h] != nil:
+        echo scope.symbols.data[h].name.s
+    if i == limit: break
+    inc i
 
 proc searchInScopes*(c: PContext, s: PIdent, filter: TSymKinds): PSym =
   for scope in walkScopes(c.currentScope):
@@ -124,7 +130,7 @@ proc errorSym*(c: PContext, n: PNode): PSym =
   ## creates an error symbol to avoid cascading errors (for IDE support)
   var m = n
   # ensure that 'considerQuotedIdent' can't fail:
-  if m.kind == nkDotExpr: m = m.sons[1]
+  if m.kind == nkDotExpr: m = m[1]
   let ident = if m.kind in {nkIdent, nkSym, nkAccQuoted}:
       considerQuotedIdent(c, m)
     else:
@@ -232,6 +238,23 @@ proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) =
   addOverloadableSymAt(c, scope, sym)
   addInterfaceDeclAux(c, sym)
 
+proc openShadowScope*(c: PContext) =
+  c.currentScope = PScope(parent: c.currentScope,
+                          symbols: newStrTable(),
+                          depthLevel: c.scopeDepth)
+
+proc closeShadowScope*(c: PContext) =
+  c.closeScope
+
+proc mergeShadowScope*(c: PContext) =
+  let shadowScope = c.currentScope
+  c.rawCloseScope
+  for sym in shadowScope.symbols:
+    if sym.kind in OverloadableSyms:
+      c.addInterfaceOverloadableSymAt(c.currentScope, sym)
+    else:
+      c.addInterfaceDecl(sym)
+
 when defined(nimfix):
   # when we cannot find the identifier, retry with a changed identifier:
   proc altSpelling(x: PIdent): PIdent =
@@ -317,8 +340,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
       fixSpelling(n, ident, searchInScopes)
       errorUndeclaredIdentifier(c, n.info, ident.s)
       result = errorSym(c, n)
-    elif checkAmbiguity in flags and result != nil and
-        contains(c.ambiguousSymbols, result.id):
+    elif checkAmbiguity in flags and result != nil and result.id in c.ambiguousSymbols:
       errorUseQualifier(c, n.info, result)
   of nkSym:
     result = n.sym
@@ -326,29 +348,29 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
       errorUseQualifier(c, n.info, n.sym)
   of nkDotExpr:
     result = nil
-    var m = qualifiedLookUp(c, n.sons[0], (flags*{checkUndeclared})+{checkModule})
+    var m = qualifiedLookUp(c, n[0], (flags*{checkUndeclared})+{checkModule})
     if m != nil and m.kind == skModule:
       var ident: PIdent = nil
-      if n.sons[1].kind == nkIdent:
-        ident = n.sons[1].ident
-      elif n.sons[1].kind == nkAccQuoted:
-        ident = considerQuotedIdent(c, n.sons[1])
+      if n[1].kind == nkIdent:
+        ident = n[1].ident
+      elif n[1].kind == nkAccQuoted:
+        ident = considerQuotedIdent(c, n[1])
       if ident != nil:
         if m == c.module:
           result = strTableGet(c.topLevelScope.symbols, ident).skipAlias(n, c.config)
         else:
           result = strTableGet(m.tab, ident).skipAlias(n, c.config)
         if result == nil and checkUndeclared in flags:
-          fixSpelling(n.sons[1], ident, searchInScopes)
-          errorUndeclaredIdentifier(c, n.sons[1].info, ident.s)
-          result = errorSym(c, n.sons[1])
-      elif n.sons[1].kind == nkSym:
-        result = n.sons[1].sym
+          fixSpelling(n[1], ident, searchInScopes)
+          errorUndeclaredIdentifier(c, n[1].info, ident.s)
+          result = errorSym(c, n[1])
+      elif n[1].kind == nkSym:
+        result = n[1].sym
       elif checkUndeclared in flags and
-           n.sons[1].kind notin {nkOpenSymChoice, nkClosedSymChoice}:
-        localError(c.config, n.sons[1].info, "identifier expected, but got: " &
-                   renderTree(n.sons[1]))
-        result = errorSym(c, n.sons[1])
+           n[1].kind notin {nkOpenSymChoice, nkClosedSymChoice}:
+        localError(c.config, n[1].info, "identifier expected, but got: " &
+                   renderTree(n[1]))
+        result = errorSym(c, n[1])
   else:
     result = nil
   when false:
@@ -372,13 +394,13 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
     o.mode = oimDone
   of nkDotExpr:
     o.mode = oimOtherModule
-    o.m = qualifiedLookUp(c, n.sons[0], {checkUndeclared, checkModule})
+    o.m = qualifiedLookUp(c, n[0], {checkUndeclared, checkModule})
     if o.m != nil and o.m.kind == skModule:
       var ident: PIdent = nil
-      if n.sons[1].kind == nkIdent:
-        ident = n.sons[1].ident
-      elif n.sons[1].kind == nkAccQuoted:
-        ident = considerQuotedIdent(c, n.sons[1], n)
+      if n[1].kind == nkIdent:
+        ident = n[1].ident
+      elif n[1].kind == nkAccQuoted:
+        ident = considerQuotedIdent(c, n[1], n)
       if ident != nil:
         if o.m == c.module:
           # a module may access its private members:
@@ -388,12 +410,12 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         else:
           result = initIdentIter(o.it, o.m.tab, ident).skipAlias(n, c.config)
       else:
-        noidentError(c.config, n.sons[1], n)
-        result = errorSym(c, n.sons[1])
+        noidentError(c.config, n[1], n)
+        result = errorSym(c, n[1])
   of nkClosedSymChoice, nkOpenSymChoice:
     o.mode = oimSymChoice
     if n[0].kind == nkSym:
-      result = n.sons[0].sym
+      result = n[0].sym
     else:
       o.mode = oimDone
       return nil
@@ -430,8 +452,8 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
   of oimOtherModule:
     result = nextIdentIter(o.it, o.m.tab).skipAlias(n, c.config)
   of oimSymChoice:
-    if o.symChoiceIndex < len(n):
-      result = n.sons[o.symChoiceIndex].sym
+    if o.symChoiceIndex < n.len:
+      result = n[o.symChoiceIndex].sym
       incl(o.inSymChoice, result.id)
       inc o.symChoiceIndex
     elif n.kind == nkOpenSymChoice:
@@ -439,19 +461,19 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
       o.mode = oimSymChoiceLocalLookup
       o.scope = c.currentScope
       result = firstIdentExcluding(o.it, o.scope.symbols,
-                                   n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
       while result == nil:
         o.scope = o.scope.parent
         if o.scope == nil: break
         result = firstIdentExcluding(o.it, o.scope.symbols,
-                                     n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                     n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
   of oimSymChoiceLocalLookup:
     result = nextIdentExcluding(o.it, o.scope.symbols, o.inSymChoice).skipAlias(n, c.config)
     while result == nil:
       o.scope = o.scope.parent
       if o.scope == nil: break
       result = firstIdentExcluding(o.it, o.scope.symbols,
-                                   n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
 
   when false:
     if result != nil and result.kind == skStub: loadStub(result)

compiler/lowerings.nim

@@ -16,41 +16,55 @@ import ast, astalgo, types, idents, magicsys, msgs, options, modulegraphs,
   lineinfos
 
 proc newDeref*(n: PNode): PNode {.inline.} =
-  result = newNodeIT(nkHiddenDeref, n.info, n.typ.sons[0])
-  addSon(result, n)
+  result = newNodeIT(nkHiddenDeref, n.info, n.typ[0])
+  result.add n
 
 proc newTupleAccess*(g: ModuleGraph; tup: PNode, i: int): PNode =
   if tup.kind == nkHiddenAddr:
     result = newNodeIT(nkHiddenAddr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar, tyLent}))
-    result.addSon(newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar, tyLent}).sons[i]))
-    addSon(result[0], tup[0])
+    result.add newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar, tyLent})[i])
+    result[0].add tup[0]
     var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
     lit.intVal = i
-    addSon(result[0], lit)
+    result[0].add lit
   else:
     result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(
-                       abstractInst).sons[i])
-    addSon(result, copyTree(tup))
+                       abstractInst)[i])
+    result.add copyTree(tup)
     var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
     lit.intVal = i
-    addSon(result, lit)
+    result.add lit
 
 proc addVar*(father, v: PNode) =
   var vpart = newNodeI(nkIdentDefs, v.info, 3)
-  vpart.sons[0] = v
-  vpart.sons[1] = newNodeI(nkEmpty, v.info)
-  vpart.sons[2] = vpart[1]
-  addSon(father, vpart)
+  vpart[0] = v
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = vpart[1]
+  father.add vpart
+
+proc addVar*(father, v, value: PNode) =
+  var vpart = newNodeI(nkIdentDefs, v.info, 3)
+  vpart[0] = v
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = value
+  father.add vpart
 
 proc newAsgnStmt*(le, ri: PNode): PNode =
   result = newNodeI(nkAsgn, le.info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
 
 proc newFastAsgnStmt*(le, ri: PNode): PNode =
   result = newNodeI(nkFastAsgn, le.info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
+
+proc newFastMoveStmt*(g: ModuleGraph, le, ri: PNode): PNode =
+  result = newNodeI(nkFastAsgn, le.info, 2)
+  result[0] = le
+  result[1] = newNodeIT(nkCall, ri.info, ri.typ)
+  result[1].add newSymNode(getSysMagic(g, ri.info, "move", mMove))
+  result[1].add ri
 
 proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   assert n.kind == nkVarTuple
@@ -63,13 +77,13 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
 
   var v = newNodeI(nkVarSection, value.info)
   let tempAsNode = newSymNode(temp)
-  v.addVar(tempAsNode)
+  v.addVar(tempAsNode, value)
   result.add(v)
 
-  result.add newAsgnStmt(tempAsNode, value)
-  for i in 0 .. n.len-3:
-    if n.sons[i].kind == nkSym: v.addVar(n.sons[i])
-    result.add newAsgnStmt(n.sons[i], newTupleAccess(g, tempAsNode, i))
+  for i in 0..<n.len-2:
+    let val = newTupleAccess(g, tempAsNode, i)
+    if n[i].kind == nkSym: v.addVar(n[i], val)
+    else: result.add newAsgnStmt(n[i], val)
 
 proc evalOnce*(g: ModuleGraph; value: PNode; owner: PSym): PNode =
   ## Turns (value) into (let tmp = value; tmp) so that 'value' can be re-used
@@ -89,10 +103,10 @@ proc evalOnce*(g: ModuleGraph; value: PNode; owner: PSym): PNode =
 
 proc newTupleAccessRaw*(tup: PNode, i: int): PNode =
   result = newNodeI(nkBracketExpr, tup.info)
-  addSon(result, copyTree(tup))
+  result.add copyTree(tup)
   var lit = newNodeI(nkIntLit, tup.info)
   lit.intVal = i
-  addSon(result, lit)
+  result.add lit
 
 proc newTryFinally*(body, final: PNode): PNode =
   result = newTree(nkHiddenTryStmt, body, newTree(nkFinally, final))
@@ -106,31 +120,32 @@ proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   let tempAsNode = newSymNode(temp) #newIdentNode(getIdent(genPrefix & $temp.id), value.info)
 
   var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
-  vpart.sons[0] = tempAsNode
-  vpart.sons[1] = newNodeI(nkEmpty, value.info)
-  vpart.sons[2] = value
-  addSon(v, vpart)
+  vpart[0] = tempAsNode
+  vpart[1] = newNodeI(nkEmpty, value.info)
+  vpart[2] = value
+  v.add vpart
   result.add(v)
 
-  let lhs = n.sons[0]
-  for i in 0 .. lhs.len-1:
-    result.add newAsgnStmt(lhs.sons[i], newTupleAccessRaw(tempAsNode, i))
+  let lhs = n[0]
+  for i in 0..<lhs.len:
+    result.add newAsgnStmt(lhs[i], newTupleAccessRaw(tempAsNode, i))
 
 proc lowerSwap*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   result = newNodeI(nkStmtList, n.info)
   # note: cannot use 'skTemp' here cause we really need the copy for the VM :-(
   var temp = newSym(skVar, getIdent(g.cache, genPrefix), owner, n.info, owner.options)
-  temp.typ = n.sons[1].typ
+  temp.typ = n[1].typ
   incl(temp.flags, sfFromGeneric)
+  incl(temp.flags, sfGenSym)
 
   var v = newNodeI(nkVarSection, n.info)
   let tempAsNode = newSymNode(temp)
 
   var vpart = newNodeI(nkIdentDefs, v.info, 3)
-  vpart.sons[0] = tempAsNode
-  vpart.sons[1] = newNodeI(nkEmpty, v.info)
-  vpart.sons[2] = n[1]
-  addSon(v, vpart)
+  vpart[0] = tempAsNode
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = n[1]
+  v.add vpart
 
   result.add(v)
   result.add newFastAsgnStmt(n[1], n[2])
@@ -144,52 +159,59 @@ proc createObj*(g: ModuleGraph; owner: PSym, info: TLineInfo; final=true): PType
   else:
     rawAddSon(result, getCompilerProc(g, "RootObj").typ)
   result.n = newNodeI(nkRecList, info)
-  let s = newSym(skType, getIdent(g.cache, "Env_" & toFilename(g.config, info)),
+  let s = newSym(skType, getIdent(g.cache, "Env_" & toFilename(g.config, info) & "_" & $owner.name.s),
                   owner, info, owner.options)
   incl s.flags, sfAnon
   s.typ = result
   result.sym = s
 
+template fieldCheck {.dirty.} =
+  when false:
+    if tfCheckedForDestructor in obj.flags:
+      echo "missed field ", field.name.s
+      writeStackTrace()
+
 proc rawAddField*(obj: PType; field: PSym) =
   assert field.kind == skField
-  field.position = len(obj.n)
-  addSon(obj.n, newSymNode(field))
+  field.position = obj.n.len
+  obj.n.add newSymNode(field)
   propagateToOwner(obj, field.typ)
+  fieldCheck()
 
 proc rawIndirectAccess*(a: PNode; field: PSym; info: TLineInfo): PNode =
   # returns a[].field as a node
   assert field.kind == skField
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
-  addSon(deref, a)
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc rawDirectAccess*(obj, field: PSym): PNode =
   # returns a.field as a node
   assert field.kind == skField
   result = newNodeI(nkDotExpr, field.info)
-  addSon(result, newSymNode obj)
-  addSon(result, newSymNode field)
+  result.add newSymNode(obj)
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc lookupInRecord(n: PNode, id: int): PSym =
   result = nil
   case n.kind
   of nkRecList:
-    for i in 0 ..< len(n):
-      result = lookupInRecord(n.sons[i], id)
+    for i in 0..<n.len:
+      result = lookupInRecord(n[i], id)
       if result != nil: return
   of nkRecCase:
-    if n.sons[0].kind != nkSym: return
-    result = lookupInRecord(n.sons[0], id)
+    if n[0].kind != nkSym: return
+    result = lookupInRecord(n[0], id)
     if result != nil: return
-    for i in 1 ..< len(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = lookupInRecord(lastSon(n.sons[i]), id)
+        result = lookupInRecord(lastSon(n[i]), id)
         if result != nil: return
       else: discard
   of nkSym:
@@ -206,8 +228,10 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache) =
   field.typ = t
   assert t.kind != tyTyped
   propagateToOwner(obj, t)
-  field.position = len(obj.n)
-  addSon(obj.n, newSymNode(field))
+  field.position = obj.n.len
+  field.flags = s.flags * {sfCursor}
+  obj.n.add newSymNode(field)
+  fieldCheck()
 
 proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache): PSym {.discardable.} =
   result = lookupInRecord(obj.n, s.id)
@@ -219,45 +243,45 @@ proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache): PSym {.discardable
     field.typ = t
     assert t.kind != tyTyped
     propagateToOwner(obj, t)
-    field.position = len(obj.n)
-    addSon(obj.n, newSymNode(field))
+    field.position = obj.n.len
+    obj.n.add newSymNode(field)
     result = field
 
 proc newDotExpr*(obj, b: PSym): PNode =
   result = newNodeI(nkDotExpr, obj.info)
   let field = lookupInRecord(obj.typ.n, b.id)
   assert field != nil, b.name.s
-  addSon(result, newSymNode(obj))
-  addSon(result, newSymNode(field))
+  result.add newSymNode(obj)
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc indirectAccess*(a: PNode, b: int, info: TLineInfo): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
   while true:
     assert t.kind == tyObject
     field = lookupInRecord(t.n, b)
     if field != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
   #if field == nil:
   #  echo "FIELD ", b
   #  debug deref.typ
   assert field != nil
-  addSon(deref, a)
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc indirectAccess*(a: PNode, b: string, info: TLineInfo; cache: IdentCache): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
   let bb = getIdent(cache, b)
@@ -265,17 +289,17 @@ proc indirectAccess*(a: PNode, b: string, info: TLineInfo; cache: IdentCache): P
     assert t.kind == tyObject
     field = getSymFromList(t.n, bb)
     if field != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
   #if field == nil:
   #  echo "FIELD ", b
   #  debug deref.typ
   assert field != nil
-  addSon(deref, a)
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc getFieldFromObj*(t: PType; v: PSym): PSym =
@@ -285,7 +309,7 @@ proc getFieldFromObj*(t: PType; v: PSym): PSym =
     assert t.kind == tyObject
     result = lookupInRecord(t.n, v.id)
     if result != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
 
@@ -296,19 +320,19 @@ proc indirectAccess*(a: PNode, b: PSym, info: TLineInfo): PNode =
 proc indirectAccess*(a, b: PSym, info: TLineInfo): PNode =
   result = indirectAccess(newSymNode(a), b, info)
 
-proc genAddrOf*(n: PNode): PNode =
+proc genAddrOf*(n: PNode, typeKind = tyPtr): PNode =
   result = newNodeI(nkAddr, n.info, 1)
-  result.sons[0] = n
-  result.typ = newType(tyPtr, n.typ.owner)
+  result[0] = n
+  result.typ = newType(typeKind, n.typ.owner)
   result.typ.rawAddSon(n.typ)
 
 proc genDeref*(n: PNode; k = nkHiddenDeref): PNode =
   result = newNodeIT(k, n.info,
-                     n.typ.skipTypes(abstractInst).sons[0])
+                     n.typ.skipTypes(abstractInst)[0])
   result.add n
 
 proc callCodegenProc*(g: ModuleGraph; name: string;
-                      info: TLineInfo = unknownLineInfo();
+                      info: TLineInfo = unknownLineInfo;
                       arg1, arg2, arg3, optionalArgs: PNode = nil): PNode =
   result = newNodeI(nkCall, info)
   let sym = magicsys.getCompilerProc(g, name)
@@ -320,9 +344,9 @@ proc callCodegenProc*(g: ModuleGraph; name: string;
     if arg2 != nil: result.add arg2
     if arg3 != nil: result.add arg3
     if optionalArgs != nil:
-      for i in 1..optionalArgs.len-3:
+      for i in 1..<optionalArgs.len-2:
         result.add optionalArgs[i]
-    result.typ = sym.typ.sons[0]
+    result.typ = sym.typ[0]
 
 proc newIntLit*(g: ModuleGraph; info: TLineInfo; value: BiggestInt): PNode =
   result = nkIntLit.newIntNode(value)
@@ -334,8 +358,8 @@ proc genHigh*(g: ModuleGraph; n: PNode): PNode =
   else:
     result = newNodeI(nkCall, n.info, 2)
     result.typ = getSysType(g, n.info, tyInt)
-    result.sons[0] = newSymNode(getSysMagic(g, n.info, "high", mHigh))
-    result.sons[1] = n
+    result[0] = newSymNode(getSysMagic(g, n.info, "high", mHigh))
+    result[1] = n
 
 proc genLen*(g: ModuleGraph; n: PNode): PNode =
   if skipTypes(n.typ, abstractVar).kind == tyArray:
@@ -343,17 +367,6 @@ proc genLen*(g: ModuleGraph; n: PNode): PNode =
   else:
     result = newNodeI(nkCall, n.info, 2)
     result.typ = getSysType(g, n.info, tyInt)
-    result.sons[0] = newSymNode(getSysMagic(g, n.info, "len", mLengthSeq))
-    result.sons[1] = n
+    result[0] = newSymNode(getSysMagic(g, n.info, "len", mLengthSeq))
+    result[1] = n
 
-proc hoistExpr*(varSection, expr: PNode, name: PIdent, owner: PSym): PSym =
-  result = newSym(skLet, name, owner, varSection.info, owner.options)
-  result.flags.incl sfHoisted
-  result.typ = expr.typ
-
-  var varDef = newNodeI(nkIdentDefs, varSection.info, 3)
-  varDef.sons[0] = newSymNode(result)
-  varDef.sons[1] = newNodeI(nkEmpty, varSection.info)
-  varDef.sons[2] = expr
-
-  varSection.add varDef

compiler/magicsys.nim

@@ -40,7 +40,7 @@ proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSy
   while r != nil:
     if r.magic == m:
       # prefer the tyInt variant:
-      if r.typ.sons[0] != nil and r.typ.sons[0].kind == tyInt: return r
+      if r.typ[0] != nil and r.typ[0].kind == tyInt: return r
       result = r
     r = nextIdentIter(ti, g.systemModule.tab)
   if result != nil: return result
@@ -68,7 +68,7 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
     of tyUInt64: result = sysTypeFromName("uint64")
     of tyFloat: result = sysTypeFromName("float")
     of tyFloat32: result = sysTypeFromName("float32")
-    of tyFloat64: return sysTypeFromName("float64")
+    of tyFloat64: result = sysTypeFromName("float64")
     of tyFloat128: result = sysTypeFromName("float128")
     of tyBool: result = sysTypeFromName("bool")
     of tyChar: result = sysTypeFromName("char")
@@ -79,7 +79,9 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
     else: internalError(g.config, "request for typekind: " & $kind)
     g.sysTypes[kind] = result
   if result.kind != kind:
-    internalError(g.config, "wanted: " & $kind & " got: " & $result.kind)
+    if kind == tyFloat64 and result.kind == tyFloat: discard # because of aliasing
+    else:
+      internalError(g.config, "wanted: " & $kind & " got: " & $result.kind)
   if result == nil: internalError(g.config, "type not found: " & $kind)
 
 proc resetSysTypes*(g: ModuleGraph) =
@@ -123,7 +125,7 @@ proc addSonSkipIntLit*(father, son: PType) =
   when not defined(nimNoNilSeqs):
     if isNil(father.sons): father.sons = @[]
   let s = son.skipIntLit
-  add(father.sons, s)
+  father.sons.add(s)
   propagateToOwner(father, s)
 
 proc setIntLitType*(g: ModuleGraph; result: PNode) =
@@ -175,3 +177,28 @@ proc getNimScriptSymbol*(g: ModuleGraph; name: string): PSym =
   strTableGet(g.exposed, getIdent(g.cache, name))
 
 proc resetNimScriptSymbols*(g: ModuleGraph) = initStrTable(g.exposed)
+
+proc getMagicEqSymForType*(g: ModuleGraph; t: PType; info: TLineInfo): PSym =
+  case t.kind
+  of tyInt,  tyInt8, tyInt16, tyInt32, tyInt64,
+     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64: 
+    result = getSysMagic(g, info, "==", mEqI)
+  of tyEnum: 
+    result = getSysMagic(g, info, "==", mEqEnum)
+  of tyBool: 
+    result = getSysMagic(g, info, "==", mEqB)
+  of tyRef, tyPtr, tyPointer: 
+    result = getSysMagic(g, info, "==", mEqRef)
+  of tyString:
+    result = getSysMagic(g, info, "==", mEqStr)
+  of tyChar:
+    result = getSysMagic(g, info, "==", mEqCh)
+  of tySet:
+    result = getSysMagic(g, info, "==", mEqSet)
+  of tyProc:
+    result = getSysMagic(g, info, "==", mEqProc)
+  else:
+    globalError(g.config, info,
+      "can't find magic equals operator for type kind " & $t.kind)
+
+

compiler/main.nim

@@ -13,13 +13,13 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  llstream, strutils, ast, lexer, syntaxes, options, msgs,
+  llstream, strutils, os, ast, lexer, syntaxes, options, msgs,
   condsyms, times,
   sem, idents, passes, extccomp,
   cgen, json, nversion,
   platform, nimconf, passaux, depends, vm, idgen,
   modules,
-  modulegraphs, tables, rod, lineinfos, pathutils
+  modulegraphs, tables, rod, lineinfos, pathutils, vmprofiler
 
 when not defined(leanCompiler):
   import jsgen, docgen, docgen2
@@ -51,7 +51,7 @@ proc commandGenDepend(graph: ModuleGraph) =
       ' ' & changeFileExt(project, "dot").string)
 
 proc commandCheck(graph: ModuleGraph) =
-  graph.config.errorMax = high(int)  # do not stop after first error
+  graph.config.setErrorMaxHighMaybe
   defineSymbol(graph.config.symbols, "nimcheck")
   semanticPasses(graph)  # use an empty backend for semantic checking only
   compileProject(graph)
@@ -59,7 +59,7 @@ proc commandCheck(graph: ModuleGraph) =
 when not defined(leanCompiler):
   proc commandDoc2(graph: ModuleGraph; json: bool) =
     handleDocOutputOptions graph.config
-    graph.config.errorMax = high(int)  # do not stop after first error
+    graph.config.setErrorMaxHighMaybe
     semanticPasses(graph)
     if json: registerPass(graph, docgen2JsonPass)
     else: registerPass(graph, docgen2Pass)
@@ -68,16 +68,7 @@ when not defined(leanCompiler):
 
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
-
-  if conf.outDir.isEmpty:
-    conf.outDir = conf.projectPath
-  if conf.outFile.isEmpty:
-    let targetName = if optGenDynLib in conf.globalOptions:
-      platform.OS[conf.target.targetOS].dllFrmt % conf.projectName
-    else:
-      conf.projectName & platform.OS[conf.target.targetOS].exeExt
-    conf.outFile = RelativeFile targetName
-
+  setOutFile(conf)
   extccomp.initVars(conf)
   semanticPasses(graph)
   registerPass(graph, cgenPass)
@@ -89,6 +80,9 @@ proc commandCompileToC(graph: ModuleGraph) =
       graph.config.notes = graph.config.mainPackageNotes
       return
 
+  if not extccomp.ccHasSaneOverflow(conf):
+    conf.symbols.defineSymbol("nimEmulateOverflowChecks")
+
   compileProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933
@@ -105,12 +99,13 @@ proc commandJsonScript(graph: ModuleGraph) =
   let proj = changeFileExt(graph.config.projectFull, "")
   extccomp.runJsonBuildInstructions(graph.config, proj)
 
-when not defined(leanCompiler):
-  proc commandCompileToJS(graph: ModuleGraph) =
+proc commandCompileToJS(graph: ModuleGraph) =
+  when defined(leanCompiler):
+    globalError(graph.config, unknownLineInfo, "compiler wasn't built with JS code generator")
+  else:
     let conf = graph.config
+    conf.exc = excCpp
 
-    if conf.outDir.isEmpty:
-      conf.outDir = conf.projectPath
     if conf.outFile.isEmpty:
       conf.outFile = RelativeFile(conf.projectName & ".js")
 
@@ -118,7 +113,6 @@ when not defined(leanCompiler):
     setTarget(graph.config.target, osJS, cpuJS)
     #initDefines()
     defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility
-    defineSymbol(graph.config.symbols, "js")
     semanticPasses(graph)
     registerPass(graph, JSgenPass)
     compileProject(graph)
@@ -135,7 +129,7 @@ proc interactivePasses(graph: ModuleGraph) =
   registerPass(graph, evalPass)
 
 proc commandInteractive(graph: ModuleGraph) =
-  graph.config.errorMax = high(int)  # do not stop after first error
+  graph.config.setErrorMaxHighMaybe
   interactivePasses(graph)
   compileSystemModule(graph)
   if graph.config.commandArgs.len > 0:
@@ -155,8 +149,8 @@ proc commandScan(cache: IdentCache, config: ConfigRef) =
   var stream = llStreamOpen(f, fmRead)
   if stream != nil:
     var
-      L: TLexer
-      tok: TToken
+      L: Lexer
+      tok: Token
     initToken(tok)
     openLexer(L, f, stream, cache, config)
     while true:
@@ -180,56 +174,105 @@ proc mainCommand*(graph: ModuleGraph) =
   conf.lastCmdTime = epochTime()
   conf.searchPaths.add(conf.libpath)
   setId(100)
-  case conf.command.normalize
-  of "c", "cc", "compile", "compiletoc":
-    # compile means compileToC currently
-    conf.cmd = cmdCompileToC
-    defineSymbol(graph.config.symbols, "c")
-    commandCompileToC(graph)
-  of "cpp", "compiletocpp":
-    conf.cmd = cmdCompileToCpp
-    defineSymbol(graph.config.symbols, "cpp")
-    commandCompileToC(graph)
-  of "objc", "compiletooc":
-    conf.cmd = cmdCompileToOC
-    defineSymbol(graph.config.symbols, "objc")
-    commandCompileToC(graph)
-  of "run":
-    conf.cmd = cmdRun
-    when hasTinyCBackend:
-      extccomp.setCC("tcc")
-      commandCompileToC(graph)
-    else:
-      rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
-  of "js", "compiletojs":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with JS code generator"
-    else:
-      conf.cmd = cmdCompileToJS
+
+  proc customizeForBackend(backend: TBackend) =
+    ## Sets backend specific options but don't compile to backend yet in
+    ## case command doesn't require it. This must be called by all commands.
+    if conf.backend == backendInvalid:
+      # only set if wasn't already set, to allow override via `nim c -b:cpp`
+      conf.backend = backend
+
+    defineSymbol(graph.config.symbols, $conf.backend)
+    case conf.backend
+    of backendC:
+      if conf.exc == excNone: conf.exc = excSetjmp
+    of backendCpp:
+      if conf.exc == excNone: conf.exc = excCpp
+    of backendObjc: discard
+    of backendJs:
       if conf.hcrOn:
         # XXX: At the moment, system.nim cannot be compiled in JS mode
         # with "-d:useNimRtl". The HCR option has been processed earlier
         # and it has added this define implictly, so we must undo that here.
         # A better solution might be to fix system.nim
         undefSymbol(conf.symbols, "useNimRtl")
-      commandCompileToJS(graph)
-  of "doc0":
+    of backendInvalid: doAssert false
+    if conf.selectedGC in {gcArc, gcOrc} and conf.backend != backendCpp:
+      conf.exc = excGoto
+
+  var commandAlreadyProcessed = false
+
+  proc compileToBackend(backend: TBackend, cmd = cmdCompileToBackend) =
+    commandAlreadyProcessed = true
+    conf.cmd = cmd
+    customizeForBackend(backend)
+    case conf.backend
+    of backendC: commandCompileToC(graph)
+    of backendCpp: commandCompileToC(graph)
+    of backendObjc: commandCompileToC(graph)
+    of backendJs: commandCompileToJS(graph)
+    of backendInvalid: doAssert false
+
+  template docLikeCmd(body) =
     when defined(leanCompiler):
       quit "compiler wasn't built with documentation generator"
     else:
       wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandDoc(cache, conf)
-  of "doc2", "doc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
       conf.cmd = cmdDoc
       loadConfigs(DocConfig, cache, conf)
       defineSymbol(conf.symbols, "nimdoc")
+      body
+
+  block: ## command prepass
+    var docLikeCmd2 = false # includes what calls `docLikeCmd` + some more
+    case conf.command.normalize
+    of "r": conf.globalOptions.incl {optRun, optUseNimcache}
+    of "doc0",  "doc2", "doc", "rst2html", "rst2tex", "jsondoc0", "jsondoc2",
+      "jsondoc", "ctags", "buildindex": docLikeCmd2 = true
+    else: discard
+    if conf.outDir.isEmpty:
+      # doc like commands can generate a lot of files (especially with --project)
+      # so by default should not end up in $PWD nor in $projectPath.
+      conf.outDir = block:
+        var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
+                  else: conf.projectPath
+        doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
+        if docLikeCmd2: ret = ret / htmldocsDir
+        ret
+
+  ## process all backend commands
+  case conf.command.normalize
+  of "c", "cc", "compile", "compiletoc": compileToBackend(backendC) # compile means compileToC currently
+  of "cpp", "compiletocpp": compileToBackend(backendCpp)
+  of "objc", "compiletooc": compileToBackend(backendObjc)
+  of "js", "compiletojs": compileToBackend(backendJs)
+  of "r": compileToBackend(backendC) # different from `"run"`!
+  of "run":
+    when hasTinyCBackend:
+      extccomp.setCC(conf, "tcc", unknownLineInfo)
+      if conf.backend notin {backendC, backendInvalid}:
+        rawMessage(conf, errGenerated, "'run' requires c backend, got: '$1'" % $conf.backend)
+      compileToBackend(backendC, cmd = cmdRun)
+    else:
+      rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
+  else: customizeForBackend(backendC) # fallback for other commands
+
+  ## process all other commands
+  case conf.command.normalize # synchronize with `cmdUsingHtmlDocs`
+  of "doc0": docLikeCmd commandDoc(cache, conf)
+  of "doc2", "doc":
+    docLikeCmd():
+      conf.setNoteDefaults(warnLockLevel, false) # issue #13218
+      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+        # because currently generates lots of false positives due to conflation
+        # of labels links in doc comments, e.g. for random.rand:
+        #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
+        #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
       commandDoc2(graph, false)
+      if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
+        commandBuildIndex(conf, $conf.outDir)
   of "rst2html":
+    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
     when defined(leanCompiler):
       quit "compiler wasn't built with documentation generator"
     else:
@@ -243,41 +286,10 @@ proc mainCommand*(graph: ModuleGraph) =
       conf.cmd = cmdRst2tex
       loadConfigs(DocTexConfig, cache, conf)
       commandRst2TeX(cache, conf)
-  of "jsondoc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandJson(cache, conf)
-  of "jsondoc2", "jsondoc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandDoc2(graph, true)
-  of "ctags":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandTags(cache, conf)
-  of "buildindex":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandBuildIndex(cache, conf)
+  of "jsondoc0": docLikeCmd commandJson(cache, conf)
+  of "jsondoc2", "jsondoc": docLikeCmd commandDoc2(graph, true)
+  of "ctags": docLikeCmd commandTags(cache, conf)
+  of "buildindex": docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
   of "gendepend":
     conf.cmd = cmdGenDepend
     commandGenDepend(graph)
@@ -290,22 +302,26 @@ proc mainCommand*(graph: ModuleGraph) =
       for s in definedSymbolNames(conf.symbols): definedSymbols.elems.add(%s)
 
       var libpaths = newJArray()
+      var lazyPaths = newJArray()
       for dir in conf.searchPaths: libpaths.elems.add(%dir.string)
+      for dir in conf.lazyPaths: lazyPaths.elems.add(%dir.string)
 
       var hints = newJObject() # consider factoring with `listHints`
       for a in hintMin..hintMax:
-        let key = lineinfos.HintsToStr[ord(a) - ord(hintMin)]
-        hints[key] = %(a in conf.notes)
+        hints[$a] = %(a in conf.notes)
       var warnings = newJObject()
       for a in warnMin..warnMax:
-        let key = lineinfos.WarningsToStr[ord(a) - ord(warnMin)]
-        warnings[key] = %(a in conf.notes)
+        warnings[$a] = %(a in conf.notes)
 
       var dumpdata = %[
         (key: "version", val: %VersionAsString),
+        (key: "nimExe", val: %(getAppFilename())),
+        (key: "prefixdir", val: %conf.getPrefixDir().string),
+        (key: "libpath", val: %conf.libpath.string),
         (key: "project_path", val: %conf.projectFull.string),
         (key: "defined_symbols", val: definedSymbols),
         (key: "lib_paths", val: %libpaths),
+        (key: "lazyPaths", val: %lazyPaths),
         (key: "outdir", val: %conf.outDir.string),
         (key: "out", val: %conf.outFile.string),
         (key: "nimcache", val: %getNimcacheDir(conf).string),
@@ -347,22 +363,43 @@ proc mainCommand*(graph: ModuleGraph) =
     conf.cmd = cmdDump
   of "jsonscript":
     conf.cmd = cmdJsonScript
+    setOutFile(graph.config)
     commandJsonScript(graph)
+  elif commandAlreadyProcessed: discard # already handled
   else:
     rawMessage(conf, errGenerated, "invalid command: " & conf.command)
 
   if conf.errorCounter == 0 and
      conf.cmd notin {cmdInterpret, cmdRun, cmdDump}:
-    when declared(system.getMaxMem):
-      let usedMem = formatSize(getMaxMem()) & " peakmem"
+    let mem =
+      when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem"
+      else: formatSize(getTotalMem()) & " totmem"
+    let loc = $conf.linesCompiled
+    let build = if isDefined(conf, "danger"): "Dangerous Release"
+                elif isDefined(conf, "release"): "Release"
+                else: "Debug"
+    let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3)
+    let project = if optListFullPaths in conf.globalOptions: $conf.projectFull else: $conf.projectName
+
+    var output: string
+    if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonScript:
+      output = $conf.jsonBuildFile
+    elif conf.outFile.isEmpty and conf.cmd notin {cmdJsonScript, cmdCompileToBackend, cmdDoc}:
+      # for some cmd we expect a valid absOutFile
+      output = "unknownOutput"
     else:
-      let usedMem = formatSize(getTotalMem())
-    rawMessage(conf, hintSuccessX, [$conf.linesCompiled,
-               formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3),
-               usedMem,
-               if isDefined(conf, "danger"): "Dangerous Release Build"
-               elif isDefined(conf, "release"): "Release Build"
-               else: "Debug Build"])
+      output = $conf.absOutFile
+    if optListFullPaths notin conf.globalOptions: output = output.AbsoluteFile.extractFilename
+    if optProfileVM in conf.globalOptions:
+      echo conf.dump(conf.vmProfileData)
+    rawMessage(conf, hintSuccessX, [
+      "loc", loc,
+      "sec", sec,
+      "mem", mem,
+      "build", build,
+      "project", project,
+      "output", output,
+      ])
 
   when PrintRopeCacheStats:
     echo "rope cache stats: "
@@ -371,5 +408,3 @@ proc mainCommand*(graph: ModuleGraph) =
     echo "  int tries: ", gCacheIntTries
     echo "  efficiency: ", formatFloat(1-(gCacheMisses.float/gCacheTries.float),
                                        ffDecimal, 3)
-
-  resetAttributes(conf)

compiler/modulegraphs.nim

@@ -31,7 +31,7 @@ import ast, intsets, tables, options, lineinfos, hashes, idents,
 type
   SigHash* = distinct MD5Digest
 
-  ModuleGraph* = ref object
+  ModuleGraph* {.acyclic.} = ref object
     modules*: seq[PSym]  ## indexed by int32 fileIdx
     packageSyms*: TStrTable
     deps*: IntSet # the dependency graph or potentially its transitive closure.
@@ -72,6 +72,8 @@ type
     onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     globalDestructors*: seq[PNode]
+    strongSemCheck*: proc (graph: ModuleGraph; owner: PSym; body: PNode) {.nimcall.}
+    compatibleProps*: proc (graph: ModuleGraph; formal, actual: PType): bool {.nimcall.}
 
   TPassContext* = object of RootObj # the pass's context
   PPassContext* = ref TPassContext
@@ -163,8 +165,9 @@ proc stopCompile*(g: ModuleGraph): bool {.inline.} =
   result = g.doStopCompile != nil and g.doStopCompile()
 
 proc createMagic*(g: ModuleGraph; name: string, m: TMagic): PSym =
-  result = newSym(skProc, getIdent(g.cache, name), nil, unknownLineInfo(), {})
+  result = newSym(skProc, getIdent(g.cache, name), nil, unknownLineInfo, {})
   result.magic = m
+  result.flags = {sfNeverRaises}
 
 proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result = ModuleGraph()

compiler/modulepaths.nim

@@ -91,8 +91,7 @@ when false:
             if result.len > 0: return result
 
   proc scriptableImport(pkg, sub: string; info: TLineInfo): string =
-    result = resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info)
-    if result.isNil: result = ""
+    resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info)
 
   proc lookupPackage(pkg, subdir: PNode): string =
     let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: ""
@@ -112,8 +111,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
   case n.kind
   of nkStrLit, nkRStrLit, nkTripleStrLit:
     try:
-      result =
-        pathSubs(conf, n.strVal, toFullPath(conf, n.info).splitFile().dir)
+      result = pathSubs(conf, n.strVal, toFullPath(conf, n.info).splitFile().dir)
     except ValueError:
       localError(conf, n.info, "invalid path: " & n.strVal)
       result = n.strVal
@@ -139,7 +137,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
       result.add modname
   of nkPrefix:
     when false:
-      if n.sons[0].kind == nkIdent and n.sons[0].ident.s == "$":
+      if n[0].kind == nkIdent and n[0].ident.s == "$":
         result = lookupPackage(n[1], nil)
       else:
         discard
@@ -149,7 +147,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
     localError(conf, n.info, warnDeprecated, "using '.' instead of '/' in import paths is deprecated")
     result = renderTree(n, {renderNoComments}).replace(".", "/")
   of nkImportAs:
-    result = getModuleName(conf, n.sons[0])
+    result = getModuleName(conf, n[0])
   else:
     localError(conf, n.info, "invalid module name: '$1'" % n.renderTree)
     result = ""

compiler/modules.nim

@@ -17,32 +17,43 @@ import
 proc resetSystemArtifacts*(g: ModuleGraph) =
   magicsys.resetSysTypes(g)
 
-proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
+template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent =
+  getIdent(graph.cache, splitFile(filename).name)
+
+proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+  ## returns package symbol (skPackage) for yet to be defined module for fileIdx
+  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+  let name = getModuleIdent(graph, filename)
+  let info = newLineInfo(fileIdx, 1, 1)
   let
     pck = getPackageName(graph.config, filename.string)
     pck2 = if pck.len > 0: pck else: "unknown"
     pack = getIdent(graph.cache, pck2)
   var packSym = graph.packageSyms.strTableGet(pack)
   if packSym == nil:
-    packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, result.info)
+    packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, info)
     initStrTable(packSym.tab)
     graph.packageSyms.strTableAdd(packSym)
   else:
-    let existing = strTableGet(packSym.tab, result.name)
-    if existing != nil and existing.info.fileIndex != result.info.fileIndex:
+    let existing = strTableGet(packSym.tab, name)
+    if existing != nil and existing.info.fileIndex != info.fileIndex:
       when false:
         # we used to produce an error:
-        localError(graph.config, result.info,
+        localError(graph.config, info,
           "module names need to be unique per Nimble package; module clashes with " &
             toFullPath(graph.config, existing.info.fileIndex))
       else:
         # but starting with version 0.20 we now produce a fake Nimble package instead
         # to resolve the conflicts:
         let pck3 = fakePackageName(graph.config, filename)
-        packSym = newSym(skPackage, getIdent(graph.cache, pck3), nil, result.info)
+        # this makes the new `packSym`'s owner be the original `packSym`
+        packSym = newSym(skPackage, getIdent(graph.cache, pck3), packSym, info)
         initStrTable(packSym.tab)
         graph.packageSyms.strTableAdd(packSym)
+  result = packSym
 
+proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
+  let packSym = getPackage(graph, fileIdx)
   result.owner = packSym
   result.position = int fileIdx
 
@@ -55,19 +66,19 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil
   strTableAdd(packSym.tab, result)
 
 proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID.
-  new(result)
-  result.id = -1             # for better error checking
-  result.kind = skModule
-  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-  result.name = getIdent(graph.cache, splitFile(filename).name)
+  result = PSym(kind: skModule, id: -1, # for better error checking
+                name: getModuleIdent(graph, filename),
+                info: newLineInfo(fileIdx, 1, 1))
   if not isNimIdentifier(result.name.s):
     rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
-  result.info = newLineInfo(fileIdx, 1, 1)
   partialInitModule(result, graph, fileIdx, filename)
 
 proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym =
+  var flags = flags
+  if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
   result = graph.getModule(fileIdx)
   if result == nil:
     let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
@@ -75,7 +86,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
     result = r
     if result == nil:
       result = newModule(graph, fileIdx)
-      result.flags = result.flags + flags
+      result.flags.incl flags
       result.id = id
       registerModule(graph, result)
     else:
@@ -93,7 +104,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
       if sfMainModule in flags and graph.config.projectIsStdin: stdin.llStreamOpen else: nil)
     graph.markClientsDirty(fileIdx)
 
-proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym {.procvar.} =
+proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
   # this is called by the semantic checking phase
   assert graph.config != nil
   result = compileModule(graph, fileIdx, {})
@@ -105,10 +116,10 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym {.proc
   #  localError(result.info, errAttemptToRedefine, result.name.s)
   # restore the notes for outer module:
   graph.config.notes =
-    if s.owner.id == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
+    if s.getnimblePkgId == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
     else: graph.config.foreignPackageNotes
 
-proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode {.procvar.} =
+proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode =
   result = syntaxes.parseFile(fileIdx, graph.cache, graph.config)
   graph.addDep(s, fileIdx)
   graph.addIncludeDep(s.position.FileIndex, fileIdx)
@@ -126,7 +137,7 @@ proc compileSystemModule*(graph: ModuleGraph) =
 
 proc wantMainModule*(conf: ConfigRef) =
   if conf.projectFull.isEmpty:
-    fatal(conf, newLineInfo(conf, AbsoluteFile"command line", 1, 1), errGenerated,
+    fatal(conf, newLineInfo(conf, AbsoluteFile(commandLineDesc), 1, 1), errGenerated,
         "command expects a filename")
   conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt))
 
@@ -136,7 +147,12 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
   wantMainModule(conf)
   let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
   let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
+  conf.projectMainIdx2 = projectFile
+
+  let packSym = getPackage(graph, projectFile)
+  graph.config.mainPackageId = packSym.getnimblePkgId
   graph.importStack.add projectFile
+
   if projectFile == systemFileIdx:
     discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})
   else:

compiler/msgs.nim

@@ -10,6 +10,17 @@
 import
   options, strutils, os, tables, ropes, terminal, macros,
   lineinfos, pathutils
+import std/private/miscdollars
+import strutils2
+
+type InstantiationInfo* = typeof(instantiationInfo())
+template instLoc(): InstantiationInfo = instantiationInfo(-2, fullPaths = true)
+
+template flushDot(conf, stdorr) =
+  ## safe to call multiple times
+  if conf.lastMsgWasDot:
+    conf.lastMsgWasDot = false
+    write(stdorr, "\n")
 
 proc toCChar*(c: char; result: var string) =
   case c
@@ -26,14 +37,13 @@ proc makeCString*(s: string): Rope =
   const MaxLineLength = 64
   result = nil
   var res = newStringOfCap(int(s.len.toFloat * 1.1) + 1)
-  add(res, "\"")
-  for i in 0 ..< len(s):
+  res.add("\"")
+  for i in 0..<s.len:
     if (i + 1) mod MaxLineLength == 0:
-      add(res, "\"\L\"")
+      res.add("\"\L\"")
     toCChar(s[i], res)
-  add(res, '\"')
-  add(result, rope(res))
-
+  res.add('\"')
+  result.add(rope(res))
 
 proc newFileInfo(fullPath: AbsoluteFile, projPath: RelativeFile): TFileInfo =
   result.fullPath = fullPath
@@ -57,6 +67,12 @@ when defined(nimpretty):
   proc fileSection*(conf: ConfigRef; fid: FileIndex; a, b: int): string =
     substr(conf.m.fileInfos[fid.int].fullContent, a, b)
 
+proc canonicalCase(path: var string) =
+  ## the idea is to only use this for checking whether a path is already in
+  ## the table but otherwise keep the original case
+  when FileSystemCaseSensitive: discard
+  else: toLowerAscii(path)
+
 proc fileInfoKnown*(conf: ConfigRef; filename: AbsoluteFile): bool =
   var
     canon: AbsoluteFile
@@ -64,6 +80,7 @@ proc fileInfoKnown*(conf: ConfigRef; filename: AbsoluteFile): bool =
     canon = canonicalizePath(conf, filename)
   except OSError:
     canon = filename
+  canon.string.canonicalCase
   result = conf.m.filenameToIndexTbl.hasKey(canon.string)
 
 proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile; isKnownFile: var bool): FileIndex =
@@ -80,15 +97,19 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile; isKnownFile: var bool
     # This flag indicates that we are working with such a path here
     pseudoPath = true
 
-  if conf.m.filenameToIndexTbl.hasKey(canon.string):
+  var canon2: string
+  forceCopy(canon2, canon.string) # because `canon` may be shallow
+  canon2.canonicalCase
+
+  if conf.m.filenameToIndexTbl.hasKey(canon2):
     isKnownFile = true
-    result = conf.m.filenameToIndexTbl[canon.string]
+    result = conf.m.filenameToIndexTbl[canon2]
   else:
     isKnownFile = false
     result = conf.m.fileInfos.len.FileIndex
     conf.m.fileInfos.add(newFileInfo(canon, if pseudoPath: RelativeFile filename
                                             else: relativeTo(canon, conf.projectPath)))
-    conf.m.filenameToIndexTbl[canon.string] = result
+    conf.m.filenameToIndexTbl[canon2] = result
 
 proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile): FileIndex =
   var dummy: bool
@@ -108,6 +129,7 @@ proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo =
 proc newLineInfo*(conf: ConfigRef; filename: AbsoluteFile, line, col: int): TLineInfo {.inline.} =
   result = newLineInfo(fileInfoIdx(conf, filename), line, col)
 
+const gCmdLineInfo* = newLineInfo(commandLineIdx, 1, 1)
 
 proc concat(strings: openArray[string]): string =
   var totalLen = 0
@@ -132,7 +154,6 @@ proc suggestQuit*() =
 # this format is understood by many text editors: it is the same that
 # Borland and Freepascal use
 const
-  PosFormat    = "$1($2, $3) "
   KindFormat   = " [$1]"
   KindColor    = fgCyan
   ErrorTitle   = "Error: "
@@ -145,6 +166,7 @@ const
   # but column numbers start with 0, however most editors expect
   # first column to be 1, so we need to +1 here
   ColOffset*   = 1
+  commandLineDesc* = "command line"
 
 proc getInfoContextLen*(conf: ConfigRef): int = return conf.m.msgContext.len
 proc setInfoContextLen*(conf: ConfigRef; L: int) = setLen(conf.m.msgContext, L)
@@ -153,17 +175,13 @@ proc pushInfoContext*(conf: ConfigRef; info: TLineInfo; detail: string = "") =
   conf.m.msgContext.add((info, detail))
 
 proc popInfoContext*(conf: ConfigRef) =
-  setLen(conf.m.msgContext, len(conf.m.msgContext) - 1)
+  setLen(conf.m.msgContext, conf.m.msgContext.len - 1)
 
 proc getInfoContext*(conf: ConfigRef; index: int): TLineInfo =
-  let L = conf.m.msgContext.len
-  let i = if index < 0: L + index else: index
-  if i >=% L: result = unknownLineInfo()
+  let i = if index < 0: conf.m.msgContext.len + index else: index
+  if i >=% conf.m.msgContext.len: result = unknownLineInfo
   else: result = conf.m.msgContext[i].info
 
-const
-  commandLineDesc = "command line"
-
 template toFilename*(conf: ConfigRef; fileIdx: FileIndex): string =
   if fileIdx.int32 < 0 or conf == nil:
     (if fileIdx == commandLineIdx: commandLineDesc else: "???")
@@ -214,16 +232,36 @@ template toFullPath*(conf: ConfigRef; info: TLineInfo): string =
 template toFullPathConsiderDirty*(conf: ConfigRef; info: TLineInfo): string =
   string toFullPathConsiderDirty(conf, info.fileIndex)
 
+type FilenameOption* = enum
+  foAbs # absolute path, e.g.: /pathto/bar/foo.nim
+  foRelProject # relative to project path, e.g.: ../foo.nim
+  foMagicSauce # magic sauce, shortest of (foAbs, foRelProject)
+  foName # lastPathPart, e.g.: foo.nim
+  foStacktrace # if optExcessiveStackTrace: foAbs else: foName
+
+proc toFilenameOption*(conf: ConfigRef, fileIdx: FileIndex, opt: FilenameOption): string =
+  case opt
+  of foAbs: result = toFullPath(conf, fileIdx)
+  of foRelProject: result = toProjPath(conf, fileIdx)
+  of foMagicSauce:
+    let
+      absPath = toFullPath(conf, fileIdx)
+      relPath = toProjPath(conf, fileIdx)
+    result = if (optListFullPaths in conf.globalOptions) or
+                (relPath.len > absPath.len) or
+                (relPath.count("..") > 2):
+               absPath
+             else:
+               relPath
+  of foName: result = toProjPath(conf, fileIdx).lastPathPart
+  of foStacktrace:
+    if optExcessiveStackTrace in conf.globalOptions:
+      result = toFilenameOption(conf, fileIdx, foAbs)
+    else:
+      result = toFilenameOption(conf, fileIdx, foName)
+
 proc toMsgFilename*(conf: ConfigRef; info: FileIndex): string =
-  let
-    absPath = toFullPath(conf, info)
-    relPath = toProjPath(conf, info)
-  result = if (optListFullPaths in conf.globalOptions) or
-              (relPath.len > absPath.len) or
-              (relPath.count("..") > 2):
-             absPath
-           else:
-             relPath
+  toFilenameOption(conf, info, foMagicSauce)
 
 template toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
   toMsgFilename(conf, info.fileIndex)
@@ -234,10 +272,11 @@ proc toLinenumber*(info: TLineInfo): int {.inline.} =
 proc toColumn*(info: TLineInfo): int {.inline.} =
   result = info.col
 
+proc toFileLineCol(info: InstantiationInfo): string {.inline.} =
+  result.toLocation(info.filename, info.line, info.column + ColOffset)
+
 proc toFileLineCol*(conf: ConfigRef; info: TLineInfo): string {.inline.} =
-  # consider calling `helpers.lineInfoToString` instead
-  result = toMsgFilename(conf, info) & "(" & $info.line & ", " &
-    $(info.col + ColOffset) & ")"
+  result.toLocation(toMsgFilename(conf, info), info.line.int, info.col.int + ColOffset)
 
 proc `$`*(conf: ConfigRef; info: TLineInfo): string = toFileLineCol(conf, info)
 
@@ -262,15 +301,16 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) =
   ## This is used for 'nim dump' etc. where we don't have nimsuggest
   ## support.
   #if conf.cmd == cmdIdeTools and optCDebug notin gGlobalOptions: return
-
   if not isNil(conf.writelnHook) and msgSkipHook notin flags:
     conf.writelnHook(s)
   elif optStdout in conf.globalOptions or msgStdout in flags:
     if eStdOut in conf.m.errorOutputs:
+      flushDot(conf, stdout)
       writeLine(stdout, s)
       flushFile(stdout)
   else:
     if eStdErr in conf.m.errorOutputs:
+      flushDot(conf, stderr)
       writeLine(stderr, s)
       # On Windows stderr is fully-buffered when piped, regardless of C std.
       when defined(windows):
@@ -299,63 +339,73 @@ macro callStyledWriteLineStderr(args: varargs[typed]): untyped =
   result.add(bindSym"stderr")
   for arg in children(args[0][1]):
     result.add(arg)
+  when false:
+    # not needed because styledWriteLine already ends with resetAttributes
+    result = newStmtList(result, newCall(bindSym"resetAttributes", bindSym"stderr"))
 
 template callWritelnHook(args: varargs[string, `$`]) =
   conf.writelnHook concat(args)
 
+proc msgWrite(conf: ConfigRef; s: string) =
+  if conf.m.errorOutputs != {}:
+    let stdOrr =
+      if optStdout in conf.globalOptions:
+        stdout
+      else:
+        stderr
+    write(stdOrr, s)
+    flushFile(stdOrr)
+    conf.lastMsgWasDot = true # subsequent writes need `flushDot`
+
 template styledMsgWriteln*(args: varargs[typed]) =
   if not isNil(conf.writelnHook):
     callIgnoringStyle(callWritelnHook, nil, args)
   elif optStdout in conf.globalOptions:
     if eStdOut in conf.m.errorOutputs:
+      flushDot(conf, stdout)
       callIgnoringStyle(writeLine, stdout, args)
       flushFile(stdout)
-  else:
-    if eStdErr in conf.m.errorOutputs:
-      if optUseColors in conf.globalOptions:
-        callStyledWriteLineStderr(args)
-      else:
-        callIgnoringStyle(writeLine, stderr, args)
-      # On Windows stderr is fully-buffered when piped, regardless of C std.
-      when defined(windows):
-        flushFile(stderr)
+  elif eStdErr in conf.m.errorOutputs:
+    flushDot(conf, stderr)
+    if optUseColors in conf.globalOptions:
+      callStyledWriteLineStderr(args)
+    else:
+      callIgnoringStyle(writeLine, stderr, args)
+    # On Windows stderr is fully-buffered when piped, regardless of C std.
+    when defined(windows):
+      flushFile(stderr)
 
-proc coordToStr(coord: int): string =
-  if coord == -1: result = "???"
-  else: result = $coord
-
-proc msgKindToString*(kind: TMsgKind): string =
+proc msgKindToString*(kind: TMsgKind): string = MsgKindToStr[kind]
   # later versions may provide translated error messages
-  result = MsgKindToStr[kind]
 
-proc getMessageStr(msg: TMsgKind, arg: string): string =
-  result = msgKindToString(msg) % [arg]
+proc getMessageStr(msg: TMsgKind, arg: string): string = msgKindToString(msg) % [arg]
 
-type
-  TErrorHandling = enum doNothing, doAbort, doRaise
+type TErrorHandling* = enum doNothing, doAbort, doRaise
 
-proc log*(s: string) {.procvar.} =
+proc log*(s: string) =
   var f: File
   if open(f, getHomeDir() / "nimsuggest.log", fmAppend):
     f.writeLine(s)
     close(f)
 
 proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} =
-  if defined(debug) or msg == errInternal or hintStackTrace in conf.notes:
+  if defined(debug) or msg == errInternal or conf.hasHint(hintStackTrace):
     {.gcsafe.}:
       if stackTraceAvailable() and isNil(conf.writelnHook):
         writeStackTrace()
       else:
-        styledMsgWriteln(fgRed, "No stack traceback available\n" &
-            "To create a stacktrace, rerun compilation with ./koch temp " &
-            conf.command & " <file>")
+        styledMsgWriteln(fgRed, """
+No stack traceback available
+To create a stacktrace, rerun compilation with './koch temp $1 <file>', see $2 for details""" %
+          [conf.command, "intern.html#debugging-the-compiler".createDocLink])
   quit 1
 
 proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) =
   if msg >= fatalMin and msg <= fatalMax:
     if conf.cmd == cmdIdeTools: log(s)
     quit(conf, msg)
-  if msg >= errMin and msg <= errMax:
+  if msg >= errMin and msg <= errMax or
+      (msg in warnMin..warnMax and msg in conf.warningAsErrors):
     inc(conf.errorCounter)
     conf.exitcode = 1'i8
     if conf.errorCounter >= conf.errorMax:
@@ -375,93 +425,45 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
   const instantiationFrom = "template/generic instantiation from here"
   const instantiationOfFrom = "template/generic instantiation of `$1` from here"
   var info = lastinfo
-  for i in 0 ..< len(conf.m.msgContext):
+  for i in 0..<conf.m.msgContext.len:
     let context = conf.m.msgContext[i]
     if context.info != lastinfo and context.info != info:
       if conf.structuredErrorHook != nil:
         conf.structuredErrorHook(conf, context.info, instantiationFrom,
-                                 Severity.Error)
+                                 Severity.Hint)
       else:
         let message = if context.detail == "":
           instantiationFrom
         else:
           instantiationOfFrom.format(context.detail)
-        styledMsgWriteln(styleBright,
-                         PosFormat % [toMsgFilename(conf, context.info),
-                                      coordToStr(context.info.line.int),
-                                      coordToStr(context.info.col+ColOffset)],
-                         resetStyle,
-                         message)
+        styledMsgWriteln(styleBright, conf.toFileLineCol(context.info), " ", resetStyle, message)
     info = context.info
 
 proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
   msg >= errGenerated and conf.cmd == cmdIdeTools and optIdeDebug notin conf.globalOptions
 
-proc rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
-  var
-    title: string
-    color: ForegroundColor
-    kind: string
-    sev: Severity
-  case msg
-  of errMin..errMax:
-    sev = Severity.Error
-    writeContext(conf, unknownLineInfo())
-    title = ErrorTitle
-    color = ErrorColor
-  of warnMin..warnMax:
-    sev = Severity.Warning
-    if optWarns notin conf.options: return
-    if msg notin conf.notes: return
-    writeContext(conf, unknownLineInfo())
-    title = WarningTitle
-    color = WarningColor
-    kind = WarningsToStr[ord(msg) - ord(warnMin)]
-    inc(conf.warnCounter)
-  of hintMin..hintMax:
-    sev = Severity.Hint
-    if optHints notin conf.options: return
-    if msg notin conf.notes: return
-    title = HintTitle
-    color = HintColor
-    if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
-    inc(conf.hintCounter)
-  let s = msgKindToString(msg) % args
-
-  if conf.structuredErrorHook != nil:
-    conf.structuredErrorHook(conf, unknownLineInfo(),
-      s & (if kind.len > 0: KindFormat % kind else: ""), sev)
-
-  if not ignoreMsgBecauseOfIdeTools(conf, msg):
-    if kind.len > 0:
-      styledMsgWriteln(color, title, resetStyle, s,
-                       KindColor, `%`(KindFormat, kind))
-    else:
-      styledMsgWriteln(color, title, resetStyle, s)
-  handleError(conf, msg, doAbort, s)
-
-proc rawMessage*(conf: ConfigRef; msg: TMsgKind, arg: string) =
-  rawMessage(conf, msg, [arg])
-
-proc resetAttributes*(conf: ConfigRef) =
-  if {optUseColors, optStdout} * conf.globalOptions == {optUseColors}:
-    terminal.resetAttributes(stderr)
-
 proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
   conf.m.fileInfos[fileIdx.int32].lines.add line
 
-proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
-  if i.fileIndex.int32 < 0: return ""
-
-  if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
+proc numLines*(conf: ConfigRef, fileIdx: FileIndex): int =
+  ## xxx there's an off by 1 error that should be fixed; if a file ends with "foo" or "foo\n"
+  ## it will return same number of lines (ie, a trailing empty line is discounted)
+  result = conf.m.fileInfos[fileIdx.int32].lines.len
+  if result == 0:
     try:
-      for line in lines(toFullPathConsiderDirty(conf, i)):
-        addSourceLine conf, i.fileIndex, line.string
+      for line in lines(toFullPathConsiderDirty(conf, fileIdx).string):
+        addSourceLine conf, fileIdx, line.string
     except IOError:
       discard
-  assert i.fileIndex.int32 < conf.m.fileInfos.len
+    result = conf.m.fileInfos[fileIdx.int32].lines.len
+
+proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
+  ## 1-based index (matches editor line numbers); 1st line is for i.line = 1
+  ## last valid line is `numLines` inclusive
+  if i.fileIndex.int32 < 0: return ""
+  let num = numLines(conf, i.fileIndex)
   # can happen if the error points to EOF:
-  if i.line.int > conf.m.fileInfos[i.fileIndex.int32].lines.len: return ""
+  if i.line.int > num: return ""
 
   result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1]
 
@@ -476,101 +478,127 @@ proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): s
               of warnMin..warnMax: WarningTitle
               of hintMin..hintMax: HintTitle
               else: ErrorTitle
-  result = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                        coordToStr(info.col+ColOffset)] &
-           title &
-           getMessageStr(msg, arg)
+  conf.toFileLineCol(info) & " " & title & getMessageStr(msg, arg)
 
-proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
-               eh: TErrorHandling) =
+proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
+               eh: TErrorHandling, info2: InstantiationInfo, isRaw = false) {.noinline.} =
   var
     title: string
     color: ForegroundColor
-    kind:  string
     ignoreMsg = false
     sev: Severity
+  let kind = if msg in warnMin..hintMax and msg != hintUserRaw: $msg else: "" # xxx not sure why hintUserRaw is special
   case msg
   of errMin..errMax:
     sev = Severity.Error
     writeContext(conf, info)
     title = ErrorTitle
     color = ErrorColor
-    # we try to filter error messages so that not two error message
-    # in the same file and line are produced:
-    #ignoreMsg = lastError == info and eh != doAbort
-    conf.m.lastError = info
+    when false:
+      # we try to filter error messages so that not two error message
+      # in the same file and line are produced:
+      # xxx `lastError` is only used in this disabled code; but could be useful to revive
+      ignoreMsg = conf.m.lastError == info and info != unknownLineInfo and eh != doAbort
+    if info != unknownLineInfo: conf.m.lastError = info
   of warnMin..warnMax:
     sev = Severity.Warning
-    ignoreMsg = optWarns notin conf.options or msg notin conf.notes
+    ignoreMsg = not conf.hasWarn(msg)
+    if msg in conf.warningAsErrors:
+      ignoreMsg = false
+      title = ErrorTitle
+    else:
+      title = WarningTitle
     if not ignoreMsg: writeContext(conf, info)
-    title = WarningTitle
     color = WarningColor
-    kind = WarningsToStr[ord(msg) - ord(warnMin)]
     inc(conf.warnCounter)
   of hintMin..hintMax:
     sev = Severity.Hint
-    ignoreMsg = optHints notin conf.options or msg notin conf.notes
+    ignoreMsg = not conf.hasHint(msg)
     title = HintTitle
     color = HintColor
-    if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
     inc(conf.hintCounter)
-  let x = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                       coordToStr(info.col+ColOffset)]
-  let s = getMessageStr(msg, arg)
 
+  let s = if isRaw: arg else: getMessageStr(msg, arg)
   if not ignoreMsg:
+    let loc = if info != unknownLineInfo: conf.toFileLineCol(info) & " " else: ""
+    var kindmsg = if kind.len > 0: KindFormat % kind else: ""
     if conf.structuredErrorHook != nil:
-      conf.structuredErrorHook(conf, info, s & (if kind.len > 0: KindFormat % kind else: ""), sev)
+      conf.structuredErrorHook(conf, info, s & kindmsg, sev)
     if not ignoreMsgBecauseOfIdeTools(conf, msg):
-      if kind.len > 0:
-        styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s,
-                         KindColor, `%`(KindFormat, kind))
+      if msg == hintProcessing:
+        msgWrite(conf, ".")
       else:
-        styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s)
-      if hintSource in conf.notes:
-        conf.writeSurroundingSrc(info)
+        styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg)
+        if conf.hasHint(hintSource) and info != unknownLineInfo:
+          conf.writeSurroundingSrc(info)
+        if hintMsgOrigin in conf.mainPackageNotes:
+          styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle,
+            " compiler msg initiated here", KindColor,
+            KindFormat % $hintMsgOrigin,
+            resetStyle)
   handleError(conf, msg, eh, s)
 
-proc fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  # this fixes bug #7080 so that it is at least obvious 'fatal'
-  # was executed.
+template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
+  let arg = msgKindToString(msg) % args
+  liMessage(conf, unknownLineInfo, msg, arg, eh = doAbort, instLoc(), isRaw = true)
+
+template rawMessage*(conf: ConfigRef; msg: TMsgKind, arg: string) =
+  liMessage(conf, unknownLineInfo, msg, arg, eh = doAbort, instLoc())
+
+template fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  # this fixes bug #7080 so that it is at least obvious 'fatal' was executed.
   conf.m.errorOutputs = {eStdOut, eStdErr}
-  liMessage(conf, info, msg, arg, doAbort)
+  liMessage(conf, info, msg, arg, doAbort, instLoc())
 
-proc globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  liMessage(conf, info, msg, arg, doRaise)
+template globalAssert*(conf: ConfigRef; cond: untyped, info: TLineInfo = unknownLineInfo, arg = "") =
+  ## avoids boilerplate
+  if not cond:
+    var arg2 = "'$1' failed" % [astToStr(cond)]
+    if arg.len > 0: arg2.add "; " & astToStr(arg) & ": " & arg
+    liMessage(conf, info, errGenerated, arg2, doRaise, instLoc())
 
-proc globalError*(conf: ConfigRef; info: TLineInfo, arg: string) =
-  liMessage(conf, info, errGenerated, arg, doRaise)
+template globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  ## `local` means compilation keeps going until errorMax is reached (via `doNothing`),
+  ## `global` means it stops.
+  liMessage(conf, info, msg, arg, doRaise, instLoc())
 
-proc localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  liMessage(conf, info, msg, arg, doNothing)
+template globalError*(conf: ConfigRef; info: TLineInfo, arg: string) =
+  liMessage(conf, info, errGenerated, arg, doRaise, instLoc())
 
-proc localError*(conf: ConfigRef; info: TLineInfo, arg: string) =
-  liMessage(conf, info, errGenerated, arg, doNothing)
+template localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  liMessage(conf, info, msg, arg, doNothing, instLoc())
 
-proc localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openArray[string]) =
-  localError(conf, info, format % params)
+template localError*(conf: ConfigRef; info: TLineInfo, arg: string) =
+  liMessage(conf, info, errGenerated, arg, doNothing, instLoc())
 
-proc message*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  liMessage(conf, info, msg, arg, doNothing)
+template localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openArray[string]) =
+  liMessage(conf, info, errGenerated, format % params, doNothing, instLoc())
 
-proc internalError*(conf: ConfigRef; info: TLineInfo, errMsg: string) =
+template message*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  liMessage(conf, info, msg, arg, doNothing, instLoc())
+
+proc internalErrorImpl(conf: ConfigRef; info: TLineInfo, errMsg: string, info2: InstantiationInfo) =
   if conf.cmd == cmdIdeTools and conf.structuredErrorHook.isNil: return
   writeContext(conf, info)
-  liMessage(conf, info, errInternal, errMsg, doAbort)
+  liMessage(conf, info, errInternal, errMsg, doAbort, info2)
 
-proc internalError*(conf: ConfigRef; errMsg: string) =
-  if conf.cmd == cmdIdeTools and conf.structuredErrorHook.isNil: return
-  writeContext(conf, unknownLineInfo())
-  rawMessage(conf, errInternal, errMsg)
+template internalError*(conf: ConfigRef; info: TLineInfo, errMsg: string) =
+  internalErrorImpl(conf, info, errMsg, instLoc())
 
-template assertNotNil*(conf: ConfigRef; e): untyped =
-  if e == nil: internalError(conf, $instantiationInfo())
-  e
+template internalError*(conf: ConfigRef; errMsg: string) =
+  internalErrorImpl(conf, unknownLineInfo, errMsg, instLoc())
 
 template internalAssert*(conf: ConfigRef, e: bool) =
-  if not e: internalError(conf, $instantiationInfo())
+  # xxx merge with `globalAssert`
+  if not e:
+    const info2 = instLoc()
+    let arg = info2.toFileLineCol
+    internalErrorImpl(conf, unknownLineInfo, arg, info2)
+
+template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string) =
+  let m = "'$2' should be: '$1'" % [beau, got]
+  let msg = if optStyleError in conf.globalOptions: errGenerated else: hintName
+  liMessage(conf, info, msg, m, doNothing, instLoc())
 
 proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
   if i.fileIndex.int32 < 0:
@@ -580,25 +608,9 @@ proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
   else:
     result = conf.m.fileInfos[i.fileIndex.int32].quotedName
 
-proc listWarnings*(conf: ConfigRef) =
-  msgWriteln(conf, "Warnings:")
-  for warn in warnMin..warnMax:
-    msgWriteln(conf, "  [$1] $2" % [
-      if warn in conf.notes: "x" else: " ",
-      lineinfos.WarningsToStr[ord(warn) - ord(warnMin)]
-    ])
+template listMsg(title, r) =
+  msgWriteln(conf, title)
+  for a in r: msgWriteln(conf, "  [$1] $2" % [if a in conf.notes: "x" else: " ", $a])
 
-proc listHints*(conf: ConfigRef) =
-  msgWriteln(conf, "Hints:")
-  for hint in hintMin..hintMax:
-    msgWriteln(conf, "  [$1] $2" % [
-      if hint in conf.notes: "x" else: " ",
-      lineinfos.HintsToStr[ord(hint) - ord(hintMin)]
-    ])
-
-proc lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string) =
-  let m = "'$2' should be: '$1'" % [beau, got]
-  if optStyleError in conf.globalOptions:
-    localError(conf, info, m)
-  else:
-    message(conf, info, hintName, m)
+proc listWarnings*(conf: ConfigRef) = listMsg("Warnings:", warnMin..warnMax)
+proc listHints*(conf: ConfigRef) = listMsg("Hints:", hintMin..hintMax)

compiler/ndi.nim

@@ -17,6 +17,8 @@ type
     enabled: bool
     f: File
     buf: string
+    filename: AbsoluteFile
+    syms: seq[PSym]
 
 proc doWrite(f: var NdiFile; s: PSym; conf: ConfigRef) =
   f.buf.setLen 0
@@ -28,13 +30,20 @@ proc doWrite(f: var NdiFile; s: PSym; conf: ConfigRef) =
   f.f.writeLine("\t", toFullPath(conf, s.info), "\t", f.buf)
 
 template writeMangledName*(f: NdiFile; s: PSym; conf: ConfigRef) =
-  if f.enabled: doWrite(f, s, conf)
+  if f.enabled: f.syms.add s
 
 proc open*(f: var NdiFile; filename: AbsoluteFile; conf: ConfigRef) =
   f.enabled = not filename.isEmpty
   if f.enabled:
-    f.f = open(filename.string, fmWrite, 8000)
+    f.filename = filename
     f.buf = newStringOfCap(20)
 
-proc close*(f: var NdiFile) =
-  if f.enabled: close(f.f)
+proc close*(f: var NdiFile, conf: ConfigRef) =
+  if f.enabled:
+    f.f = open(f.filename.string, fmWrite, 8000)
+    doAssert f.f != nil, f.filename.string
+    for s in f.syms:
+      doWrite(f, s, conf)
+    close(f.f)
+    f.syms.reset
+    f.filename.reset

compiler/nim.cfg

@@ -14,15 +14,15 @@ define:nimcore
 @end
 
 define:useStdoutAsStdmsg
-define:nimOldCaseObjects
 
 @if nimHasStyleChecks:
   styleCheck:error
 @end
 
+
 #define:useNodeIds
 #gc:markAndSweep
 
-@if nimHasWarningCaseTransition:
-warning[CaseTransition]:off
+@if nimHasWarningObservableStores:
+  warning[ObservableStores]: off
 @end

compiler/nim.nim

@@ -22,9 +22,10 @@ import
   commands, options, msgs,
   extccomp, strutils, os, main, parseopt,
   idents, lineinfos, cmdlinehelper,
-  pathutils
+  pathutils, modulegraphs
 
-include nodejs
+from browsers import openDefaultBrowser
+from nodejs import findNodeJs
 
 when hasTinyCBackend:
   import tccgen
@@ -33,20 +34,6 @@ when defined(profiler) or defined(memProfiler):
   {.hint: "Profiling support is turned on!".}
   import nimprof
 
-proc prependCurDir(f: AbsoluteFile): AbsoluteFile =
-  when defined(unix):
-    if os.isAbsolute(f.string): result = f
-    else: result = AbsoluteFile("./" & f.string)
-  else:
-    result = f
-
-proc addCmdPrefix*(result: var string, kind: CmdLineKind) =
-  # consider moving this to std/parseopt
-  case kind
-  of cmdLongOption: result.add "--"
-  of cmdShortOption: result.add "-"
-  of cmdArgument, cmdEnd: discard
-
 proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
   var p = parseopt.initOptParser(cmd)
   var argsCount = 0
@@ -66,7 +53,7 @@ proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
         config.commandLine.add ':'
         config.commandLine.add p.val.quoteShell
 
-      if p.key == " ":
+      if p.key == "": # `-` was passed to indicate main project is stdin
         p.key = "-"
         if processArgument(pass, p, argsCount, config): break
       else:
@@ -77,14 +64,13 @@ proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
       if processArgument(pass, p, argsCount, config): break
   if pass == passCmd2:
     if {optRun, optWasNimscript} * config.globalOptions == {} and
-        config.arguments.len > 0 and config.command.normalize notin ["run", "e"]:
+        config.arguments.len > 0 and config.command.normalize notin ["run", "e", "r"]:
       rawMessage(config, errGenerated, errArgsNeedRunOption)
 
 proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
   let self = NimProg(
     supportsStdinFile: true,
-    processCmdLine: processCmdLine,
-    mainCommand: mainCommand
+    processCmdLine: processCmdLine
   )
   self.initDefinesProg(conf, "nim_compiler")
   if paramCount() == 0:
@@ -92,19 +78,33 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
     return
 
   self.processCmdLineAndProjectPath(conf)
-  if not self.loadConfigsAndRunMainCommand(cache, conf): return
-  if optHints in conf.options and hintGCStats in conf.notes: echo(GC_getStatistics())
+  var graph = newModuleGraph(cache, conf)
+  if not self.loadConfigsAndRunMainCommand(cache, conf, graph): return
+  mainCommand(graph)
+  if conf.hasHint(hintGCStats): echo(GC_getStatistics())
   #echo(GC_getStatistics())
   if conf.errorCounter != 0: return
   when hasTinyCBackend:
     if conf.cmd == cmdRun:
-      tccgen.run(conf.arguments)
+      tccgen.run(conf, conf.arguments)
   if optRun in conf.globalOptions:
-    var ex = quoteShell conf.absOutFile
-    if conf.cmd == cmdCompileToJS:
-      execExternalProgram(conf, findNodeJs() & " " & ex & ' ' & conf.arguments)
+    let output = conf.absOutFile
+    case conf.cmd
+    of cmdCompileToBackend:
+      var cmdPrefix = ""
+      case conf.backend
+      of backendC, backendCpp, backendObjc: discard
+      of backendJs: cmdPrefix = findNodeJs() & " "
+      else: doAssert false, $conf.backend
+      execExternalProgram(conf, cmdPrefix & output.quoteShell & ' ' & conf.arguments)
+    of cmdDoc, cmdRst2html:
+      if conf.arguments.len > 0:
+        # reserved for future use
+        rawMessage(conf, errGenerated, "'$1 cannot handle arguments" % [$conf.cmd])
+      openDefaultBrowser($output)
     else:
-      execExternalProgram(conf, ex & ' ' & conf.arguments)
+      # support as needed
+      rawMessage(conf, errGenerated, "'$1 cannot handle --run" % [$conf.cmd])
 
 when declared(GC_setMaxPause):
   GC_setMaxPause 2_000

compiler/nimblecmd.nim

@@ -48,7 +48,7 @@ proc `<`*(ver: Version, ver2: Version): bool =
   # Handling for normal versions such as "0.1.0" or "1.0".
   var sVer = string(ver).split('.')
   var sVer2 = string(ver2).split('.')
-  for i in 0..max(sVer.len, sVer2.len)-1:
+  for i in 0..<max(sVer.len, sVer2.len):
     var sVerI = 0
     if i < sVer.len:
       discard parseInt(sVer[i], sVerI)
@@ -83,7 +83,7 @@ proc getPathVersion*(p: string): tuple[name, version: string] =
       result.name = p
       return
 
-  result.name = p[0 .. sepIdx - 1]
+  result.name = p[0..sepIdx - 1]
   result.version = p.substr(sepIdx + 1)
 
 proc addPackage(conf: ConfigRef; packages: StringTableRef, p: string; info: TLineInfo) =
@@ -130,30 +130,7 @@ proc addPathRec(conf: ConfigRef; dir: string, info: TLineInfo) =
 proc nimblePath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
   addPathRec(conf, path.string, info)
   addNimblePath(conf, path.string, info)
-
-when isMainModule:
-  proc v(s: string): Version = s.newVersion
-  # #head is special in the sense that it's assumed to always be newest.
-  doAssert v"1.0" < v"#head"
-  doAssert v"1.0" < v"1.1"
-  doAssert v"1.0.1" < v"1.1"
-  doAssert v"1" < v"1.1"
-  doAssert v"#aaaqwe" < v"1.1" # We cannot assume that a branch is newer.
-  doAssert v"#a111" < v"#head"
-
-  let conf = newConfigRef()
-  var rr = newStringTable()
-  addPackage conf, rr, "irc-#a111", unknownLineInfo()
-  addPackage conf, rr, "irc-#head", unknownLineInfo()
-  addPackage conf, rr, "irc-0.1.0", unknownLineInfo()
-  #addPackage conf, rr, "irc", unknownLineInfo()
-  #addPackage conf, rr, "another", unknownLineInfo()
-  addPackage conf, rr, "another-0.1", unknownLineInfo()
-
-  addPackage conf, rr, "ab-0.1.3", unknownLineInfo()
-  addPackage conf, rr, "ab-0.1", unknownLineInfo()
-  addPackage conf, rr, "justone-1.0", unknownLineInfo()
-
-  doAssert toSeq(rr.chosen) ==
-    @["irc-#head", "another-0.1", "ab-0.1.3", "justone-1.0"]
-
+  let i = conf.nimblePaths.find(path)
+  if i != -1:
+    conf.nimblePaths.delete(i)
+  conf.nimblePaths.insert(path, 0)

compiler/nimconf.nim

@@ -11,18 +11,18 @@
 
 import
   llstream, commands, os, strutils, msgs, lexer,
-  options, idents, wordrecg, strtabs, lineinfos, pathutils
+  options, idents, wordrecg, strtabs, lineinfos, pathutils, scriptconfig
 
 # ---------------- configuration file parser -----------------------------
 # we use Nim's scanner here to save space and work
 
-proc ppGetTok(L: var TLexer, tok: var TToken) =
+proc ppGetTok(L: var Lexer, tok: var Token) =
   # simple filter
   rawGetTok(L, tok)
   while tok.tokType in {tkComment}: rawGetTok(L, tok)
 
-proc parseExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool
-proc parseAtom(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
+proc parseExpr(L: var Lexer, tok: var Token; config: ConfigRef): bool
+proc parseAtom(L: var Lexer, tok: var Token; config: ConfigRef): bool =
   if tok.tokType == tkParLe:
     ppGetTok(L, tok)
     result = parseExpr(L, tok, config)
@@ -35,21 +35,21 @@ proc parseAtom(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
     result = isDefined(config, tok.ident.s)
     ppGetTok(L, tok)
 
-proc parseAndExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
+proc parseAndExpr(L: var Lexer, tok: var Token; config: ConfigRef): bool =
   result = parseAtom(L, tok, config)
   while tok.tokType == tkAnd:
     ppGetTok(L, tok)          # skip "and"
     var b = parseAtom(L, tok, config)
     result = result and b
 
-proc parseExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
+proc parseExpr(L: var Lexer, tok: var Token; config: ConfigRef): bool =
   result = parseAndExpr(L, tok, config)
   while tok.tokType == tkOr:
     ppGetTok(L, tok)          # skip "or"
     var b = parseAndExpr(L, tok, config)
     result = result or b
 
-proc evalppIf(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
+proc evalppIf(L: var Lexer, tok: var Token; config: ConfigRef): bool =
   ppGetTok(L, tok)            # skip 'if' or 'elif'
   result = parseExpr(L, tok, config)
   if tok.tokType == tkColon: ppGetTok(L, tok)
@@ -57,7 +57,7 @@ proc evalppIf(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
 
 #var condStack: seq[bool] = @[]
 
-proc doEnd(L: var TLexer, tok: var TToken; condStack: var seq[bool]) =
+proc doEnd(L: var Lexer, tok: var Token; condStack: var seq[bool]) =
   if high(condStack) < 0: lexMessage(L, errGenerated, "expected @if")
   ppGetTok(L, tok)            # skip 'end'
   setLen(condStack, high(condStack))
@@ -66,21 +66,21 @@ type
   TJumpDest = enum
     jdEndif, jdElseEndif
 
-proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest; config: ConfigRef;
+proc jumpToDirective(L: var Lexer, tok: var Token, dest: TJumpDest; config: ConfigRef;
                      condStack: var seq[bool])
-proc doElse(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var seq[bool]) =
+proc doElse(L: var Lexer, tok: var Token; config: ConfigRef; condStack: var seq[bool]) =
   if high(condStack) < 0: lexMessage(L, errGenerated, "expected @if")
   ppGetTok(L, tok)
   if tok.tokType == tkColon: ppGetTok(L, tok)
   if condStack[high(condStack)]: jumpToDirective(L, tok, jdEndif, config, condStack)
 
-proc doElif(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var seq[bool]) =
+proc doElif(L: var Lexer, tok: var Token; config: ConfigRef; condStack: var seq[bool]) =
   if high(condStack) < 0: lexMessage(L, errGenerated, "expected @if")
   var res = evalppIf(L, tok, config)
   if condStack[high(condStack)] or not res: jumpToDirective(L, tok, jdElseEndif, config, condStack)
   else: condStack[high(condStack)] = true
 
-proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest; config: ConfigRef;
+proc jumpToDirective(L: var Lexer, tok: var Token, dest: TJumpDest; config: ConfigRef;
                      condStack: var seq[bool]) =
   var nestedIfs = 0
   while true:
@@ -110,11 +110,11 @@ proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest; config: Co
     else:
       ppGetTok(L, tok)
 
-proc parseDirective(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var seq[bool]) =
+proc parseDirective(L: var Lexer, tok: var Token; config: ConfigRef; condStack: var seq[bool]) =
   ppGetTok(L, tok)            # skip @
   case whichKeyword(tok.ident)
   of wIf:
-    setLen(condStack, len(condStack) + 1)
+    setLen(condStack, condStack.len + 1)
     let res = evalppIf(L, tok, config)
     condStack[high(condStack)] = res
     if not res: jumpToDirective(L, tok, jdElseEndif, config, condStack)
@@ -149,16 +149,16 @@ proc parseDirective(L: var TLexer, tok: var TToken; config: ConfigRef; condStack
     else:
       lexMessage(L, errGenerated, "invalid directive: '$1'" % $tok)
 
-proc confTok(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var seq[bool]) =
+proc confTok(L: var Lexer, tok: var Token; config: ConfigRef; condStack: var seq[bool]) =
   ppGetTok(L, tok)
   while tok.ident != nil and tok.ident.s == "@":
     parseDirective(L, tok, config, condStack)    # else: give the token to the parser
 
-proc checkSymbol(L: TLexer, tok: TToken) =
+proc checkSymbol(L: Lexer, tok: Token) =
   if tok.tokType notin {tkSymbol..tkInt64Lit, tkStrLit..tkTripleStrLit}:
     lexMessage(L, errGenerated, "expected identifier, but got: " & $tok)
 
-proc parseAssignment(L: var TLexer, tok: var TToken;
+proc parseAssignment(L: var Lexer, tok: var Token;
                      config: ConfigRef; condStack: var seq[bool]) =
   if tok.ident != nil:
     if tok.ident.s == "-" or tok.ident.s == "--":
@@ -169,38 +169,38 @@ proc parseAssignment(L: var TLexer, tok: var TToken;
   confTok(L, tok, config, condStack)             # skip symbol
   var val = ""
   while tok.tokType == tkDot:
-    add(s, '.')
+    s.add('.')
     confTok(L, tok, config, condStack)
     checkSymbol(L, tok)
-    add(s, $tok)
+    s.add($tok)
     confTok(L, tok, config, condStack)
   if tok.tokType == tkBracketLe:
     # BUGFIX: val, not s!
     confTok(L, tok, config, condStack)
     checkSymbol(L, tok)
-    add(val, '[')
-    add(val, $tok)
+    val.add('[')
+    val.add($tok)
     confTok(L, tok, config, condStack)
     if tok.tokType == tkBracketRi: confTok(L, tok, config, condStack)
     else: lexMessage(L, errGenerated, "expected closing ']'")
-    add(val, ']')
+    val.add(']')
   let percent = tok.ident != nil and tok.ident.s == "%="
   if tok.tokType in {tkColon, tkEquals} or percent:
-    if len(val) > 0: add(val, ':')
+    if val.len > 0: val.add(':')
     confTok(L, tok, config, condStack)           # skip ':' or '=' or '%'
     checkSymbol(L, tok)
-    add(val, $tok)
+    val.add($tok)
     confTok(L, tok, config, condStack)           # skip symbol
     if tok.tokType in {tkColon, tkEquals}:
-      add(val, $tok) # add the :
+      val.add($tok) # add the :
       confTok(L, tok, config, condStack)           # skip symbol
       checkSymbol(L, tok)
-      add(val, $tok) # add the token after it
+      val.add($tok) # add the token after it
       confTok(L, tok, config, condStack)           # skip symbol
     while tok.ident != nil and tok.ident.s == "&":
       confTok(L, tok, config, condStack)
       checkSymbol(L, tok)
-      add(val, $tok)
+      val.add($tok)
       confTok(L, tok, config, condStack)
   if percent:
     processSwitch(s, strtabs.`%`(val, config.configVars,
@@ -211,8 +211,8 @@ proc parseAssignment(L: var TLexer, tok: var TToken;
 proc readConfigFile*(filename: AbsoluteFile; cache: IdentCache;
                     config: ConfigRef): bool =
   var
-    L: TLexer
-    tok: TToken
+    L: Lexer
+    tok: Token
     stream: PLLStream
   stream = llStreamOpen(filename, fmRead)
   if stream != nil:
@@ -222,7 +222,7 @@ proc readConfigFile*(filename: AbsoluteFile; cache: IdentCache;
     var condStack: seq[bool] = @[]
     confTok(L, tok, config, condStack)           # read in the first token
     while tok.tokType != tkEof: parseAssignment(L, tok, config, condStack)
-    if len(condStack) > 0: lexMessage(L, errGenerated, "expected @end")
+    if condStack.len > 0: lexMessage(L, errGenerated, "expected @end")
     closeLexer(L)
     return true
 
@@ -246,19 +246,34 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef) =
   template readConfigFile(path) =
     let configPath = path
     if readConfigFile(configPath, cache, conf):
-      add(configFiles, configPath)
+      configFiles.add(configPath)
+
+  template runNimScriptIfExists(path: AbsoluteFile) =
+    let p = path # eval once
+    if fileExists(p):
+      configFiles.add(p)
+      runNimScript(cache, p, freshDefines = false, conf)
 
   if optSkipSystemConfigFile notin conf.globalOptions:
     readConfigFile(getSystemConfigPath(conf, cfg))
 
+    if cfg == DefaultConfig:
+      runNimScriptIfExists(getSystemConfigPath(conf, DefaultConfigNims))
+
   if optSkipUserConfigFile notin conf.globalOptions:
     readConfigFile(getUserConfigPath(cfg))
 
+    if cfg == DefaultConfig:
+      runNimScriptIfExists(getUserConfigPath(DefaultConfigNims))
+
   let pd = if not conf.projectPath.isEmpty: conf.projectPath else: AbsoluteDir(getCurrentDir())
   if optSkipParentConfigFiles notin conf.globalOptions:
     for dir in parentDirs(pd.string, fromRoot=true, inclusive=false):
       readConfigFile(AbsoluteDir(dir) / cfg)
 
+      if cfg == DefaultConfig:
+        runNimScriptIfExists(AbsoluteDir(dir) / DefaultConfigNims)
+
   if optSkipProjConfigFile notin conf.globalOptions:
     readConfigFile(pd / cfg)
 
@@ -269,6 +284,20 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef) =
         projectConfig = changeFileExt(conf.projectFull, "nim.cfg")
       readConfigFile(projectConfig)
 
+    if cfg == DefaultConfig:
+      runNimScriptIfExists(pd / DefaultConfigNims)
+
   for filename in configFiles:
     # delayed to here so that `hintConf` is honored
     rawMessage(conf, hintConf, filename.string)
+
+  block:
+    let scriptFile = conf.projectFull.changeFileExt("nims")
+    if conf.command != "nimsuggest":
+      runNimScriptIfExists(scriptFile)
+    else:
+      if scriptFile != conf.projectFull:
+        runNimScriptIfExists(scriptFile)
+      else:
+        # 'nimsuggest foo.nims' means to just auto-complete the NimScript file
+        discard

compiler/nimeval.nim

@@ -10,9 +10,9 @@
 ## exposes the Nim VM to clients.
 import
   ast, astalgo, modules, passes, condsyms,
-  options, sem, semdata, llstream, vm, vmdef,
-  modulegraphs, idents, os, pathutils, passaux,
-  scriptconfig
+  options, sem, semdata, llstream, lineinfos, vm,
+  vmdef, modulegraphs, idents, os, pathutils,
+  passaux, scriptconfig
 
 type
   Interpreter* = ref object ## Use Nim as an interpreter with this object
@@ -81,6 +81,9 @@ proc findNimStdLib*(): string =
   ## Returns "" on failure.
   try:
     let nimexe = os.findExe("nim")
+      # this can't work with choosenim shims, refs https://github.com/dom96/choosenim/issues/189
+      # it'd need `nim dump --dump.format:json . | jq -r .libpath`
+      # which we should simplify as `nim dump --key:libpath`
     if nimexe.len == 0: return ""
     result = nimexe.splitPath()[0] /../ "lib"
     if not fileExists(result / "system.nim"):
@@ -93,20 +96,22 @@ proc findNimStdLib*(): string =
 proc findNimStdLibCompileTime*(): string =
   ## Same as ``findNimStdLib`` but uses source files used at compile time,
   ## and asserts on error.
-  const sourcePath = currentSourcePath()
-  result = sourcePath.parentDir.parentDir / "lib"
+  const exe = getCurrentCompilerExe()
+  result = exe.splitFile.dir.parentDir / "lib"
   doAssert fileExists(result / "system.nim"), "result:" & result
 
 proc createInterpreter*(scriptName: string;
                         searchPaths: openArray[string];
-                        flags: TSandboxFlags = {}): Interpreter =
+                        flags: TSandboxFlags = {},
+                        defines = @[("nimscript", "true")],
+                        registerOps = true): Interpreter =
   var conf = newConfigRef()
   var cache = newIdentCache()
   var graph = newModuleGraph(cache, conf)
   connectCallbacks(graph)
   initDefines(conf.symbols)
-  defineSymbol(conf.symbols, "nimscript")
-  defineSymbol(conf.symbols, "nimconfig")
+  for define in defines:
+    defineSymbol(conf.symbols, define[0], define[1])
   registerPass(graph, semPass)
   registerPass(graph, evalPass)
 
@@ -119,6 +124,8 @@ proc createInterpreter*(scriptName: string;
   var vm = newCtx(m, cache, graph)
   vm.mode = emRepl
   vm.features = flags
+  if registerOps:
+    vm.registerAdditionalOps() # Required to register parts of stdlib modules
   graph.vm = vm
   graph.compileSystemModule()
   result = Interpreter(mainModule: m, graph: graph, scriptName: scriptName)
@@ -127,9 +134,15 @@ proc destroyInterpreter*(i: Interpreter) =
   ## destructor.
   discard "currently nothing to do."
 
+proc registerErrorHook*(i: Interpreter, hook:
+                        proc (config: ConfigRef; info: TLineInfo; msg: string;
+                              severity: Severity) {.gcsafe.}) =
+  i.graph.config.structuredErrorHook = hook
+
 proc runRepl*(r: TLLRepl;
               searchPaths: openArray[string];
               supportNimscript: bool) =
+  ## deadcode but please don't remove... might be revived
   var conf = newConfigRef()
   var cache = newIdentCache()
   var graph = newModuleGraph(cache, conf)
@@ -139,7 +152,7 @@ proc runRepl*(r: TLLRepl;
     if conf.libpath.isEmpty: conf.libpath = AbsoluteDir p
 
   conf.cmd = cmdInteractive
-  conf.errorMax = high(int)
+  conf.setErrorMaxHighMaybe
   initDefines(conf.symbols)
   defineSymbol(conf.symbols, "nimscript")
   if supportNimscript: defineSymbol(conf.symbols, "nimconfig")