revert adding a test, should have been part of 00cd7b8ca backport

When calling procs from Nim in Nimscript you could hit the VM iteration limit even though the code is functioning properly. This resolves that by making the iteration limit reset eachtime you call a proc.

(cherry picked from commit a87617956f)

.builds/freebsd.yml

@@ -0,0 +1,37 @@
+# see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd
+image: freebsd/latest
+
+# packages:
+# - databases/sqlite3
+# - devel/boehm-gc-threaded
+# - devel/pcre
+# - devel/sdl20
+# - devel/sfml
+# - www/node
+# - devel/gmake
+# - devel/git
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    # workaround https://github.com/timotheecour/Nim/issues/76
+    sudo pkg update -q -f
+    sudo pkg install -y -q databases/sqlite3 devel/boehm-gc-threaded devel/pcre \
+      devel/sdl20 devel/sfml www/node devel/gmake devel/git
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    gmake -C csources -j $(sysctl -n hw.ncpu)
+    bin/nim c --skipUserCfg --skipParentCfg koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.gitattributes

@@ -0,0 +1,6 @@
+# Avoids changelog conflicts by assuming additions-only, which is by far the common case.
+# In the rare case where branch b1 rebases against branch b2 and both branches
+# modified the same changelog entry, you'll end up with that changelog entry
+# duplicated, which is easily identifiable and fixable.
+/changelog.md merge=union
+

.github/FUNDING.yml

@@ -1,7 +1,7 @@
 # These are supported funding model platforms
 
 github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
+patreon: 'araq' # Replace with a single Patreon username
 open_collective: 'nim' # Replace with a single Open Collective username
 ko_fi: # Replace with a single Ko-fi username
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,45 +1,39 @@
 ---
 name: Bug report
 about: Have you found an unexpected behavior? Use this template.
+title: Think about the title, twice
+labels: ''
+assignees: ''
+
 ---
 
-<!-- Think about the title, twice. -->
-
-<!-- Summarize the problem here, keep it short and simple. -->
 Function `echo` outputs the wrong string.
 
-
 ### Example
-<!-- Paste your example in the code-block below. -->
 ```nim
 echo "Hello World!"
 ```
 
-
 ### Current Output
 ```
 Hola mundo!
 ```
 
-
 ### Expected Output
-<!-- What should be the correct output? -->
 ```
 Hello World!
 ```
 
-
 ### Possible Solution
-<!--- Have you found a possible solution? Post it here. -->
 
+* In file xyz there is a call that might be the cause of it.
 
 ### Additional Information
-<!--- For Example:
-* Your Nim version (output of `nim -v`).
-* Was it working in the previous Nim releases?
-* A link to a related issue or discussion.
-* A project reference where (and how) the issue causes problems.
--->
+
+* It was working in version a.b.c
+* Issue #abc is related, but different because of ...
+* This issue is blocking my project xyz
+
 ```
 $ nim -v
 Nim Compiler Version 0.1.2

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,9 @@
 ---
 name: Feature request
 about: Do you want to suggest a new feature? Use this template.
+title: ''
+labels: ''
+assignees: ''
 
 ---
 

.github/workflows/ci.yml.disabled

@@ -0,0 +1,122 @@
+name: Continous Integration
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64, i386]
+        cpp: ['false', 'true']
+        pkg: ['false', 'true']
+        exclude:
+          - os: ubuntu-18.04
+            cpp: 'true'
+          - os: ubuntu-18.04
+            pkg: 'true'
+          - os: macos-10.15
+            cpu: i386
+          - os: macos-10.15
+            pkg: 'true'
+          - os: windows-2019
+            cpu: i386
+          - os: windows-2019
+            cpp: 'true'
+    name: '${{ matrix.os }} (${{ matrix.cpu }}, cpp: ${{ matrix.cpp }}, pkg: ${{ matrix.pkg }})'
+    runs-on: ${{ matrix.os }}
+    env:
+      NIM_COMPILE_TO_CPP: ${{ matrix.cpp }}
+      NIM_TEST_PACKAGES: ${{ matrix.pkg }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install node.js 8.x'
+        uses: actions/setup-node@v1
+        with:
+          node-version: '8.x'
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -yq \
+              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
+              valgrind libc6-dbg
+      - name: 'Install dependencies (Linux i386)'
+        if: runner.os == 'Linux' && matrix.cpu == 'i386'
+        run: |
+          sudo dpkg --add-architecture i386
+
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+              g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
+              libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 \
+              libffi-dev:i386
+
+          cat << EOF > bin/gcc
+          #!/bin/bash
+
+          exec $(which gcc) -m32 "\$@"
+          EOF
+          cat << EOF > bin/g++
+          #!/bin/bash
+
+          exec $(which g++) -m32 "\$@"
+          EOF
+
+          chmod 755 bin/gcc
+          chmod 755 bin/g++
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install boehmgc make sfml
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "::add-path::${{ github.workspace }}/bin"
+
+      - name: 'Build csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+      - name: 'Run CI'
+        shell: bash
+        run: ./koch runCI
+
+      - name: 'Show failed tests'
+        if: failure()
+        shell: bash
+        run: nim c -r tools/ci_testresults.nim

.github/workflows/ci_docs.yml

@@ -0,0 +1,133 @@
+name: Nim Docs CI
+on:
+  push:
+    # Run only on changes on these files
+    paths:
+      - 'lib/**.nim'
+      - 'doc/**.rst'
+      - 'doc/nimdoc.css'
+      - '.github/workflows/ci_docs.yml'
+
+  pull_request:
+    # Run only on changes on these files
+    paths:
+      - 'lib/**.nim'
+      - 'doc/**.rst'
+      - '.github/workflows/ci_docs.yml'
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [linux, windows, osx]
+        include:
+          - target: linux
+            os: ubuntu-18.04
+          - target: windows
+            os: windows-2019
+          - target: osx
+            os: macos-10.15
+
+    name: ${{ matrix.target }}
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+
+      - name: 'Install build dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install make
+
+      - name: 'Install build dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "::add-path::${{ github.workspace }}/bin"
+
+      - name: 'Get current csources version'
+        id: csources-version
+        shell: bash
+        run: |
+          sha=$(git ls-remote https://github.com/nim-lang/csources master | cut -f 1)
+          echo "::set-output name=sha::$sha"
+
+      - name: 'Get prebuilt csources from cache'
+        id: csources-cache
+        uses: actions/cache@v1
+        with:
+          path: bin
+          key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}'
+
+      - name: 'Checkout csources'
+        if: steps.csources-cache.outputs.cache-hit != 'true'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Build 1-stage compiler from csources'
+        shell: bash
+        run: |
+          ext=
+          [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe
+          if [[ ! -x bin/nim-csources$ext ]]; then
+            ncpu=
+            case '${{ runner.os }}' in
+            'Linux')
+              ncpu=$(nproc)
+              ;;
+            'macOS')
+              ncpu=$(sysctl -n hw.ncpu)
+              ;;
+            'Windows')
+              ncpu=$NUMBER_OF_PROCESSORS
+              ;;
+            esac
+            [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+            make -C csources -j $ncpu CC=gcc
+            cp bin/nim{,-csources}$ext
+          else
+            echo 'Cache hit, using prebuilt csources'
+            cp bin/nim{-csources,}$ext
+          fi
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build the real compiler'
+        shell: bash
+        run: ./koch boot -d:release
+
+      - name: 'Build documentation'
+        shell: bash
+        run: ./koch doc --git.commit:devel
+
+      - name: 'Prepare documentation for deployment'
+        if: |
+          github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
+          matrix.target == 'linux'
+        shell: bash
+        run: cp -f doc/html/{overview,index}.html
+
+      - name: 'Publish documentation to Github Pages'
+        if: |
+          github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
+          matrix.target == 'linux'
+        uses: crazy-max/ghaction-github-pages@v1
+        with:
+          build_dir: doc/html
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci_ssl.yml

@@ -0,0 +1,91 @@
+name: Nim SSL CI
+on:
+  pull_request:
+    # Run only on changes on related files
+    paths:
+      - 'lib/pure/httpclient.nim'
+      - 'lib/pure/net.nim'
+      - 'lib/pure/ssl_certs.nim'
+      - 'lib/wrappers/openssl.nim'
+      - 'tests/stdlib/thttpclient_ssl*'
+      - 'tests/untestable/thttpclient_ssl*'
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64]
+    name: '${{ matrix.os }} (${{ matrix.cpu }})'
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -y libssl1.1
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install make
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "::add-path::${{ github.workspace }}/bin"
+
+      - name: 'Build 1-stage compiler from csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build the real compiler'
+        shell: bash
+        run: ./koch boot
+
+      - name: 'Run SSL nimDisableCertificateValidation integration tests'
+        shell: bash
+        run: nim c -d:nimDisableCertificateValidation -d:ssl -r -p:. tests/untestable/thttpclient_ssl_disabled.nim
+
+      - name: 'Run SSL certificate check integration tests'
+        # Not supported on Windows due to old openssl version
+        if: runner.os != 'Windows'
+        shell: bash
+        run: nim c -d:ssl -p:. --threads:on -r tests/untestable/thttpclient_ssl.nim

.gitignore

@@ -65,7 +65,7 @@ testament.db
 /tests/**/*.json
 /tests/**/*.js
 /csources
-dist/
+/dist/
 
 # Private directories and files (IDEs)
 .*/
@@ -84,3 +84,15 @@ megatest.nim
 /outputGotten.txt
 
 /lib/pure/*.js
+
+!/.builds/
+
+!/.github
+
+# ignore debug dirs generated by dsymutil on OSX
+*.dSYM
+
+nimdoc.out.css
+
+# for `nim c -r nimdoc/tester` etc; this can be in multiple places
+htmldocs

.travis.yml

@@ -4,45 +4,12 @@ language: c
 dist: xenial
 
 matrix:
-  fast_finish: true
-
   include:
     - os: linux
       env:
         - NIM_COMPILE_TO_CPP=false
         - CPU=amd64
 
-    - os: linux
-      env:
-        - NIM_COMPILE_TO_CPP=false
-        - CPU=i386
-      addons:
-        apt:
-          packages:
-            - gcc-multilib
-            - g++-multilib
-            - libcurl4-openssl-dev:i386
-            - libgc-dev:i386
-            - libglib2.0-dev:i386
-            - libpulse-dev:i386
-            - libsdl1.2-dev:i386
-            - libsfml-dev:i386
-
-    - os: osx
-      env:
-        - NIM_COMPILE_TO_CPP=false
-        - CPU=amd64
-
-    - os: osx
-      env:
-        - NIM_COMPILE_TO_CPP=true
-        - CPU=amd64
-
-# To allow failures for a failing configuration, use something like:
-#  allow_failures:
-#    - env: NIM_COMPILE_TO_CPP=true
-#    - os: osx
-
 addons:
   apt:
     # update the list above if more deps are introduced
@@ -51,17 +18,8 @@ addons:
     - libsdl1.2-dev
     - libgc-dev
     - libsfml-dev
-  homebrew:
-    packages:
-    - boehmgc
-    - make
-    - sfml
-
-install:
-  - if [[ $CPU = i386 ]]; then echo -e "#!/bin/bash\n$(which gcc) -m32 \"\$@\"" > bin/gcc; fi
-  - if [[ $CPU = i386 ]]; then chmod 755 bin/gcc; fi
-  - if [[ $CPU = i386 ]]; then echo -e "#!/bin/bash\n$(which g++) -m32 \"\$@\"" > bin/g++; fi
-  - if [[ $CPU = i386 ]]; then chmod 755 bin/g++; fi
+    - libc6-dbg
+    - valgrind
 
 before_script:
   - git clone --depth 1 https://github.com/nim-lang/csources.git
@@ -72,11 +30,17 @@ script:
   - echo "travis_fold:start:nim_c_koch"
   - nim c koch
   - echo "travis_fold:end:nim_c_koch"
-  - ./koch runCI
+  - echo "travis_fold:start:koch_boot"
+  - ./koch boot
+  - echo "travis_fold:end:koch_boot"
+  - echo "travis_fold:start:koch_doc"
+  - ./koch doc
+  - echo "travis_fold:end:koch_doc"
 
 before_deploy:
   # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html
   - cp -f ./doc/html/overview.html ./doc/html/index.html
+
 deploy:                         # https://nim-lang.github.io/Nim
   provider: pages
   # local-dir *has* to be a relative path from the repo root.
@@ -86,6 +50,3 @@ deploy:                         # https://nim-lang.github.io/Nim
   keep-history: false
   on:
     branch: devel
-
-# Extract failed tests
-after_failure: nim c -r tools/ci_testresults.nim

appveyor.yml

@@ -1,14 +1,11 @@
 version: '{build}'
 
 environment:
+  DLLS_URL: https://nim-lang.org/download/dlls.zip
+  DLLS_ARCHIVE: dlls.zip
   MINGW_DIR: mingw64
-  MINGW_URL: https://nim-lang.org/download/mingw64-6.3.0.7z
-  MINGW_ARCHIVE: mingw64-6.3.0.7z
-  OPENBLAS_URL: https://sourceforge.net/projects/openblas/files/v0.3.5/OpenBLAS%200.3.5%20version.zip/download
-  OPENBLAS_ARCHIVE: OpenBLAS-0.3.5.zip
-  SQLITE_URL: http://www.sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-  SQLITE_ARCHIVE: sqlite-dll-win64-x64-3160200.zip
-  platform: x64
+  MINGW_URL: https://nim-lang.org/download/mingw64.7z
+  MINGW_ARCHIVE: mingw64.7z
 
   matrix:
     - NIM_TEST_PACKAGES: false
@@ -16,36 +13,24 @@ environment:
 
 cache:
     - '%MINGW_ARCHIVE%'
-    - '%SQLITE_ARCHIVE%'
-    - '%OPENBLAS_ARCHIVE%'
+    - '%DLLS_ARCHIVE%'
 
-matrix:
-  #allow_failures:
-  #  - NIM_TEST_PACKAGES: true
-  fast_finish: true
 
 install:
   - ps: Install-Product node 8 # node 8 or later is required to test js async stuff
-  - MKDIR %CD%\DIST
-  - MKDIR %CD%\DIST\PCRE
-  - nuget install pcre -Verbosity quiet -Version 8.33.0.1 -OutputDirectory %CD%\DIST\PCRE
-  - IF not exist "%SQLITE_ARCHIVE%" appveyor DownloadFile "%SQLITE_URL%" -FileName "%SQLITE_ARCHIVE%"
-  - 7z x -y "%SQLITE_ARCHIVE%" -o"%CD%\DIST"> nul
+  - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%"
+  - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\BIN"> nul
   - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%"
   - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul
-  - IF not exist "%OPENBLAS_ARCHIVE%" appveyor DownloadFile "%OPENBLAS_URL%" -FileName "%OPENBLAS_ARCHIVE%"
-  - 7z x -y "%OPENBLAS_ARCHIVE%" -o"%CD%\DIST"> nul
   - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH%
-  - IF "%PLATFORM%" == "x64" ( copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay64.dll & copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl64.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl32.dll )
-    ELSE ( copy C:\OpenSSL-Win32\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win32\libssl32.dll %CD%\BIN\libssl32.dll )
-  - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\sqlite3.dll %CD%\BIN\sqlite3_64.dll ) ELSE ( copy %CD%\DIST\sqlite3.dll %CD%\BIN\sqlite3_32.dll )
-  - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\x64\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre64.dll ) ELSE ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\Win32\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre32.dll )
   - git clone --depth 1 https://github.com/nim-lang/csources
   - cd csources
-  - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat )
+  - build64.bat
   - cd ..
 
 build_script:
+  - openssl version
+  - openssl version -d
   - bin\nim c koch
   - koch runCI
 

azure-pipelines.yml

@@ -0,0 +1,178 @@
+trigger:
+  branches:
+    include:
+    - '*'
+pr:
+  branches:
+    include:
+    - '*'
+
+jobs:
+- job: packages
+
+  timeoutInMinutes: 90 # default `60` led to lots of cancelled jobs; use 0 for unlimited (may be undesirable)
+
+  strategy:
+    matrix:
+      Linux_amd64:
+        vmImage: 'ubuntu-16.04'
+        CPU: amd64
+      Linux_i386:
+        vmImage: 'ubuntu-16.04'
+        CPU: i386
+      OSX_amd64:
+        vmImage: 'macOS-10.15'
+        CPU: amd64
+      OSX_amd64_cpp:
+        vmImage: 'macOS-10.15'
+        CPU: amd64
+        NIM_COMPILE_TO_CPP: true
+      Windows_amd64:
+        vmImage: 'windows-2019'
+        CPU: amd64
+      Linux_amd64_pkg:
+        vmImage: 'ubuntu-16.04'
+        CPU: amd64
+        NIM_TEST_PACKAGES: true
+      OSX_amd64_pkg:
+        vmImage: 'macOS-10.15'
+        CPU: amd64
+        NIM_TEST_PACKAGES: true
+
+  pool:
+    vmImage: $(vmImage)
+
+  workspace:
+    clean: all
+
+  steps:
+    - bash: git config --global core.autocrlf false
+      displayName: 'Disable auto conversion to CRLF by git (Windows-only)'
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+    - checkout: self
+      fetchDepth: 1
+
+    - bash: git clone --depth 1 https://github.com/nim-lang/csources
+      displayName: 'Checkout Nim csources'
+
+    - task: NodeTool@0
+      inputs:
+        versionSpec: '12.x'
+      displayName: 'Install node.js 12.x'
+
+    - bash: |
+        sudo apt-fast update -qq
+        DEBIAN_FRONTEND='noninteractive' \
+          sudo apt-fast install --no-install-recommends -yq \
+            libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg
+      displayName: 'Install dependencies (amd64 Linux)'
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
+
+    - bash: |
+        sudo dpkg --add-architecture i386
+
+        # Downgrade llvm:
+        # - llvm has to be downgraded to have 32bit version installed for sfml.
+        cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
+        Package: libllvm6.0
+        Pin: origin "azure.archive.ubuntu.com"
+        Pin-Priority: 1001
+        EOF
+
+        sudo apt-fast update -qq
+        # `:i386` (e.g. in `libffi-dev:i386`) is needed otherwise you may get:
+        # `could not load: libffi.so` during dynamic loading.
+        DEBIAN_FRONTEND='noninteractive' \
+          sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+            g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
+            libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 libffi-dev:i386
+
+        cat << EOF > bin/gcc
+        #!/bin/bash
+
+        exec $(which gcc) -m32 "\$@"
+        EOF
+        cat << EOF > bin/g++
+        #!/bin/bash
+
+        exec $(which g++) -m32 "\$@"
+        EOF
+
+        chmod 755 bin/gcc
+        chmod 755 bin/g++
+
+      displayName: 'Install dependencies (i386 Linux)'
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
+
+    - bash: brew install boehmgc make sfml gtk+3
+      displayName: 'Install dependencies (OSX)'
+      condition: eq(variables['Agent.OS'], 'Darwin')
+
+    - bash: |
+        mkdir dist
+        curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+        curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+        7z x dist/mingw64.7z -odist
+        7z x dist/dlls.zip -obin
+        echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
+
+      displayName: 'Install dependencies (Windows)'
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+    - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin'
+      displayName: 'Add build binaries to PATH'
+
+    - bash: |
+        echo 'PATH:' "$PATH"
+        echo '##[section]gcc version'
+        gcc -v
+        echo '##[section]nodejs version'
+        node -v
+        echo '##[section]make version'
+        make -v
+      displayName: 'System information'
+
+    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
+      displayName: 'Get csources version'
+
+    - task: Cache@2
+      inputs:
+        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+        path: csources/bin
+      displayName: 'Restore built csources'
+
+    - bash: |
+        ncpu=
+        ext=
+        case '$(Agent.OS)' in
+        'Linux')
+          ncpu=$(nproc)
+          ;;
+        'Darwin')
+          ncpu=$(sysctl -n hw.ncpu)
+          ;;
+        'Windows_NT')
+          ncpu=$NUMBER_OF_PROCESSORS
+          ext=.exe
+          ;;
+        esac
+        [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+        if [[ -x csources/bin/nim$ext ]]; then
+          echo "Found cached compiler, skipping build"
+        else
+          make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
+        fi
+
+        cp csources/bin/nim$ext bin
+      displayName: 'Build 1-stage compiler from csources'
+
+    - bash: nim c koch
+      displayName: 'Build koch'
+
+      # set result to omit the "bash exited with error code '1'" message
+    - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
+      displayName: 'Run CI'
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)

bin/nim-gdb

@@ -6,8 +6,16 @@ set -e
 which nim > /dev/null || (echo "nim not in PATH"; exit 1)
 which gdb > /dev/null || (echo "gdb not in PATH"; exit 1)
 
+if [[ "$(uname -s)" == "Darwin" ]]; then
+  which greadlink > /dev/null || (echo "readlink not in PATH. Please install coreutils from homebrew."; exit 1)
+  READLINK=greadlink
+else
+  which readlink > /dev/null || (echo "readlink not in PATH."; exit 1)
+  READLINK=readlink
+fi
+
 # Find out where the pretty printer Python module is
-NIM_SYSROOT=$(dirname $(dirname $(readlink -e $(which nim))))
+NIM_SYSROOT=$(dirname $(dirname $($READLINK -e $(which nim))))
 GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"
 
 # Run GDB with the additional arguments that load the pretty printers
@@ -16,6 +24,4 @@ GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"
 NIM_GDB="${NIM_GDB:-gdb}"
 # exec replaces the new process of bash with gdb. It is always good to
 # have fewer processes.
-exec ${NIM_GDB} \
-  -eval-command "source $GDB_PYTHON_MODULE_PATH" \
-  "$@"
+exec "${NIM_GDB}" -eval-command="source $GDB_PYTHON_MODULE_PATH" "$@"

bin/nim-gdb.bash

@@ -0,0 +1 @@
+nim-gdb

bin/nim-gdb.bat

@@ -0,0 +1,14 @@
+@echo off
+for %%i in (nim.exe) do (set NIM_BIN=%%~dp$PATH:i)
+
+for %%i in ("%NIM_BIN%\..\") do (set NIM_ROOT=%%~fi)
+
+set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\nim-gdb.py
+set @NIM_GDB=gdb.exe
+
+@echo source %@GDB_PYTHON_MODULE_PATH%> wingdbcommand.txt
+%@NIM_GDB% --command="wingdbcommand.txt" %*
+del wingdbcommand.txt /f /q
+
+EXIT /B %ERRORLEVEL%
+@echo on

build_all.bat

@@ -0,0 +1,17 @@
+@echo off
+rem build development version of the compiler; can be rerun safely
+if not exist csources (
+  git clone --depth 1 https://github.com/nim-lang/csources.git
+)
+if not exist bin\nim.exe (
+  cd csources
+  if PROCESSOR_ARCHITECTURE == AMD64 (
+    SET ARCH=64
+  )
+  CALL build.bat
+  cd ..
+)
+bin\nim.exe c --skipUserCfg --skipParentCfg koch
+koch.exe boot -d:release --skipUserCfg --skipParentCfg
+koch.exe tools --skipUserCfg --skipParentCfg
+

build_all.sh

@@ -1,23 +1,41 @@
 #! /bin/sh
 
-# build development version of the compiler; can be rerun safely
+# build development version of the compiler; can be rerun safely.
+# arguments can be passed, eg `--os freebsd`
 
 set -u # error on undefined variables
 set -e # exit on first error
 
 echo_run(){
-  printf "\n$*\n"
+  echo "$*"
   "$@"
 }
 
-[ -d csources ] || echo_run git clone --depth 1 https://github.com/nim-lang/csources.git
+[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources.git
 
 nim_csources=bin/nim_csources
+
+build_nim_csources_via_script(){
+  echo_run cd csources
+  echo_run sh build.sh "$@"
+}
+
 build_nim_csources(){
-  ## avoid changing dir in case of failure
+  # avoid changing dir in case of failure
   (
-    echo_run cd csources
-    echo_run sh build.sh $@
+    if [ $# -ne 0 ]; then
+      # some args were passed (eg: `--cpu i386`), need to call build.sh
+      build_nim_csources_via_script "$@"
+    else
+      # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s)
+      makeX=make
+      unamestr=$(uname)
+      if [ "$unamestr" = 'FreeBSD' ]; then
+        makeX=gmake
+      fi
+      nCPU=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
+      which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script
+    fi
   )
   # keep $nim_csources in case needed to investigate bootstrap issues
   # without having to rebuild from csources
@@ -29,5 +47,6 @@ build_nim_csources(){
 # Note: if fails, may need to `cd csources && git pull`
 echo_run bin/nim c --skipUserCfg --skipParentCfg koch
 
-echo_run ./koch boot -d:release
-echo_run ./koch tools # Compile Nimble and other tools.
+echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg
+echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools.
+

changelog.md

@@ -1,33 +1,177 @@
-## v0.20.2 - XXXX-XX-XX
-
-
-### Changes affecting backwards compatibility
-
-
-#### Breaking changes in the standard library
-
-
-#### Breaking changes in the compiler
-
-
-### Library additions
-
-
-### Library changes
+# v1.4.0 - yyyy-mm-dd
 
 
 
-### Language additions
+## Standard library additions and changes
 
-### Language changes
+  For `net` and `nativesockets`, an `inheritable` flag has been added to all
+  `proc`s that create sockets, allowing the user to control whether the
+  resulting socket is inheritable. This flag is provided to ease the writing of
+  multi-process servers, where sockets inheritance is desired.
+
+  For a transistion period, define `nimInheritHandles` to enable file handle
+  inheritance by default. This flag does **not** affect the `selectors` module
+  due to the differing semantics between operating systems.
+
+  `system.setInheritable` and `nativesockets.setInheritable` is also introduced
+  for setting file handle or socket inheritance. Not all platform have these
+  `proc`s defined.
+
+- The file descriptors created for internal bookkeeping by `ioselector_kqueue`
+  and `ioselector_epoll` will no longer be leaked to child processes.
+
+- `strutils.formatFloat` with `precision = 0` has been restored to the version
+  1 behaviour that produces a trailing dot, e.g. `formatFloat(3.14159, precision = 0)`
+  is now `3.`, not `3`.
+- `critbits` adds `commonPrefixLen`.
+
+- `relativePath(rel, abs)` and `relativePath(abs, rel)` used to silently give wrong results
+  (see #13222); instead they now use `getCurrentDir` to resolve those cases,
+  and this can now throw in edge cases where `getCurrentDir` throws.
+  `relativePath` also now works for js with `-d:nodejs`.
+
+- JavaScript and NimScript standard library changes: `streams.StringStream` is
+  now supported in JavaScript, with the limitation that any buffer `pointer`s
+  used must be castable to `ptr string`, any incompatible pointer type will not
+  work. The `lexbase` and `streams` modules used to fail to compile on
+  NimScript due to a bug, but this has been fixed.
+  
+  The following modules now compile on both JS and NimScript: `parsecsv`,
+  `parsecfg`, `parsesql`, `xmlparser`, `htmlparser` and `ropes`. Additionally
+  supported for JS is `cstrutils.startsWith` and `cstrutils.endsWith`, for
+  NimScript: `json`, `parsejson`, `strtabs` and `unidecode`. 
+
+- Added `streams.readStr` and `streams.peekStr` overloads to
+  accept an existing string to modify, which avoids memory
+  allocations, similar to `streams.readLine` (#13857).
+
+- Added high-level `asyncnet.sendTo` and `asyncnet.recvFrom`. UDP functionality.
+
+- `paramCount` & `paramStr` are now defined in os.nim instead of nimscript.nim for nimscript/nimble.
+- `dollars.$` now works for unsigned ints with `nim js`
+
+- Improvements to the `bitops` module, including bitslices, non-mutating versions
+  of the original masking functions, `mask`/`masked`, and varargs support for
+  `bitand`, `bitor`, and `bitxor`.
+
+- `sugar.=>` and `sugar.->` changes: Previously `(x, y: int)` was transformed
+  into `(x: auto, y: int)`, it now becomes `(x: int, y: int)` in consistency
+  with regular proc definitions (although you cannot use semicolons).
+  
+  Pragmas and using a name are now allowed on the lefthand side of `=>`. Here
+  is an aggregate example of these changes:
+  ```nim
+  import sugar
+
+  foo(x, y: int) {.noSideEffect.} => x + y
+
+  # is transformed into
+
+  proc foo(x: int, y: int): auto {.noSideEffect.} = x + y
+  ```
+- The fields of `times.DateTime` are now private, and are accessed with getters and deprecated setters.
+
+- The `times` module now handles the default value for `DateTime` more consistently. Most procs raise an assertion error when given
+  an uninitialized `DateTime`, the exceptions are `==` and `$` (which returns `"Uninitialized DateTime"`). The proc `times.isInitialized`
+  has been added which can be used to check if a `DateTime` has been initialized.
+
+- Fix a bug where calling `close` on io streams in osproc.startProcess was a noop and led to
+  hangs if a process had both reads from stdin and writes (eg to stdout).
+
+- The callback that is passed to `system.onThreadDestruction` must now be `.raises: []`.
+- The callback that is assigned to `system.onUnhandledException` must now be `.gcsafe`.
+
+- `osproc.execCmdEx` now takes an optional `input` for stdin, `workingDir` and `env`
+  parameters.
+
+- Added a `ssl_config` module containing lists of secure ciphers as recommended by
+  [Mozilla OpSec](https://wiki.mozilla.org/Security/Server_Side_TLS)
+
+- `net.newContext` now defaults to the list of ciphers targeting
+  ["Intermediate compatibility"](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29)
+  per Mozilla's recommendation instead of `ALL`. This change should protect
+  users from the use of weak and insecure ciphers while still provides
+  adequate compatibility with the majority of the Internet.
+
+- A new module `std/jsonutils` with hookable `jsonTo,toJson,fromJson` operations for json
+  serialization/deserialization of custom types was added.
+
+- A new proc `heapqueue.find[T](heap: HeapQueue[T], x: T): int` to get index of element ``x``
+  was added.
+- Added `rstgen.rstToLatex` convenience proc for `renderRstToOut` and `initRstGenerator`
+  with `outLatex` output.
+- Added `os.normalizeExe`, e.g.: `koch` => `./koch`.
+- `macros.newLit` now preserves named vs unnamed tuples; use `-d:nimHasWorkaround14720`
+  to keep old behavior.
+- Added `random.gauss`, that uses the ratio of uniforms method of sampling from a Gaussian distribution.
+- Added `typetraits.elementType` to get element type of an iterable.
+- `typetraits.$` changes: `$(int,)` is now `"(int,)"` instead of `"(int)"`;
+  `$tuple[]` is now `"tuple[]"` instead of `"tuple"`;
+  `$((int, float), int)` is now `"((int, float), int)"` instead of `"(tuple of (int, float), int)"`
+- Added `macros.extractDocCommentsAndRunnables` helper
+
+- `strformat.fmt` and `strformat.&` support `= specifier`. `fmt"{expr=}"` now
+  expands to `fmt"expr={expr}"`.
+- deprecations: `os.existsDir` => `dirExists`, `os.existsFile` => `fileExists`
+
+- Added `jsre` module, [Regular Expressions for the JavaScript target.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions)
+- Made `maxLines` argument `Positive` in `logging.newRollingFileLogger`,
+  because negative values will result in a new file being created for each logged
+  line which doesn't make sense.
+- Changed `log` in `logging` to use proper log level on JavaScript target,
+  e.g. `debug` uses `console.debug`, `info` uses `console.info`, `warn` uses `console.warn`, etc.
+- Tables, HashSets, SharedTables and deques don't require anymore that the passed
+  initial size must be a power of two - this is done internally.
+  Proc `rightSize` for Tables and HashSets is deprecated, as it is not needed anymore.
+  `CountTable.inc` takes `val: int` again not `val: Positive`; I.e. it can "count down" again.
+- Removed deprecated symbols from `macros` module, deprecated as far back as `0.15`.
 
 
-### Tool changes
+## Language changes
+- In newruntime it is now allowed to assign discriminator field without restrictions as long as case object doesn't have custom destructor. Discriminator value doesn't have to be a constant either. If you have custom destructor for case object and you do want to freely assign discriminator fields, it is recommended to refactor object into 2 objects like this: 
+
+- The `=destroy` hook no longer has to reset its target, as the compiler now automatically inserts
+  `wasMoved` calls where needed.
+- The `=` hook is now called `=copy` for clarity. The old name `=` is still available so there
+  is no need to update your code. This change was backported to 1.2 too so you can use the
+  more readability `=copy` without loss of compatibility.
+
+- In the newruntime it is now allowed to assign to the discriminator field
+  without restrictions as long as case object doesn't have custom destructor.
+  The discriminator value doesn't have to be a constant either. If you have a
+  custom destructor for a case object and you do want to freely assign discriminator
+  fields, it is recommended to refactor object into 2 objects like this:
+
+  ```nim
+  type
+    MyObj = object
+      case kind: bool
+        of true: y: ptr UncheckedArray[float]
+        of false: z: seq[int]
+
+  proc `=destroy`(x: MyObj) =
+    if x.kind and x.y != nil:
+      deallocShared(x.y)
+      x.y = nil
+  ```
+  Refactor into:
+  ```nim
+  type
+    MySubObj = object
+      val: ptr UncheckedArray[float]
+    MyObj = object
+      case kind: bool
+      of true: y: MySubObj
+      of false: z: seq[int]
+
+  proc `=destroy`(x: MySubObj) =
+    if x.val != nil:
+      deallocShared(x.val)
+      x.val = nil
+  ```
+
+## Compiler changes
 
 
+## Tool changes
 
-### Compiler changes
-
-
-
-### Bugfixes

changelogs/changelog_0_18_1.md

@@ -100,7 +100,7 @@
 
 - We changed how array accesses "from backwards" like ``a[^1]`` or ``a[0..^1]`` are
   implemented. These are now implemented purely in ``system.nim`` without compiler
-  support. There is a new "heterogenous" slice type ``system.HSlice`` that takes 2
+  support. There is a new "heterogeneous" slice type ``system.HSlice`` that takes 2
   generic parameters which can be ``BackwardsIndex`` indices. ``BackwardsIndex`` is
   produced by ``system.^``.
   This means if you overload ``[]`` or ``[]=`` you need to ensure they also work
@@ -660,4 +660,4 @@ for i in a..b:
 - Fixed "C++: SIGABRT instead of IndexError for out-of-bounds"
   ([#6512](https://github.com/nim-lang/Nim/issues/6512))
 - Fixed "An uncaught exception in cpp mode doesn't show the exception name/msg"
-  ([#6431](https://github.com/nim-lang/Nim/issues/6431))
+  ([#6431](https://github.com/nim-lang/Nim/issues/6431))

changelogs/changelog_0_20_2.md

@@ -0,0 +1,55 @@
+# v0.20.2 - 2019-07-17
+
+
+## Changes affecting backwards compatibility
+
+- All `strutils.rfind` procs now take `start` and `last` like `strutils.find`
+  with the same data slice/index meaning. This is backwards compatible for
+  calls *not* changing the `rfind` `start` parameter from its default. (#11487)
+
+  In the unlikely case that you were using `rfind X, start=N`, or `rfind X, N`,
+  then you need to change that to `rfind X, last=N` or `rfind X, 0, N`. (This
+  should minimize gotchas porting code from other languages like Python or C++.)
+
+- On Windows stderr/stdout/stdin are not opened as binary files anymore. Use the switch
+  `-d:nimBinaryStdFiles` for a transition period.
+
+### Breaking changes in the standard library
+
+- Mac OS X / BSD: TSa_Family is now the ``uint8`` type, so type
+  conversions like ``x.sin_family = uint16 toInt(nativesockets.AF_INET)``
+  need to be changed into ``x.sin_family = TSa_Family toInt(nativesockets.AF_INET)``.
+
+
+### Breaking changes in the compiler
+
+
+## Library additions
+
+- `toOpenArray` is now available for the JS target.
+
+## Library changes
+
+- Fix async IO operations stalling even after socket is closed. (#11232)
+
+- More informative error message for `streams.openFileStream`. (#11438)
+
+
+## Language additions
+
+
+## Language changes
+
+
+### Tool changes
+
+
+### Compiler changes
+
+- Better error message for IndexError for empty containers. (#11476)
+
+- Fix regression in semfold for old right shift. (#11477)
+
+- Fix for passing tuples as static params to macros. (#11423)
+
+## Bugfixes

changelogs/changelog_1_0_0.md

@@ -0,0 +1,138 @@
+# v1.0 - 2019-09-23
+
+
+## Changes affecting backwards compatibility
+
+- The switch ``-d:nimBinaryStdFiles`` does not exist anymore. Instead
+  stdin/stdout/stderr are binary files again. This change only affects
+  Windows.
+
+- On Windows console applications the code-page is set at program startup
+  to UTF-8. Use the new switch `-d:nimDontSetUtf8CodePage` to disable this
+  feature.
+
+- The language definition and compiler are now stricter about ``gensym``'ed
+  symbols in hygienic templates. See the section in the
+  [manual](https://nim-lang.org/docs/manual.html#templates-hygiene-in-templates)
+  for further details. Use the compiler switch `--oldgensym:on` for a
+  transition period.
+
+
+### Breaking changes in the standard library
+
+- We removed `unicode.Rune16` without any deprecation period as the name
+  was wrong (see the [RFC](https://github.com/nim-lang/RFCs/issues/151) for details)
+  and we didn't find any usages of it in the wild. If you still need it, add this
+  piece of code to your project:
+```nim
+type
+  Rune16* = distinct int16
+```
+
+- `exportc` now uses C instead of C++ mangling with `nim cpp`, matching behavior
+  of `importc`, see #10578.
+  Use the new `exportcpp` to mangle as C++ when using `nim cpp`.
+
+
+### Breaking changes in the compiler
+
+- A bug allowing `int` to be implicitly converted to range types of smaller size
+  (e.g `range[0'i8..10'i8]`) has been fixed.
+
+
+
+## Library additions
+
+- `encodings.getCurrentEncoding` now distinguishes between the console's
+  encoding and the OS's encoding. This distinction is only meaningful on
+  Windows.
+- Added `system.getOsFileHandle` which is usually more useful
+  than `system.getFileHandle`. This distinction is only meaningful on
+  Windows.
+- Added a `json.parseJsonFragments` iterator that can be used to speedup
+  JSON processing substantially when there are JSON fragments separated
+  by whitespace.
+
+
+
+## Library changes
+
+- Added `os.delEnv` and `nimscript.delEnv`. (#11466)
+
+- Enabled Oid usage in hashtables. (#11472)
+
+- Added `unsafeColumnAt` procs, that return unsafe cstring from InstantRow. (#11647)
+
+- Make public `Sha1Digest` and `Sha1State` types and `newSha1State`,
+  `update` and `finalize` procedures from `sha1` module. (#11694)
+
+- Added the `std/monotimes` module which implements monotonic timestamps.
+
+- Consistent error handling of two `exec` overloads. (#10967)
+
+- Officially the following modules now have an unstable API:
+  - std/varints
+  - core/allocators
+  - core/hotcodereloading
+  - asyncstreams
+  - base64
+  - browsers
+  - collections/rtarrays
+  - collections/sharedlist
+  - collections/sharedtable
+  - concurrency/atomics
+  - concurrency/cpuload
+  - concurrency/threadpool
+  - coro
+  - endians
+  - httpcore
+  - parsesql
+  - pathnorm
+  - reservedmem
+  - typetraits
+
+  Every other stdlib module is API stable with respect to version 1.
+
+
+
+## Language additions
+
+- Inline iterators returning `lent T` types are now supported, similarly to
+  iterators returning `var T`:
+```nim
+iterator myitems[T](x: openarray[T]): lent T
+iterator mypairs[T](x: openarray[T]): tuple[idx: int, val: lent T]
+```
+
+- Added an `importjs` pragma that can now be used instead of `importcpp`
+  and `importc` to import symbols from JavaScript. `importjs` for routines always
+  takes a "pattern" for maximum flexibility.
+
+
+
+## Language changes
+
+- `uint64` is now finally a regular ordinal type. This means `high(uint64)` compiles
+  and yields the correct value.
+
+
+### Tool changes
+
+- The Nim compiler now does not recompile the Nim project via ``nim c -r`` if
+  no dependent Nim file changed. This feature can be overridden by
+  the ``--forceBuild`` command line option.
+- The Nim compiler now warns about unused module imports. You can use a
+  top level ``{.used.}`` pragma in the module that you want to be importable
+  without producing this warning.
+- The "testament" testing tool's name was changed
+  from `tester` to `testament` and is generally available as a tool to run Nim
+  tests automatically.
+
+
+### Compiler changes
+
+- VM can now cast integer type arbitrarily. (#11459)
+
+
+
+## Bugfixes

changelogs/changelog_1_0_2.md

@@ -0,0 +1,56 @@
+# v1.0.2 - 2019-10-23
+
+
+## Bugfixes
+
+* fixes the --verbosity:2 regression
+* Fixed "Fail to compile a file twice under Windows (v1.0 bug)." [#12242](https://github.com/nim-lang/Nim/issues/12242)
+* fix nimpretty removing space before pragma
+* JS: gensym is stricter for 'this'
+* Fixed "VM Assertion Error with newruntime" [#12294](https://github.com/nim-lang/Nim/issues/12294)
+* Fixed "Assertion error when running `nim check` on compiler/nim.nim" [#12281](https://github.com/nim-lang/Nim/issues/12281)
+* Fixed "Compiler crash with empty array and generic instantiation with int as parameter" [#12264](https://github.com/nim-lang/Nim/issues/12264)
+* Fixed "Regression in JS backend codegen "Error: request to generate code for .compileTime proc"" [#12240](https://github.com/nim-lang/Nim/issues/12240)
+* Fix how `relativePath` handle case sensitiviy
+* Fixed "SIGSEGV in compiler when using generic types and seqs" [#12336](https://github.com/nim-lang/Nim/issues/12336)
+* Fixed "[1.0.0] weird interaction between `import os` and casting integer to char on macosx trigger bad codegen" [#12291](https://github.com/nim-lang/Nim/issues/12291)
+* VM: no special casing for big endian machines
+* Fixed "`internal error: environment misses` with a simple template inside one of Jester macros" [#12323](https://github.com/nim-lang/Nim/issues/12323)
+* nimsuggest: fix tcp socket leak
+* nimsuggest: fix tcp socket leak for epc backend
+* Fixed "`writeFile` and `write(f, str)` skip null bytes on Windows" [#12315](https://github.com/nim-lang/Nim/issues/12315)
+* Fixed "Crash in intsets symmetric_difference" [#12366](https://github.com/nim-lang/Nim/issues/12366)
+* Fixed "[regression] VM crash when dealing with var param of a proc result" [#12244](https://github.com/nim-lang/Nim/issues/12244)
+* fixes a koch regression that made 'koch boot --listcmd' not work anymore
+* Fixed "[regression] inconsistent signed int `mod` operator between runtime, compiletime, and semfold" [#12332](https://github.com/nim-lang/Nim/issues/12332)
+* Fixed "Boehm disables interior pointer checking" [#12286](https://github.com/nim-lang/Nim/issues/12286)
+* Fixes semCustomPragma when nkSym
+* Fixed yield in nkCheckedFieldExpr
+* Fixed "`randomize()` from `random` not working on JS" [#12418](https://github.com/nim-lang/Nim/issues/12418)
+* Fixed "Compiler crash with invalid object variant" [#12379](https://github.com/nim-lang/Nim/issues/12379)
+* fix type's case in random.nim
+* Fixed "Update docs with a better way to signal unimplemented methods" [#10804](https://github.com/nim-lang/Nim/issues/10804)
+* Fixed "Nim language manual, push pragma is not explained well" [#10824](https://github.com/nim-lang/Nim/issues/10824)
+* Fixed "[regression] Importing more than one module with same name from different packages produce bad codegen" [#12420](https://github.com/nim-lang/Nim/issues/12420)
+* Namespace unittest enums to avoid name conflicts
+* Fixed "VM checks unsigned integers for overflow." [#12310](https://github.com/nim-lang/Nim/issues/12310)
+* Fixed "line directive is not generated for first line of function definition" [#12426](https://github.com/nim-lang/Nim/issues/12426)
+
+
+
+## Documentation improvements
+
+* threadpool: fix link in docs (#12258)
+* Fix spellings (#12277)
+* fix #12278, don't expose internal PCRE documentation
+* Fixed "Documentation of quitprocs is wrong" [#12279(https://github.com/nim-lang/Nim/issues/12279)
+* Fix typo in docs
+* Fix reference to parseSpec proc in readme
+* [doc/tut1] removed discard discussion in comments
+* Documentation improvements around the db interface 
+* Easier build instructions for windows - just run `build_all.bat`.
+* fix a few dead links and a missing sentence in documentation
+* Macro docs additions
+* Updated the code example in the os module to use better grammar.
+* Mention "lambdas" and `=>` in the manual
+* Better documentation on Garbage Collector

changelogs/changelog_1_2_0.md

@@ -0,0 +1,532 @@
+# v1.2.0 - 2020-04-02
+
+
+## Standard library additions and changes
+
+- Added overloaded `strformat.fmt` macro that use specified characters as
+  delimiter instead of '{' and '}'.
+- Added new procs in `tables.nim`: `OrderedTable.pop`, `CountTable.del`,
+  `CountTable.pop`, `Table.pop`.
+- Added `strtabs.clear` overload that reuses the existing mode.
+- Added `browsers.osOpen` const alias for the operating system specific *"open"* command.
+- Added `sugar.dup` for turning in-place algorithms like `sort` and `shuffle`
+  into operations that work on a copy of the data and return the mutated copy,
+  like the existing `sorted` does.
+- Added `sugar.collect` that does comprehension for seq/set/table collections.
+- Added `sugar.capture` for capturing some local loop variables when creating a
+  closure. This is an enhanced version of `closureScope`.
+- Added `typetraits.tupleLen` to get number of elements of a tuple/type tuple,
+  and `typetraits.get` to get the ith element of a type tuple.
+- Added `typetraits.genericParams` to return a tuple of generic params from a
+  generic instantiation.
+- `options` now treats `proc` like other pointer types, meaning `nil` proc variables
+  are converted to `None`.
+- Added `os.normalizePathEnd` for additional path sanitization.
+- Added `times.fromUnixFloat,toUnixFloat`, sub-second resolution versions of
+  `fromUnix`,`toUnixFloat`.
+- Added `wrapnils` module for chains of field-access and indexing where the LHS
+  can be nil. This simplifies code by reducing need for if-else branches around
+  intermediate maybe nil values. E.g. `echo ?.n.typ.kind`.
+- Added `minIndex`, `maxIndex` and `unzip` to the `sequtils` module.
+- Added `os.isRelativeTo` to tell whether a path is relative to another.
+- Added `resetOutputFormatters` to `unittest`.
+- Added `expectIdent` to the `macros` module.
+- Added `os.isValidFilename` that returns `true` if `filename` argument is valid
+  for cross-platform use.
+- Added `times.isLeapDay`
+- `base64` adds URL-Safe Base64, implements RFC-4648 Section-7.
+- Added a new module, `std / compilesettings` for querying the compiler about
+  diverse configuration settings.
+- Added `net.getPeerCertificates` and `asyncnet.getPeerCertificates` for
+  retrieving the verified certificate chain of the peer we are connected to
+  through an SSL-wrapped `Socket`/`AsyncSocket`.
+- Added `browsers.openDefaultBrowser` without URL, implements IETF RFC-6694 Section-3.
+- Added `jsconsole.trace`, `jsconsole.table`, `jsconsole.exception` for JavaScript target.
+- Added `distinctBase` overload for values: `assert 12.MyInt.distinctBase == 12`
+- Added new module `std/stackframes`, in particular `setFrameMsg`, which enables
+  custom runtime annotation of stackframes, see #13351 for examples.
+  Turn on/off via `--stackTraceMsgs:on/off`.
+- Added `sequtils.countIt`, allowing for counting items using a predicate.
+- Added a `with` macro for easy function chaining that's available everywhere,
+  there is no need to concern your APIs with returning the first argument
+  to enable "chaining", instead use the dedicated macro `with` that
+  was designed for it. For example:
+
+```nim
+type
+  Foo = object
+    col, pos: string
+
+proc setColor(f: var Foo; r, g, b: int) = f.col = $(r, g, b)
+proc setPosition(f: var Foo; x, y: float) = f.pos = $(x, y)
+
+var f: Foo
+with(f, setColor(2, 3, 4), setPosition(0.0, 1.0))
+echo f
+```
+
+- `macros.newLit` now works for ref object types.
+- `macro pragmas` can now be used in type sections.
+- 5 new pragmas were added to Nim in order to make the upcoming tooling more
+  convenient to use. Nim compiler checks these pragmas for syntax but otherwise
+  ignores them. The pragmas are `requires`, `ensures`, `assume`, `assert`, `invariant`.
+- `system.writeFile` has been overloaded to also support `openarray[byte]`.
+- `asyncdispatch.drain` now properly takes into account `selector.hasPendingOperations`
+  and only returns once all pending async operations are guaranteed to have completed.
+- `sequtils.zip` now returns a sequence of anonymous tuples i.e. those tuples
+  now do not have fields named "a" and "b".
+- `distinctBase` has been moved from `sugar` to `typetraits` and now it is
+  implemented as compiler type trait instead of macro. `distinctBase` in sugar
+  module is now deprecated.
+- `CountTable.mget` has been removed from `tables.nim`. It didn't work, and it
+  was an oversight to be included in v1.0.
+- `tables.merge(CountTable, CountTable): CountTable` has been removed.
+  It didn't work well together with the existing inplace version of the same proc
+  (`tables.merge(var CountTable, CountTable)`).
+  It was an oversight to be included in v1.0.
+- `asyncdispatch.drain` now consistently uses the passed timeout value for all
+  iterations of the event loop, and not just the first iteration.
+  This is more consistent with the other asyncdispatch APIs, and allows
+  `asyncdispatch.drain` to be more efficient.
+- `base64.encode` and `base64.decode` were made faster by about 50%.
+- `htmlgen` adds [MathML](https://wikipedia.org/wiki/MathML) support
+  (ISO 40314).
+- `macros.eqIdent` is now invariant to export markers and backtick quotes.
+- `htmlgen.html` allows `lang` in the `<html>` tag and common valid attributes.
+- `macros.basename` and `basename=` got support for `PragmaExpr`,
+  so that an expression like `MyEnum {.pure.}` is handled correctly.
+- `httpclient.maxredirects` changed from `int` to `Natural`, because negative values
+  serve no purpose whatsoever.
+- `httpclient.newHttpClient` and `httpclient.newAsyncHttpClient` added `headers`
+  argument to set initial HTTP Headers, instead of a hardcoded empty `newHttpHeader()`.
+- `parseutils.parseUntil` has now a different behaviour if the `until` parameter is
+  empty. This was required for intuitive behaviour of the strscans module
+  (see bug #13605).
+- `strutils.formatFloat` with `precision = 0` has the same behavior in all
+  backends, and it is compatible with Python's behavior,
+  e.g. `formatFloat(3.14159, precision = 0)` is now `3`, not `3.`.
+- `times.parse` now only uses input to compute its result, and not `now`:
+  `parse("2020", "YYYY", utc())` is now `2020-01-01T00:00:00Z` instead of
+  `2020-03-02T00:00:00Z` if run on 03-02; it also doesn't crash anymore when
+  used on 29th, 30th, 31st of each month.
+- `httpcore.==(string, HttpCode)` is now deprecated due to lack of practical
+  usage. The `$` operator can be used to obtain the string form of `HttpCode`
+  for comparison if desired.
+- `std/oswalkdir` was buggy, it's now deprecated and reuses `std/os` procs.
+- `net.newContext` now performs SSL Certificate checking on Linux and OSX.
+  Define `nimDisableCertificateValidation` to disable it globally.
+- `os.walkDir` and `os.walkDirRec` now have new flag, `checkDir` (default: false).
+  If it is set to true, it will throw if input dir is invalid instead of a noop
+  (which is the default behaviour, as it was before this change),
+  `os.walkDirRec` only throws if top-level dir is invalid, but ignores errors for
+  subdirs, otherwise it would be impossible to resume iteration.
+- The `FD` variant of `selector.unregister` for `ioselector_epoll` and
+  `ioselector_select` now properly handle the `Event.User` select event type.
+- `joinPath` path normalization when `/` is the first argument works correctly:
+  `assert "/" / "/a" == "/a"`. Fixed the edge case: `assert "" / "" == ""`.
+- `xmltree` now adds indentation consistently to child nodes for any number
+  of children nodes.
+- `os.splitPath()` behavior synchronized with `os.splitFile()` to return "/"
+  as the dir component of `/root_sub_dir` instead of the empty string.
+- The deprecated `lc` macro has been removed from `sugar`. It is now replaced with the
+  more powerful `collect` macro.
+- `os.relativePath("foo", "foo")` is now `"."`, not `""`, as `""` means invalid
+  path and shouldn't be conflated with `"."`; use `-d:nimOldRelativePathBehavior`
+  to restore the old behavior.
+- `os.joinPath(a, b)` now honors trailing slashes in `b` (or `a` if `b` = "").
+- `base64.encode` no longer supports `lineLen` and `newLine`.
+  Use `base64.encodeMime` instead.
+
+
+## Language changes
+
+- An `align` pragma can now be used for variables and object fields, similar
+  to the `alignas` declaration modifier in C/C++.
+- `=sink` type bound operator is now optional. Compiler can now use combination
+  of `=destroy` and `copyMem` to move objects efficiently.
+- Unsigned integer operators have been fixed to allow promotion of the first operand.
+- Conversions to unsigned integers are unchecked at runtime, imitating earlier Nim
+  versions. The documentation was improved to acknowledge this special case.
+  See https://github.com/nim-lang/RFCs/issues/175 for more details.
+- New syntax for lvalue references: `var b {.byaddr.} = expr` enabled by
+  `import std/decls`.
+- `var a {.foo.}: MyType = expr` now lowers to `foo(a, MyType, expr)` for
+  non-builtin pragmas, enabling things like lvalue references (see `decls.byaddr`).
+
+
+## Compiler changes
+
+- Generated JS code uses spaces, instead of mixed spaces and tabs.
+- The Nim compiler now supports the ``--asm`` command option for easier
+  inspection of the produced assembler code.
+- The Nim compiler now supports a new pragma called ``.localPassc`` to
+  pass specific compiler options to the C(++) backend for the C(++) file
+  that was produced from the current Nim module.
+- The compiler now inferes "sink parameters". To disable this for a specific routine,
+  annotate it with `.nosinks`. To disable it for a section of code, use
+  `{.push sinkInference: off.}`...`{.pop.}`.
+- The compiler now supports a new switch `--panics:on` that turns runtime
+  errors like `IndexError` or `OverflowError` into fatal errors that **cannot**
+  be caught via Nim's `try` statement. `--panics:on` can improve the
+  runtime efficiency and code size of your program significantly.
+- The compiler now warns about inheriting directly from `system.Exception` as
+  this is **very bad** style. You should inherit from `ValueError`, `IOError`,
+  `OSError` or from a different specific exception type that inherits from
+  `CatchableError` and cannot be confused with a `Defect`.
+- The error reporting for Nim's effect system has been improved.
+- Implicit conversions for `const` behave correctly now, meaning that code like
+  `const SOMECONST = 0.int; procThatTakesInt32(SOMECONST)` will be illegal now.
+  Simply write `const SOMECONST = 0` instead.
+- The `{.dynlib.}` pragma is now required for exporting symbols when making
+  shared objects on POSIX and macOS, which make it consistent with the behavior
+  on Windows.
+- The compiler is now more strict about type conversions concerning proc
+  types: Type conversions cannot be used to hide `.raise` effects or side
+  effects, instead a `cast` must be used. With the flag `--useVersion:1.0` the
+  old behaviour is emulated.
+- The Nim compiler now implements a faster way to detect overflows based
+  on GCC's `__builtin_sadd_overflow` family of functions. (Clang also
+  supports these). Some versions of GCC lack this feature and unfortunately
+  we cannot detect this case reliably. So if you get compilation errors like
+  "undefined reference to `__builtin_saddll_overflow`" compile your programs
+  with `-d:nimEmulateOverflowChecks`.
+
+
+## Tool changes
+
+- Nimpretty doesn't accept negative indentation argument anymore, because it was
+  breaking files.
+
+
+## Bugfixes
+
+- Fixed "`nimgrep --nocolor` is ignored on posix; should be instead: `--nimgrep --color=[auto]|true|false`"
+  ([#7591](https://github.com/nim-lang/Nim/issues/7591))
+- Fixed "Runtime index on const array (of converted obj) causes C-compiler error"
+  ([#10514](https://github.com/nim-lang/Nim/issues/10514))
+- Fixed "windows x86 with vcc compile error with "asmNoStackFrame""
+  ([#12298](https://github.com/nim-lang/Nim/issues/12298))
+- Fixed "[TODO] regression: Error: Locks requires --threads:on option"
+  ([#12330](https://github.com/nim-lang/Nim/issues/12330))
+- Fixed "Add --cc option to --help or --fullhelp output"
+  ([#12010](https://github.com/nim-lang/Nim/issues/12010))
+- Fixed "questionable `csize` definition in `system.nim`"
+  ([#12187](https://github.com/nim-lang/Nim/issues/12187))
+- Fixed "os.getAppFilename() returns incorrect results on OpenBSD"
+  ([#12389](https://github.com/nim-lang/Nim/issues/12389))
+- Fixed "HashSet[uint64] slow insertion depending on values"
+  ([#11764](https://github.com/nim-lang/Nim/issues/11764))
+- Fixed "Differences between compiling 'classic call syntax' vs 'method call syntax' ."
+  ([#12453](https://github.com/nim-lang/Nim/issues/12453))
+- Fixed "c -d:nodejs --> SIGSEGV: Illegal storage access"
+  ([#12502](https://github.com/nim-lang/Nim/issues/12502))
+- Fixed "Closure iterator crashes on --newruntime due to "dangling references""
+  ([#12443](https://github.com/nim-lang/Nim/issues/12443))
+- Fixed "No `=destroy` for elements of closure environments other than for latest devel --gc:destructors"
+  ([#12577](https://github.com/nim-lang/Nim/issues/12577))
+- Fixed "strutils:formatBiggestFloat() gives different results in JS mode"
+  ([#8242](https://github.com/nim-lang/Nim/issues/8242))
+- Fixed "Regression (devel): the new `csize_t` definition isn't consistently used, nor tested thoroughly..."
+  ([#12597](https://github.com/nim-lang/Nim/issues/12597))
+- Fixed "tables.take() is defined only for `Table` and missed for other table containers"
+  ([#12519](https://github.com/nim-lang/Nim/issues/12519))
+- Fixed "`pthread_key_t` errors on OpenBSD"
+  ([#12135](https://github.com/nim-lang/Nim/issues/12135))
+- Fixed "newruntime: simple seq pop at ct results in compile error"
+  ([#12644](https://github.com/nim-lang/Nim/issues/12644))
+- Fixed "[Windows] finish.exe C:\Users\<USERNAME>\.nimble\bin is not in your PATH environment variable."
+  ([#12319](https://github.com/nim-lang/Nim/issues/12319))
+- Fixed "Error with strformat + asyncdispatch + const"
+  ([#12612](https://github.com/nim-lang/Nim/issues/12612))
+- Fixed "MultipartData needs $"
+  ([#11863](https://github.com/nim-lang/Nim/issues/11863))
+- Fixed "Nim stdlib style issues with --styleCheck:error"
+  ([#12687](https://github.com/nim-lang/Nim/issues/12687))
+- Fixed "new $nimbleDir path substitution yields unexpected search paths"
+  ([#12767](https://github.com/nim-lang/Nim/issues/12767))
+- Fixed "Regression: inlined procs now get multiple rounds of destructor injection"
+  ([#12766](https://github.com/nim-lang/Nim/issues/12766))
+- Fixed "newruntime: compiler generates defective code"
+  ([#12669](https://github.com/nim-lang/Nim/issues/12669))
+- Fixed "broken windows modules path handling because of 'os.relativePath' breaking changes"
+  ([#12734](https://github.com/nim-lang/Nim/issues/12734))
+- Fixed "for loop tuple syntax not rendered correctly"
+  ([#12740](https://github.com/nim-lang/Nim/issues/12740))
+- Fixed "Crash when trying to use `type.name[0]`"
+  ([#12804](https://github.com/nim-lang/Nim/issues/12804))
+- Fixed "Enums should be considered Trivial types in Atomics"
+  ([#12812](https://github.com/nim-lang/Nim/issues/12812))
+- Fixed "Produce static/const initializations for variables when possible"
+  ([#12216](https://github.com/nim-lang/Nim/issues/12216))
+- Fixed "Assigning descriminator field leads to internal assert with --gc:destructors"
+  ([#12821](https://github.com/nim-lang/Nim/issues/12821))
+- Fixed "nimsuggest `use` command does not return all instances of symbol"
+  ([#12832](https://github.com/nim-lang/Nim/issues/12832))
+- Fixed "@[] is a problem for --gc:destructors"
+  ([#12820](https://github.com/nim-lang/Nim/issues/12820))
+- Fixed "Codegen ICE in allPathsAsgnResult"
+  ([#12827](https://github.com/nim-lang/Nim/issues/12827))
+- Fixed "seq[Object with ref and destructor type] doesn't work in old runtime"
+  ([#12882](https://github.com/nim-lang/Nim/issues/12882))
+- Fixed "Destructor not invoked because it is instantiated too late, old runtime"
+  ([#12883](https://github.com/nim-lang/Nim/issues/12883))
+- Fixed "The collect macro does not handle if/case correctly"
+  ([#12874](https://github.com/nim-lang/Nim/issues/12874))
+- Fixed "allow typed/untyped params in magic procs (even if not in stdlib)"
+  ([#12911](https://github.com/nim-lang/Nim/issues/12911))
+- Fixed "ARC/newruntime memory corruption"
+  ([#12899](https://github.com/nim-lang/Nim/issues/12899))
+- Fixed "tasyncclosestall.nim still flaky test: Address already in use"
+  ([#12919](https://github.com/nim-lang/Nim/issues/12919))
+- Fixed "newruntime and computed goto: variables inside the loop are in generated code uninitialised"
+  ([#12785](https://github.com/nim-lang/Nim/issues/12785))
+- Fixed "osx: dsymutil needs to be called for debug builds to keep debug info"
+  ([#12735](https://github.com/nim-lang/Nim/issues/12735))
+- Fixed "codegen ICE with ref objects, gc:destructors"
+  ([#12826](https://github.com/nim-lang/Nim/issues/12826))
+- Fixed "mutable iterator cannot yield named tuples"
+  ([#12945](https://github.com/nim-lang/Nim/issues/12945))
+- Fixed "parsecfg stores "\r\n" line breaks just as "\n""
+  ([#12970](https://github.com/nim-lang/Nim/issues/12970))
+- Fixed "db_postgres.getValue issues warning when no rows found"
+  ([#12973](https://github.com/nim-lang/Nim/issues/12973))
+- Fixed "ARC: Unpacking tuple with seq causes segfault"
+  ([#12989](https://github.com/nim-lang/Nim/issues/12989))
+- Fixed "ARC/newruntime: strutils.join on seq with only empty strings causes segfault"
+  ([#12965](https://github.com/nim-lang/Nim/issues/12965))
+- Fixed "regression (1.0.4): `{.push exportc.}` wrongly affects generic instantiations, causing codegen errors"
+  ([#12985](https://github.com/nim-lang/Nim/issues/12985))
+- Fixed "cdt, crash with --gc:arc, works fine with default gc"
+  ([#12978](https://github.com/nim-lang/Nim/issues/12978))
+- Fixed "ARC: No indexError thrown on out-of-bound seq access, SIGSEGV instead"
+  ([#12961](https://github.com/nim-lang/Nim/issues/12961))
+- Fixed "ARC/async: Returning in a try-block results in wrong codegen"
+  ([#12956](https://github.com/nim-lang/Nim/issues/12956))
+- Fixed "asm keyword is generating wrong output C code when --cc:tcc"
+  ([#12988](https://github.com/nim-lang/Nim/issues/12988))
+- Fixed "Destructor not invoked"
+  ([#13026](https://github.com/nim-lang/Nim/issues/13026))
+- Fixed "ARC/newruntime: Adding inherited var ref object to seq with base type causes segfault"
+  ([#12964](https://github.com/nim-lang/Nim/issues/12964))
+- Fixed "Style check error with JS compiler target"
+  ([#13032](https://github.com/nim-lang/Nim/issues/13032))
+- Fixed "regression(1.0.4): undeclared identifier: 'readLines'; plus another regression and bug"
+  ([#13013](https://github.com/nim-lang/Nim/issues/13013))
+- Fixed "regression(1.04) `invalid pragma: since` with nim js"
+  ([#12996](https://github.com/nim-lang/Nim/issues/12996))
+- Fixed "Sink to MemMove optimization in injectdestructors"
+  ([#13002](https://github.com/nim-lang/Nim/issues/13002))
+- Fixed "--gc:arc: `catch` doesn't work with exception subclassing"
+  ([#13072](https://github.com/nim-lang/Nim/issues/13072))
+- Fixed "nim c --gc:arc --exceptions:{setjmp,goto} incorrectly handles raise; `nim cpp --gc:arc` is ok"
+  ([#13070](https://github.com/nim-lang/Nim/issues/13070))
+- Fixed "typetraits feature request - get subtype of a generic type"
+  ([#6454](https://github.com/nim-lang/Nim/issues/6454))
+- Fixed "CountTable inconsistencies between keys() and len() after setting value to 0"
+  ([#12813](https://github.com/nim-lang/Nim/issues/12813))
+- Fixed "{.align.} pragma is not applied if there is a generic field"
+  ([#13122](https://github.com/nim-lang/Nim/issues/13122))
+- Fixed "ARC, finalizer, allow rebinding the same function multiple times"
+  ([#13112](https://github.com/nim-lang/Nim/issues/13112))
+- Fixed "`nim doc` treats `export localSymbol` incorrectly"
+  ([#13100](https://github.com/nim-lang/Nim/issues/13100))
+- Fixed "--gc:arc SIGSEGV (double free?)"
+  ([#13119](https://github.com/nim-lang/Nim/issues/13119))
+- Fixed "codegen bug with arc"
+  ([#13105](https://github.com/nim-lang/Nim/issues/13105))
+- Fixed "symbols not defined in the grammar"
+  ([#10665](https://github.com/nim-lang/Nim/issues/10665))
+- Fixed "[JS] Move is not defined"
+  ([#9674](https://github.com/nim-lang/Nim/issues/9674))
+- Fixed "[TODO] pathutils.`/` can return invalid AbsoluteFile"
+  ([#13121](https://github.com/nim-lang/Nim/issues/13121))
+- Fixed "regression(1.04) `nim doc main.nim` generates broken html (no css)"
+  ([#12998](https://github.com/nim-lang/Nim/issues/12998))
+- Fixed "Wrong supportsCopyMem on string in type section"
+  ([#13095](https://github.com/nim-lang/Nim/issues/13095))
+- Fixed "Arc, finalizer, out of memory"
+  ([#13157](https://github.com/nim-lang/Nim/issues/13157))
+- Fixed "`--genscript` messes up nimcache and future nim invocations"
+  ([#13144](https://github.com/nim-lang/Nim/issues/13144))
+- Fixed "--gc:arc with --exceptions:goto for "nim c" generate invalid c code"
+  ([#13186](https://github.com/nim-lang/Nim/issues/13186))
+- Fixed "[regression] duplicate member `_i1` codegen bug"
+  ([#13195](https://github.com/nim-lang/Nim/issues/13195))
+- Fixed "RTree investigations with valgrind for --gc:arc"
+  ([#13110](https://github.com/nim-lang/Nim/issues/13110))
+- Fixed "relativePath("foo", ".") returns wrong path"
+  ([#13211](https://github.com/nim-lang/Nim/issues/13211))
+- Fixed "asyncftpclient - problem with welcome.msg"
+  ([#4684](https://github.com/nim-lang/Nim/issues/4684))
+- Fixed "Unclear error message, lowest priority"
+  ([#13256](https://github.com/nim-lang/Nim/issues/13256))
+- Fixed "Channel messages are corrupted"
+  ([#13219](https://github.com/nim-lang/Nim/issues/13219))
+- Fixed "Codegen bug with exportc and case objects"
+  ([#13281](https://github.com/nim-lang/Nim/issues/13281))
+- Fixed "[bugfix] fix #11590: c compiler warnings silently ignored, giving undefined behavior"
+  ([#11591](https://github.com/nim-lang/Nim/issues/11591))
+- Fixed "[CI] tnetdial flaky test"
+  ([#13132](https://github.com/nim-lang/Nim/issues/13132))
+- Fixed "Cross-Compiling with -d:mingw fails to locate compiler under OSX"
+  ([#10717](https://github.com/nim-lang/Nim/issues/10717))
+- Fixed "`nim doc --project` broken with imports below main project file or duplicate names"
+  ([#13150](https://github.com/nim-lang/Nim/issues/13150))
+- Fixed "regression: isNamedTuple(MyGenericTuple[int]) is false, should be true"
+  ([#13349](https://github.com/nim-lang/Nim/issues/13349))
+- Fixed "--gc:arc codegen bug copying objects bound to C structs with missing C struct fields"
+  ([#13269](https://github.com/nim-lang/Nim/issues/13269))
+- Fixed "write requires conversion to string"
+  ([#13182](https://github.com/nim-lang/Nim/issues/13182))
+- Fixed "Some remarks to stdlib documentation"
+  ([#13352](https://github.com/nim-lang/Nim/issues/13352))
+- Fixed "a `check` in unittest generated by template doesn't show actual value"
+  ([#6736](https://github.com/nim-lang/Nim/issues/6736))
+- Fixed "Implicit return with case expression fails with 'var' return."
+  ([#3339](https://github.com/nim-lang/Nim/issues/3339))
+- Fixed "Segfault with closure on arc"
+  ([#13314](https://github.com/nim-lang/Nim/issues/13314))
+- Fixed "[Macro] Crash on malformed case statement with multiple else"
+  ([#13255](https://github.com/nim-lang/Nim/issues/13255))
+- Fixed "regression: `echo 'discard' | nim c -r -` generates a file '-' ; `-` should be treated specially"
+  ([#13374](https://github.com/nim-lang/Nim/issues/13374))
+- Fixed "on OSX, debugging (w gdb or lldb) a nim program crashes at the 1st call to `execCmdEx`"
+  ([#9634](https://github.com/nim-lang/Nim/issues/9634))
+- Fixed "Internal error in getTypeDescAux"
+  ([#13378](https://github.com/nim-lang/Nim/issues/13378))
+- Fixed "gc:arc mode breaks tuple let"
+  ([#13368](https://github.com/nim-lang/Nim/issues/13368))
+- Fixed "Nim compiler hangs for certain C/C++ compiler errors"
+  ([#8648](https://github.com/nim-lang/Nim/issues/8648))
+- Fixed "htmlgen does not support `data-*` attributes"
+  ([#13444](https://github.com/nim-lang/Nim/issues/13444))
+- Fixed "[gc:arc] setLen will cause string not to be null-terminated."
+  ([#13457](https://github.com/nim-lang/Nim/issues/13457))
+- Fixed "joinPath("", "") is "/" ; should be """
+  ([#13455](https://github.com/nim-lang/Nim/issues/13455))
+- Fixed "[CI] flaky test on windows: tests/osproc/texitcode.nim"
+  ([#13449](https://github.com/nim-lang/Nim/issues/13449))
+- Fixed "Casting to float32 on NimVM is broken"
+  ([#13479](https://github.com/nim-lang/Nim/issues/13479))
+- Fixed "`--hints:off` doesn't work (doesn't override ~/.config/nim.cfg)"
+  ([#8312](https://github.com/nim-lang/Nim/issues/8312))
+- Fixed "joinPath("", "") is "/" ; should be """
+  ([#13455](https://github.com/nim-lang/Nim/issues/13455))
+- Fixed "tables.values is broken"
+  ([#13496](https://github.com/nim-lang/Nim/issues/13496))
+- Fixed "global user config can override project specific config"
+  ([#9405](https://github.com/nim-lang/Nim/issues/9405))
+- Fixed "Non deterministic macros and id consistency problem"
+  ([#12627](https://github.com/nim-lang/Nim/issues/12627))
+- Fixed "try expression doesn't work with return on expect branch"
+  ([#13490](https://github.com/nim-lang/Nim/issues/13490))
+- Fixed "CI will break every 4 years on feb 28: times doesn't handle leap years properly"
+  ([#13543](https://github.com/nim-lang/Nim/issues/13543))
+- Fixed "[minor] `nimgrep --word` doesn't work with operators (eg misses  `1 +% 2`)"
+  ([#13528](https://github.com/nim-lang/Nim/issues/13528))
+- Fixed "`as` is usable as infix operator but its existence and precedence are not documented"
+  ([#13409](https://github.com/nim-lang/Nim/issues/13409))
+- Fixed "JSON unmarshalling drops seq's items"
+  ([#13531](https://github.com/nim-lang/Nim/issues/13531))
+- Fixed "os.joinPath returns wrong path when head ends '\' or '/' and tail starts '..'."
+  ([#13579](https://github.com/nim-lang/Nim/issues/13579))
+- Fixed "Block-local types with the same name lead to bad codegen (sighashes regression)"
+  ([#5170](https://github.com/nim-lang/Nim/issues/5170))
+- Fixed "tuple codegen error"
+  ([#12704](https://github.com/nim-lang/Nim/issues/12704))
+- Fixed "newHttpHeaders does not accept repeated headers"
+  ([#13573](https://github.com/nim-lang/Nim/issues/13573))
+- Fixed "regression: --incremental:on fails on simplest example"
+  ([#13319](https://github.com/nim-lang/Nim/issues/13319))
+- Fixed "strscan can't get value of last element in format"
+  ([#13605](https://github.com/nim-lang/Nim/issues/13605))
+- Fixed "hashes_examples crashes with "Bus Error" (unaligned access) on sparc64"
+  ([#12508](https://github.com/nim-lang/Nim/issues/12508))
+- Fixed "gc:arc bug with re-used `seq[T]`"
+  ([#13596](https://github.com/nim-lang/Nim/issues/13596))
+- Fixed "`raise CatchableError` is broken with --gc:arc  when throwing inside a proc"
+  ([#13599](https://github.com/nim-lang/Nim/issues/13599))
+- Fixed "cpp --gc:arc --exceptions:goto fails to raise with discard"
+  ([#13436](https://github.com/nim-lang/Nim/issues/13436))
+- Fixed "terminal doesn't compile with -d:useWinAnsi"
+  ([#13607](https://github.com/nim-lang/Nim/issues/13607))
+- Fixed "Parsing "sink ptr T" - region needs to be an object type"
+  ([#12757](https://github.com/nim-lang/Nim/issues/12757))
+- Fixed "gc:arc + threads:on + closures compilation error"
+  ([#13519](https://github.com/nim-lang/Nim/issues/13519))
+- Fixed "[ARC] segmentation fault"
+  ([#13240](https://github.com/nim-lang/Nim/issues/13240))
+- Fixed "times.toDateTime buggy on 29th, 30th and 31th of each month"
+  ([#13558](https://github.com/nim-lang/Nim/issues/13558))
+- Fixed "Deque misbehaves on VM"
+  ([#13310](https://github.com/nim-lang/Nim/issues/13310))
+- Fixed "Nimscript listFiles should throw exception when path is not found"
+  ([#12676](https://github.com/nim-lang/Nim/issues/12676))
+- Fixed "koch boot fails if even an empty config.nims is present in ~/.config/nims/ [devel regression]"
+  ([#13633](https://github.com/nim-lang/Nim/issues/13633))
+- Fixed "nim doc generates lots of false positive LockLevel warnings"
+  ([#13218](https://github.com/nim-lang/Nim/issues/13218))
+- Fixed "Arrays are passed by copy to iterators, causing crashes, unnecessary allocations and slowdowns"
+  ([#12747](https://github.com/nim-lang/Nim/issues/12747))
+- Fixed "Range types always uses signed integer as a base type"
+  ([#13646](https://github.com/nim-lang/Nim/issues/13646))
+- Fixed "Generate c code cannot compile with recent devel version"
+  ([#13645](https://github.com/nim-lang/Nim/issues/13645))
+- Fixed "[regression] VM: Error: cannot convert -1 to uint64"
+  ([#13661](https://github.com/nim-lang/Nim/issues/13661))
+- Fixed "Spurious raiseException(Exception) detected"
+  ([#13654](https://github.com/nim-lang/Nim/issues/13654))
+- Fixed "gc:arc memory leak"
+  ([#13659](https://github.com/nim-lang/Nim/issues/13659))
+- Fixed "Error: cannot convert -1 to uint (inside tuples)"
+  ([#13671](https://github.com/nim-lang/Nim/issues/13671))
+- Fixed "strformat issue with --gc:arc"
+  ([#13622](https://github.com/nim-lang/Nim/issues/13622))
+- Fixed "astToStr doesn't work inside generics"
+  ([#13524](https://github.com/nim-lang/Nim/issues/13524))
+- Fixed "oswalkdir.walkDirRec wont return folders"
+  ([#11458](https://github.com/nim-lang/Nim/issues/11458))
+- Fixed "`echo 'echo 1' | nim c -r -`  silently gives wrong results (nimBetterRun not updated for stdin)"
+  ([#13412](https://github.com/nim-lang/Nim/issues/13412))
+- Fixed "gc:arc destroys the global variable accidentally."
+  ([#13691](https://github.com/nim-lang/Nim/issues/13691))
+- Fixed "[minor] sigmatch errors should be sorted, for reproducible errors"
+  ([#13538](https://github.com/nim-lang/Nim/issues/13538))
+- Fixed "Exception when converting csize to clong"
+  ([#13698](https://github.com/nim-lang/Nim/issues/13698))
+- Fixed "ARC: variables are no copied on the thread spawn causing crashes"
+  ([#13708](https://github.com/nim-lang/Nim/issues/13708))
+- Fixed "Illegal distinct seq causes compiler crash"
+  ([#13720](https://github.com/nim-lang/Nim/issues/13720))
+- Fixed "cyclic seq definition crashes the compiler"
+  ([#13715](https://github.com/nim-lang/Nim/issues/13715))
+- Fixed "Iterator with openArray parameter make the argument evaluated many times"
+  ([#13417](https://github.com/nim-lang/Nim/issues/13417))
+- Fixed "net/asyncnet: Unable to access peer's certificate chain"
+  ([#13299](https://github.com/nim-lang/Nim/issues/13299))
+- Fixed "Accidentally "SIGSEGV: Illegal storage access" error after arc optimizations (#13325)"
+  ([#13709](https://github.com/nim-lang/Nim/issues/13709))
+- Fixed "Base64 Regression"
+  ([#13722](https://github.com/nim-lang/Nim/issues/13722))
+- Fixed "A regression (?) with --gc:arc and repr"
+  ([#13731](https://github.com/nim-lang/Nim/issues/13731))
+- Fixed "Internal compiler error when using the new variable pragmas"
+  ([#13737](https://github.com/nim-lang/Nim/issues/13737))
+- Fixed "bool conversion produces vcc 2019 warning at cpp compilation stage"
+  ([#13744](https://github.com/nim-lang/Nim/issues/13744))
+- Fixed "Compiler "does not detect" a type recursion error in the wrong code, remaining frozen"
+  ([#13763](https://github.com/nim-lang/Nim/issues/13763))
+- Fixed "[minor] regression: `Foo[0.0] is Foo[-0.0]` is now false"
+  ([#13730](https://github.com/nim-lang/Nim/issues/13730))
+- Fixed "`nim doc` - only whitespace on first line causes segfault"
+  ([#13631](https://github.com/nim-lang/Nim/issues/13631))
+- Fixed "hashset regression"
+  ([#13794](https://github.com/nim-lang/Nim/issues/13794))
+- Fixed "`os.getApplFreebsd` could return incorrect paths in the case of a long path"
+  ([#13806](https://github.com/nim-lang/Nim/issues/13806))
+- Fixed "Destructors are not inherited"
+  ([#13810](https://github.com/nim-lang/Nim/issues/13810))
+- Fixed "io.readLines AssertionError on devel"
+  ([#13829](https://github.com/nim-lang/Nim/issues/13829))
+- Fixed "exceptions:goto accidentally reset the variable during exception handling"
+  ([#13782](https://github.com/nim-lang/Nim/issues/13782))

changelogs/changelog_X_XX_X.md

@@ -1,27 +1,20 @@
-## v0.XX.0 - XX/XX/2018
+# v1.xx.x - yyyy-mm-dd
 
-### Changes affecting backwards compatibility
-
-- Example item: ``Foo`` changed to ``Bar``.
-
-#### Breaking changes in the standard library
+This is an example file.
+The changes should go to changelog.md!
 
 
-#### Breaking changes in the compiler
+## Standard library additions and changes
 
-### Library additions
-
-### Library changes
+- Added `example.exampleProc`.
+- Changed `example.foo` to take additional `bar` parameter.
 
 
-### Language additions
+## Language changes
 
 
-### Language changes
+## Compiler changes
 
 
-### Tool changes
+## Tool changes
 
-### Compiler changes
-
-### Bugfixes

compiler/aliases.nim

@@ -10,7 +10,7 @@
 ## Simple alias analysis for the HLO and the code generators.
 
 import
-  ast, astalgo, types, trees, intsets, msgs
+  ast, astalgo, types, trees, intsets
 
 type
   TAnalysisResult* = enum
@@ -22,17 +22,17 @@ proc isPartOfAux(n: PNode, b: PType, marker: var IntSet): TAnalysisResult =
   result = arNo
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      result = isPartOfAux(n.sons[i], b, marker)
+    for i in 0..<n.len:
+      result = isPartOfAux(n[i], b, marker)
       if result == arYes: return
   of nkRecCase:
-    assert(n.sons[0].kind == nkSym)
-    result = isPartOfAux(n.sons[0], b, marker)
+    assert(n[0].kind == nkSym)
+    result = isPartOfAux(n[0], b, marker)
     if result == arYes: return
-    for i in 1 ..< sonsLen(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = isPartOfAux(lastSon(n.sons[i]), b, marker)
+        result = isPartOfAux(lastSon(n[i]), b, marker)
         if result == arYes: return
       else: discard "isPartOfAux(record case branch)"
   of nkSym:
@@ -46,14 +46,14 @@ proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult =
   if compareTypes(a, b, dcEqIgnoreDistinct): return arYes
   case a.kind
   of tyObject:
-    if a.sons[0] != nil:
-      result = isPartOfAux(a.sons[0].skipTypes(skipPtrs), b, marker)
+    if a[0] != nil:
+      result = isPartOfAux(a[0].skipTypes(skipPtrs), b, marker)
     if result == arNo: result = isPartOfAux(a.n, b, marker)
   of tyGenericInst, tyDistinct, tyAlias, tySink:
     result = isPartOfAux(lastSon(a), b, marker)
   of tyArray, tySet, tyTuple:
-    for i in 0 ..< sonsLen(a):
-      result = isPartOfAux(a.sons[i], b, marker)
+    for i in 0..<a.len:
+      result = isPartOfAux(a[i], b, marker)
       if result == arYes: return
   else: discard
 
@@ -108,7 +108,7 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
           result = arMaybe
     of nkBracketExpr:
       result = isPartOf(a[0], b[0])
-      if len(a) >= 2 and len(b) >= 2:
+      if a.len >= 2 and b.len >= 2:
         # array accesses:
         if result == arYes and isDeepConstExpr(a[1]) and isDeepConstExpr(b[1]):
           # we know it's the same array and we have 2 constant indexes;
@@ -181,16 +181,19 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
       else: discard
     of nkObjConstr:
       result = arNo
-      for i in 1 ..< b.len:
+      for i in 1..<b.len:
         let res = isPartOf(a, b[i][1])
         if res != arNo:
           result = res
           if res == arYes: break
-    of nkCall:
+    of nkCallKinds:
       result = arNo
-      for i in 1 ..< b.len:
+      for i in 1..<b.len:
         let res = isPartOf(a, b[i])
         if res != arNo:
           result = res
           if res == arYes: break
+    of nkBracket:
+      if b.len > 0:
+        result = isPartOf(a, b[0])
     else: discard

compiler/asciitables.nim

@@ -71,7 +71,7 @@ iterator parseTableCells*(s: string, delim = '\t'): Cell =
 proc alignTable*(s: string, delim = '\t', fill = ' ', sep = " "): string =
   ## formats a `delim`-delimited `s` representing a table; each cell is aligned
   ## to a width that's computed for each column; consecutive columns are
-  ## delimted by `sep`, and alignment space is filled using `fill`.
+  ## delimited by `sep`, and alignment space is filled using `fill`.
   ## More customized formatting can be done by calling `parseTableCells` directly.
   for cell in parseTableCells(s, delim):
     result.add cell.text

compiler/astalgo.nim

@@ -24,46 +24,55 @@ proc typeToYaml*(conf: ConfigRef; n: PType, indent: int = 0, maxRecDepth: int =
 proc symToYaml*(conf: ConfigRef; n: PSym, indent: int = 0, maxRecDepth: int = - 1): Rope
 proc lineInfoToStr*(conf: ConfigRef; info: TLineInfo): Rope
 
-when declared(echo):
-  # these are for debugging only: They are not really deprecated, but I want
-  # the warning so that release versions do not contain debugging statements:
-  proc debug*(n: PSym; conf: ConfigRef = nil) {.exportc: "debugSym", deprecated.}
-  proc debug*(n: PType; conf: ConfigRef = nil) {.exportc: "debugType", deprecated.}
-  proc debug*(n: PNode; conf: ConfigRef = nil) {.exportc: "debugNode", deprecated.}
 
-  template debug*(x: PSym|PType|PNode) {.deprecated.} =
-    when compiles(c.config):
-      debug(c.config, x)
-    elif compiles(c.graph.config):
-      debug(c.graph.config, x)
+# these are for debugging only: They are not really deprecated, but I want
+# the warning so that release versions do not contain debugging statements:
+proc debug*(n: PSym; conf: ConfigRef = nil) {.exportc: "debugSym", deprecated.}
+proc debug*(n: PType; conf: ConfigRef = nil) {.exportc: "debugType", deprecated.}
+proc debug*(n: PNode; conf: ConfigRef = nil) {.exportc: "debugNode", deprecated.}
+
+proc typekinds*(t: PType) {.deprecated.} =
+  var t = t
+  var s = ""
+  while t != nil and t.len > 0:
+    s.add $t.kind
+    s.add " "
+    t = t.lastSon
+  echo s
+
+template debug*(x: PSym|PType|PNode) {.deprecated.} =
+  when compiles(c.config):
+    debug(c.config, x)
+  elif compiles(c.graph.config):
+    debug(c.graph.config, x)
+  else:
+    error()
+
+template debug*(x: auto) {.deprecated.} =
+  echo x
+
+template mdbg*: bool {.deprecated.} =
+  when compiles(c.graph):
+    c.module.fileIdx == c.graph.config.projectMainIdx
+  elif compiles(c.module):
+    c.module.fileIdx == c.config.projectMainIdx
+  elif compiles(c.c.module):
+    c.c.module.fileIdx == c.c.config.projectMainIdx
+  elif compiles(m.c.module):
+    m.c.module.fileIdx == m.c.config.projectMainIdx
+  elif compiles(cl.c.module):
+    cl.c.module.fileIdx == cl.c.config.projectMainIdx
+  elif compiles(p):
+    when compiles(p.lex):
+      p.lex.fileIdx == p.lex.config.projectMainIdx
     else:
-      error()
-
-  template debug*(x: auto) {.deprecated.} =
-    echo x
-
-  template mdbg*: bool {.deprecated.} =
-    when compiles(c.graph):
-      c.module.fileIdx == c.graph.config.projectMainIdx
-    elif compiles(c.module):
-      c.module.fileIdx == c.config.projectMainIdx
-    elif compiles(c.c.module):
-      c.c.module.fileIdx == c.c.config.projectMainIdx
-    elif compiles(m.c.module):
-      m.c.module.fileIdx == m.c.config.projectMainIdx
-    elif compiles(cl.c.module):
-      cl.c.module.fileIdx == cl.c.config.projectMainIdx
-    elif compiles(p):
-      when compiles(p.lex):
-        p.lex.fileIdx == p.lex.config.projectMainIdx
-      else:
-        p.module.module.fileIdx == p.config.projectMainIdx
-    elif compiles(m.module.fileIdx):
-      m.module.fileIdx == m.config.projectMainIdx
-    elif compiles(L.fileIdx):
-      L.fileIdx == L.config.projectMainIdx
-    else:
-      error()
+      p.module.module.fileIdx == p.config.projectMainIdx
+  elif compiles(m.module.fileIdx):
+    m.module.fileIdx == m.config.projectMainIdx
+  elif compiles(L.fileIdx):
+    L.fileIdx == L.config.projectMainIdx
+  else:
+    error()
 
 # --------------------------- ident tables ----------------------------------
 proc idTableGet*(t: TIdTable, key: PIdObj): RootRef
@@ -77,7 +86,6 @@ proc idNodeTablePut*(t: var TIdNodeTable, key: PIdObj, val: PNode)
 
 # ---------------------------------------------------------------------------
 
-proc getSymFromList*(list: PNode, ident: PIdent, start: int = 0): PSym
 proc lookupInRecord*(n: PNode, field: PIdent): PSym
 proc mustRehash*(length, counter: int): bool
 proc nextTry*(h, maxHash: Hash): Hash {.inline.}
@@ -108,16 +116,16 @@ proc skipConvAndClosure*(n: PNode): PNode =
     case result.kind
     of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64,
        nkClosure:
-      result = result.sons[0]
+      result = result[0]
     of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-      result = result.sons[1]
+      result = result[1]
     else: break
 
 proc sameValue*(a, b: PNode): bool =
   result = false
   case a.kind
   of nkCharLit..nkUInt64Lit:
-    if b.kind in {nkCharLit..nkUInt64Lit}: result = a.intVal == b.intVal
+    if b.kind in {nkCharLit..nkUInt64Lit}: result = getInt(a) == getInt(b)
   of nkFloatLit..nkFloat64Lit:
     if b.kind in {nkFloatLit..nkFloat64Lit}: result = a.floatVal == b.floatVal
   of nkStrLit..nkTripleStrLit:
@@ -131,8 +139,8 @@ proc leValue*(a, b: PNode): bool =
   # a <= b?
   result = false
   case a.kind
-  of nkCharLit..nkUInt32Lit:
-    if b.kind in {nkCharLit..nkUInt32Lit}: result = a.intVal <= b.intVal
+  of nkCharLit..nkUInt64Lit:
+    if b.kind in {nkCharLit..nkUInt64Lit}: result = getInt(a) <= getInt(b)
   of nkFloatLit..nkFloat64Lit:
     if b.kind in {nkFloatLit..nkFloat64Lit}: result = a.floatVal <= b.floatVal
   of nkStrLit..nkTripleStrLit:
@@ -152,17 +160,17 @@ proc lookupInRecord(n: PNode, field: PIdent): PSym =
   result = nil
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      result = lookupInRecord(n.sons[i], field)
+    for i in 0..<n.len:
+      result = lookupInRecord(n[i], field)
       if result != nil: return
   of nkRecCase:
-    if (n.sons[0].kind != nkSym): return nil
-    result = lookupInRecord(n.sons[0], field)
+    if (n[0].kind != nkSym): return nil
+    result = lookupInRecord(n[0], field)
     if result != nil: return
-    for i in 1 ..< sonsLen(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = lookupInRecord(lastSon(n.sons[i]), field)
+        result = lookupInRecord(lastSon(n[i]), field)
         if result != nil: return
       else: return nil
   of nkSym:
@@ -174,14 +182,55 @@ proc getModule*(s: PSym): PSym =
   assert((result.kind == skModule) or (result.owner != result))
   while result != nil and result.kind != skModule: result = result.owner
 
-proc getSymFromList(list: PNode, ident: PIdent, start: int = 0): PSym =
-  for i in start ..< sonsLen(list):
-    if list.sons[i].kind == nkSym:
-      result = list.sons[i].sym
+proc fromSystem*(op: PSym): bool {.inline.} = sfSystemModule in getModule(op).flags
+proc getSymFromList*(list: PNode, ident: PIdent, start: int = 0): PSym =
+  for i in start..<list.len:
+    if list[i].kind == nkSym:
+      result = list[i].sym
       if result.name.id == ident.id: return
     else: return nil
   result = nil
 
+proc sameIgnoreBacktickGensymInfo(a, b: string): bool =
+  if a[0] != b[0]: return false
+  var alen = a.len - 1
+  while alen > 0 and a[alen] != '`': dec(alen)
+  if alen <= 0: alen = a.len
+
+  var i = 1
+  var j = 1
+  while true:
+    while i < alen and a[i] == '_': inc i
+    while j < b.len and b[j] == '_': inc j
+    var aa = if i < alen: toLowerAscii(a[i]) else: '\0'
+    var bb = if j < b.len: toLowerAscii(b[j]) else: '\0'
+    if aa != bb: return false
+
+    # the characters are identical:
+    if i >= alen:
+      # both cursors at the end:
+      if j >= b.len: return true
+      # not yet at the end of 'b':
+      return false
+    elif j >= b.len:
+      return false
+    inc i
+    inc j
+
+proc getNamedParamFromList*(list: PNode, ident: PIdent): PSym =
+  ## Named parameters are special because a named parameter can be
+  ## gensym'ed and then they have '\`<number>' suffix that we need to
+  ## ignore, see compiler / evaltempl.nim, snippet:
+  ##
+  ##..code-block:: nim
+  ##
+  ##    result.add newIdentNode(getIdent(c.ic, x.name.s & "\`gensym" & $x.id),
+  ##            if c.instLines: actual.info else: templ.info)
+  for i in 1..<list.len:
+    let it = list[i].sym
+    if it.name.id == ident.id or
+        sameIgnoreBacktickGensymInfo(it.name.s, ident.s): return it
+
 proc hashNode(p: RootRef): Hash =
   result = hash(cast[pointer](p))
 
@@ -206,15 +255,15 @@ proc makeYamlString*(s: string): Rope =
   const MaxLineLength = 64
   result = nil
   var res = "\""
-  for i in 0 ..< s.len:
+  for i in 0..<s.len:
     if (i + 1) mod MaxLineLength == 0:
-      add(res, '\"')
-      add(res, "\n")
-      add(result, rope(res))
+      res.add('\"')
+      res.add("\n")
+      result.add(rope(res))
       res = "\""              # reset
-    add(res, toYamlChar(s[i]))
-  add(res, '\"')
-  add(result, rope(res))
+    res.add(toYamlChar(s[i]))
+  res.add('\"')
+  result.add(rope(res))
 
 proc flagsToStr[T](flags: set[T]): Rope =
   if flags == {}:
@@ -222,8 +271,8 @@ proc flagsToStr[T](flags: set[T]): Rope =
   else:
     result = nil
     for x in items(flags):
-      if result != nil: add(result, ", ")
-      add(result, makeYamlString($x))
+      if result != nil: result.add(", ")
+      result.add(makeYamlString($x))
     result = "[" & result & "]"
 
 proc lineInfoToStr(conf: ConfigRef; info: TLineInfo): Rope =
@@ -250,25 +299,25 @@ proc symToYamlAux(conf: ConfigRef; n: PSym, marker: var IntSet, indent: int,
                                  #  indent + 2, maxRecDepth - 1),
     let istr = rspaces(indent + 2)
     result = rope("{")
-    addf(result, "$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
-    addf(result, "$N$1\"name\": $2", [istr, makeYamlString(n.name.s)])
-    addf(result, "$N$1\"typ\": $2", [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
+    result.addf("$N$1\"name\": $2", [istr, makeYamlString(n.name.s)])
+    result.addf("$N$1\"typ\": $2", [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth - 1)])
     if conf != nil:
       # if we don't pass the config, we probably don't care about the line info
-      addf(result, "$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
+      result.addf("$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
     if card(n.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
-    addf(result, "$N$1\"magic\": $2", [istr, makeYamlString($n.magic)])
-    addf(result, "$N$1\"ast\": $2", [istr, ast])
-    addf(result, "$N$1\"options\": $2", [istr, flagsToStr(n.options)])
-    addf(result, "$N$1\"position\": $2", [istr, rope(n.position)])
-    addf(result, "$N$1\"k\": $2", [istr, makeYamlString($n.loc.k)])
-    addf(result, "$N$1\"storage\": $2", [istr, makeYamlString($n.loc.storage)])
+      result.addf("$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
+    result.addf("$N$1\"magic\": $2", [istr, makeYamlString($n.magic)])
+    result.addf("$N$1\"ast\": $2", [istr, ast])
+    result.addf("$N$1\"options\": $2", [istr, flagsToStr(n.options)])
+    result.addf("$N$1\"position\": $2", [istr, rope(n.position)])
+    result.addf("$N$1\"k\": $2", [istr, makeYamlString($n.loc.k)])
+    result.addf("$N$1\"storage\": $2", [istr, makeYamlString($n.loc.storage)])
     if card(n.loc.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, makeYamlString($n.loc.flags)])
-    addf(result, "$N$1\"r\": $2", [istr, n.loc.r])
-    addf(result, "$N$1\"lode\": $2", [istr, treeToYamlAux(conf, n.loc.lode, marker, indent + 2, maxRecDepth - 1)])
-    addf(result, "$N$1}", [rspaces(indent)])
+      result.addf("$N$1\"flags\": $2", [istr, makeYamlString($n.loc.flags)])
+    result.addf("$N$1\"r\": $2", [istr, n.loc.r])
+    result.addf("$N$1\"lode\": $2", [istr, treeToYamlAux(conf, n.loc.lode, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1}", [rspaces(indent)])
 
 proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int,
                    maxRecDepth: int): Rope =
@@ -279,27 +328,27 @@ proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int,
     sonsRope = "\"$1 @$2\"" % [rope($n.kind), rope(
         strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))]
   else:
-    if sonsLen(n) > 0:
+    if n.len > 0:
       sonsRope = rope("[")
-      for i in 0 ..< sonsLen(n):
-        if i > 0: add(sonsRope, ",")
-        addf(sonsRope, "$N$1$2", [rspaces(indent + 4), typeToYamlAux(conf, n.sons[i],
+      for i in 0..<n.len:
+        if i > 0: sonsRope.add(",")
+        sonsRope.addf("$N$1$2", [rspaces(indent + 4), typeToYamlAux(conf, n[i],
             marker, indent + 4, maxRecDepth - 1)])
-      addf(sonsRope, "$N$1]", [rspaces(indent + 2)])
+      sonsRope.addf("$N$1]", [rspaces(indent + 2)])
     else:
       sonsRope = rope("null")
 
     let istr = rspaces(indent + 2)
     result = rope("{")
-    addf(result, "$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
-    addf(result, "$N$1\"sym\": $2",  [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth - 1)])
-    addf(result, "$N$1\"n\": $2",     [istr, treeToYamlAux(conf, n.n, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
+    result.addf("$N$1\"sym\": $2",  [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1\"n\": $2",     [istr, treeToYamlAux(conf, n.n, marker, indent + 2, maxRecDepth - 1)])
     if card(n.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
-    addf(result, "$N$1\"callconv\": $2", [istr, makeYamlString(CallingConvToStr[n.callConv])])
-    addf(result, "$N$1\"size\": $2", [istr, rope(n.size)])
-    addf(result, "$N$1\"align\": $2", [istr, rope(n.align)])
-    addf(result, "$N$1\"sons\": $2", [istr, sonsRope])
+      result.addf("$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
+    result.addf("$N$1\"callconv\": $2", [istr, makeYamlString(CallingConvToStr[n.callConv])])
+    result.addf("$N$1\"size\": $2", [istr, rope(n.size)])
+    result.addf("$N$1\"align\": $2", [istr, rope(n.align)])
+    result.addf("$N$1\"sons\": $2", [istr, sonsRope])
 
 proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
                    maxRecDepth: int): Rope =
@@ -310,34 +359,34 @@ proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
     result = "{$N$1\"kind\": $2" % [istr, makeYamlString($n.kind)]
     if maxRecDepth != 0:
       if conf != nil:
-        addf(result, ",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
+        result.addf(",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
       case n.kind
       of nkCharLit..nkInt64Lit:
-        addf(result, ",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
+        result.addf(",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
       of nkFloatLit, nkFloat32Lit, nkFloat64Lit:
-        addf(result, ",$N$1\"floatVal\": $2",
+        result.addf(",$N$1\"floatVal\": $2",
             [istr, rope(n.floatVal.toStrMaxPrecision)])
       of nkStrLit..nkTripleStrLit:
-        addf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
+        result.addf(",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
       of nkSym:
-        addf(result, ",$N$1\"sym\": $2",
+        result.addf(",$N$1\"sym\": $2",
              [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth)])
       of nkIdent:
         if n.ident != nil:
-          addf(result, ",$N$1\"ident\": $2", [istr, makeYamlString(n.ident.s)])
+          result.addf(",$N$1\"ident\": $2", [istr, makeYamlString(n.ident.s)])
         else:
-          addf(result, ",$N$1\"ident\": null", [istr])
+          result.addf(",$N$1\"ident\": null", [istr])
       else:
-        if sonsLen(n) > 0:
-          addf(result, ",$N$1\"sons\": [", [istr])
-          for i in 0 ..< sonsLen(n):
-            if i > 0: add(result, ",")
-            addf(result, "$N$1$2", [rspaces(indent + 4), treeToYamlAux(conf, n.sons[i],
+        if n.len > 0:
+          result.addf(",$N$1\"sons\": [", [istr])
+          for i in 0..<n.len:
+            if i > 0: result.add(",")
+            result.addf("$N$1$2", [rspaces(indent + 4), treeToYamlAux(conf, n[i],
                 marker, indent + 4, maxRecDepth - 1)])
-          addf(result, "$N$1]", [istr])
-      addf(result, ",$N$1\"typ\": $2",
+          result.addf("$N$1]", [istr])
+      result.addf(",$N$1\"typ\": $2",
            [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth)])
-    addf(result, "$N$1}", [rspaces(indent)])
+    result.addf("$N$1}", [rspaces(indent)])
 
 proc treeToYaml(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope =
   var marker = initIntSet()
@@ -379,7 +428,7 @@ proc indentLess(this: var DebugPrinter) =
 proc newlineAndIndent(this: var DebugPrinter) =
   this.res.add "\n"
   this.currentLine += 1
-  for i in 0 ..< this.indent:
+  for i in 0..<this.indent:
     this.res.add ' '
 
 proc openCurly(this: var DebugPrinter) =
@@ -515,12 +564,12 @@ proc value(this: var DebugPrinter; value: PType) =
     this.key "n"
     this.value value.n
 
-  if sonsLen(value) > 0:
+  if value.len > 0:
     this.key "sons"
     this.openBracket
-    for i in 0 ..< sonsLen(value):
-      this.value value.sons[i]
-      if i != sonsLen(value) - 1:
+    for i in 0..<value.len:
+      this.value value[i]
+      if i != value.len - 1:
         this.comma
     this.closeBracket
 
@@ -568,41 +617,41 @@ proc value(this: var DebugPrinter; value: PNode) =
     if this.renderSymType and value.typ != nil:
       this.key "typ"
       this.value value.typ
-    if sonsLen(value) > 0:
+    if value.len > 0:
       this.key "sons"
       this.openBracket
-      for i in 0 ..< sonsLen(value):
-        this.value value.sons[i]
-        if i != sonsLen(value) - 1:
+      for i in 0..<value.len:
+        this.value value[i]
+        if i != value.len - 1:
           this.comma
       this.closeBracket
 
   this.closeCurly
 
-when declared(echo):
-  proc debug(n: PSym; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
 
-  proc debug(n: PType; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
+proc debug(n: PSym; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
 
-  proc debug(n: PNode; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    #this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
+proc debug(n: PType; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
+
+proc debug(n: PNode; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  #this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
 
 proc nextTry(h, maxHash: Hash): Hash =
   result = ((5 * h) + 1) and maxHash
@@ -629,13 +678,13 @@ proc objectSetRawInsert(data: var TObjectSeq, obj: RootRef) =
 
 proc objectSetEnlarge(t: var TObjectSet) =
   var n: TObjectSeq
-  newSeq(n, len(t.data) * GrowthFactor)
-  for i in 0 .. high(t.data):
+  newSeq(n, t.data.len * GrowthFactor)
+  for i in 0..high(t.data):
     if t.data[i] != nil: objectSetRawInsert(n, t.data[i])
   swap(t.data, n)
 
 proc objectSetIncl*(t: var TObjectSet, obj: RootRef) =
-  if mustRehash(len(t.data), t.counter): objectSetEnlarge(t)
+  if mustRehash(t.data.len, t.counter): objectSetEnlarge(t)
   objectSetRawInsert(t.data, obj)
   inc(t.counter)
 
@@ -648,7 +697,7 @@ proc objectSetContainsOrIncl*(t: var TObjectSet, obj: RootRef): bool =
     if it == obj:
       return true             # found it
     h = nextTry(h, high(t.data))
-  if mustRehash(len(t.data), t.counter):
+  if mustRehash(t.data.len, t.counter):
     objectSetEnlarge(t)
     objectSetRawInsert(t.data, obj)
   else:
@@ -691,13 +740,13 @@ proc symTabReplace*(t: var TStrTable, prevSym: PSym, newSym: PSym) =
 
 proc strTableEnlarge(t: var TStrTable) =
   var n: seq[PSym]
-  newSeq(n, len(t.data) * GrowthFactor)
-  for i in 0 .. high(t.data):
+  newSeq(n, t.data.len * GrowthFactor)
+  for i in 0..high(t.data):
     if t.data[i] != nil: strTableRawInsert(n, t.data[i])
   swap(t.data, n)
 
 proc strTableAdd*(t: var TStrTable, n: PSym) =
-  if mustRehash(len(t.data), t.counter): strTableEnlarge(t)
+  if mustRehash(t.data.len, t.counter): strTableEnlarge(t)
   strTableRawInsert(t.data, n)
   inc(t.counter)
 
@@ -724,7 +773,7 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym;
     if not onConflictKeepOld:
       t.data[replaceSlot] = n # overwrite it with newer definition!
     return t.data[replaceSlot] # found it
-  elif mustRehash(len(t.data), t.counter):
+  elif mustRehash(t.data.len, t.counter):
     strTableEnlarge(t)
     strTableRawInsert(t.data, n)
   else:
@@ -828,7 +877,7 @@ iterator items*(tab: TStrTable): PSym =
     s = nextIter(it, tab)
 
 proc hasEmptySlot(data: TIdPairSeq): bool =
-  for h in 0 .. high(data):
+  for h in 0..high(data):
     if data[h].key == nil:
       return true
   result = false
@@ -881,9 +930,9 @@ proc idTablePut(t: var TIdTable, key: PIdObj, val: RootRef) =
     assert(t.data[index].key != nil)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(t.data):
+    if mustRehash(t.data.len, t.counter):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(t.data):
         if t.data[i].key != nil:
           idTableRawInsert(n, t.data[i].key, t.data[i].val)
       assert(hasEmptySlot(n))
@@ -892,7 +941,7 @@ proc idTablePut(t: var TIdTable, key: PIdObj, val: RootRef) =
     inc(t.counter)
 
 iterator idTablePairs*(t: TIdTable): tuple[key: PIdObj, val: RootRef] =
-  for i in 0 .. high(t.data):
+  for i in 0..high(t.data):
     if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val)
 
 proc idNodeTableRawGet(t: TIdNodeTable, key: PIdObj): int =
@@ -926,10 +975,10 @@ proc idNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) =
     assert(t.data[index].key != nil)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
+    if mustRehash(t.data.len, t.counter):
       var n: TIdNodePairSeq
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(t.data):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(t.data):
         if t.data[i].key != nil:
           idNodeTableRawInsert(n, t.data[i].key, t.data[i].val)
       swap(t.data, n)
@@ -937,13 +986,13 @@ proc idNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) =
     inc(t.counter)
 
 iterator pairs*(t: TIdNodeTable): tuple[key: PIdObj, val: PNode] =
-  for i in 0 .. high(t.data):
+  for i in 0..high(t.data):
     if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val)
 
 proc initIITable(x: var TIITable) =
   x.counter = 0
   newSeq(x.data, StartSize)
-  for i in 0 ..< StartSize: x.data[i].key = InvalidKey
+  for i in 0..<StartSize: x.data[i].key = InvalidKey
 
 proc iiTableRawGet(t: TIITable, key: int): int =
   var h: Hash
@@ -974,11 +1023,11 @@ proc iiTablePut(t: var TIITable, key, val: int) =
     assert(t.data[index].key != InvalidKey)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
+    if mustRehash(t.data.len, t.counter):
       var n: TIIPairSeq
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(n): n[i].key = InvalidKey
-      for i in 0 .. high(t.data):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(n): n[i].key = InvalidKey
+      for i in 0..high(t.data):
         if t.data[i].key != InvalidKey:
           iiTableRawInsert(n, t.data[i].key, t.data[i].val)
       swap(t.data, n)
@@ -992,3 +1041,14 @@ proc isAddrNode*(n: PNode): bool =
       if n[0].kind == nkSym and n[0].sym.magic == mAddr: true
       else: false
     else: false
+
+proc listSymbolNames*(symbols: openArray[PSym]): string =
+  for sym in symbols:
+    if result.len > 0:
+      result.add ", "
+    result.add sym.name.s
+
+proc isDiscriminantField*(n: PNode): bool =
+  if n.kind == nkCheckedFieldExpr: sfDiscriminant in n[0][1].sym.flags
+  elif n.kind == nkDotExpr: sfDiscriminant in n[1].sym.flags
+  else: false

compiler/bitsets.nim

@@ -11,12 +11,17 @@
 # the code here should be reused in the Nim standard library
 
 type
-  TBitSet* = seq[int8]        # we use byte here to avoid issues with
+  ElemType = byte
+  TBitSet* = seq[ElemType]    # we use byte here to avoid issues with
                               # cross-compiling; uint would be more efficient
                               # however
-
 const
-  ElemSize* = sizeof(int8) * 8
+  ElemSize* = 8
+  One = ElemType(1)
+  Zero = ElemType(0)
+
+template modElemSize(arg: untyped): untyped = arg and 7
+template divElemSize(arg: untyped): untyped = arg shr 3
 
 proc bitSetInit*(b: var TBitSet, length: int)
 proc bitSetUnion*(x: var TBitSet, y: TBitSet)
@@ -32,48 +37,47 @@ proc bitSetCard*(x: TBitSet): BiggestInt
 # implementation
 
 proc bitSetIn(x: TBitSet, e: BiggestInt): bool =
-  result = (x[int(e div ElemSize)] and toU8(int(1 shl (e mod ElemSize)))) !=
-      toU8(0)
+  result = (x[int(e.divElemSize)] and (One shl e.modElemSize)) != Zero
 
 proc bitSetIncl(x: var TBitSet, elem: BiggestInt) =
   assert(elem >= 0)
-  x[int(elem div ElemSize)] = x[int(elem div ElemSize)] or
-      toU8(int(1 shl (elem mod ElemSize)))
+  x[int(elem.divElemSize)] = x[int(elem.divElemSize)] or
+      (One shl elem.modElemSize)
 
 proc bitSetExcl(x: var TBitSet, elem: BiggestInt) =
-  x[int(elem div ElemSize)] = x[int(elem div ElemSize)] and
-      not toU8(int(1 shl (elem mod ElemSize)))
+  x[int(elem.divElemSize)] = x[int(elem.divElemSize)] and
+      not(One shl elem.modElemSize)
 
 proc bitSetInit(b: var TBitSet, length: int) =
   newSeq(b, length)
 
 proc bitSetUnion(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] or y[i]
+  for i in 0..high(x): x[i] = x[i] or y[i]
 
 proc bitSetDiff(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] and not y[i]
+  for i in 0..high(x): x[i] = x[i] and not y[i]
 
 proc bitSetSymDiff(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] xor y[i]
+  for i in 0..high(x): x[i] = x[i] xor y[i]
 
 proc bitSetIntersect(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] and y[i]
+  for i in 0..high(x): x[i] = x[i] and y[i]
 
 proc bitSetEquals(x, y: TBitSet): bool =
-  for i in 0 .. high(x):
+  for i in 0..high(x):
     if x[i] != y[i]:
       return false
   result = true
 
 proc bitSetContains(x, y: TBitSet): bool =
-  for i in 0 .. high(x):
-    if (x[i] and not y[i]) != int8(0):
+  for i in 0..high(x):
+    if (x[i] and not y[i]) != Zero:
       return false
   result = true
 
 # Number of set bits for all values of int8
-const populationCount: array[low(int8)..high(int8), int8] = block:
-    var arr: array[low(int8)..high(int8), int8]
+const populationCount: array[uint8, uint8] = block:
+    var arr: array[uint8, uint8]
 
     proc countSetBits(x: uint8): uint8 =
       return
@@ -87,11 +91,11 @@ const populationCount: array[low(int8)..high(int8), int8] = block:
         ((x and 0b10000000'u8) shr 7)
 
 
-    for it in low(int8)..high(int8):
-      arr[it] = cast[int8](countSetBits(cast[uint8](it)))
+    for it in low(uint8)..high(uint8):
+      arr[it] = countSetBits(cast[uint8](it))
 
     arr
 
 proc bitSetCard(x: TBitSet): BiggestInt =
   for it in x:
-    result.inc populationCount[it]
+    result.inc int(populationCount[it])

compiler/btrees.nim

@@ -37,34 +37,34 @@ template eq(a, b): bool = cmp(a, b) == 0
 proc getOrDefault*[Key, Val](b: BTree[Key, Val], key: Key): Val =
   var x = b.root
   while x.isInternal:
-    for j in 0 ..< x.entries:
+    for j in 0..<x.entries:
       if j+1 == x.entries or less(key, x.keys[j+1]):
         x = x.links[j]
         break
   assert(not x.isInternal)
-  for j in 0 ..< x.entries:
+  for j in 0..<x.entries:
     if eq(key, x.keys[j]): return x.vals[j]
 
 proc contains*[Key, Val](b: BTree[Key, Val], key: Key): bool =
   var x = b.root
   while x.isInternal:
-    for j in 0 ..< x.entries:
+    for j in 0..<x.entries:
       if j+1 == x.entries or less(key, x.keys[j+1]):
         x = x.links[j]
         break
   assert(not x.isInternal)
-  for j in 0 ..< x.entries:
+  for j in 0..<x.entries:
     if eq(key, x.keys[j]): return true
   return false
 
 proc copyHalf[Key, Val](h, result: Node[Key, Val]) =
-  for j in 0 ..< Mhalf:
+  for j in 0..<Mhalf:
     result.keys[j] = h.keys[Mhalf + j]
   if h.isInternal:
-    for j in 0 ..< Mhalf:
+    for j in 0..<Mhalf:
       result.links[j] = h.links[Mhalf + j]
   else:
-    for j in 0 ..< Mhalf:
+    for j in 0..<Mhalf:
       shallowCopy(result.vals[j], h.vals[Mhalf + j])
 
 proc split[Key, Val](h: Node[Key, Val]): Node[Key, Val] =
@@ -191,7 +191,7 @@ when isMainModule:
     st.add("www.yahoo.com",        "216.109.118.65")
 
     assert st.getOrDefault("www.cs.princeton.edu") == "abc"
-    assert st.getOrDefault("www.harvardsucks.com") == nil
+    assert st.getOrDefault("www.harvardsucks.com") == ""
 
     assert st.getOrDefault("www.simpsons.com") == "209.052.165.60"
     assert st.getOrDefault("www.apple.com") == "17.112.152.32"

compiler/canonicalizer.nim

@@ -102,7 +102,7 @@ proc hashTree(c: var MD5Context, n: PNode) =
   of nkStrLit..nkTripleStrLit:
     c &= n.strVal
   else:
-    for i in 0..<n.len: hashTree(c, n.sons[i])
+    for i in 0..<n.len: hashTree(c, n[i])
 
 proc hashType(c: var MD5Context, t: PType) =
   # modelled after 'typeToString'
@@ -120,44 +120,44 @@ proc hashType(c: var MD5Context, t: PType) =
 
   case t.kind
   of tyGenericBody, tyGenericInst, tyGenericInvocation:
-    for i in 0 ..< sonsLen(t)-ord(t.kind != tyGenericInvocation):
-      c.hashType t.sons[i]
+    for i in 0..<t.len-ord(t.kind != tyGenericInvocation):
+      c.hashType t[i]
   of tyUserTypeClass:
     internalAssert t.sym != nil and t.sym.owner != nil
     c &= t.sym.owner.name.s
   of tyUserTypeClassInst:
     let body = t.base
     c.hashSym body.sym
-    for i in 1 .. sonsLen(t) - 2:
-      c.hashType t.sons[i]
+    for i in 1..<t.len-1:
+      c.hashType t[i]
   of tyFromExpr:
     c.hashTree(t.n)
   of tyArray:
-    c.hashTree(t.sons[0].n)
-    c.hashType(t.sons[1])
+    c.hashTree(t[0].n)
+    c.hashType(t[1])
   of tyTuple:
     if t.n != nil:
-      assert(sonsLen(t.n) == sonsLen(t))
-      for i in 0 ..< sonsLen(t.n):
-        assert(t.n.sons[i].kind == nkSym)
-        c &= t.n.sons[i].sym.name.s
+      assert(t.n.len == t.len)
+      for i in 0..<t.n.len:
+        assert(t.n[i].kind == nkSym)
+        c &= t.n[i].sym.name.s
         c &= ":"
-        c.hashType(t.sons[i])
+        c.hashType(t[i])
         c &= ","
     else:
-      for i in 0 ..< sonsLen(t): c.hashType t.sons[i]
+      for i in 0..<t.len: c.hashType t[i]
   of tyRange:
     c.hashTree(t.n)
-    c.hashType(t.sons[0])
+    c.hashType(t[0])
   of tyProc:
     c &= (if tfIterator in t.flags: "iterator " else: "proc ")
-    for i in 0..<t.len: c.hashType(t.sons[i])
+    for i in 0..<t.len: c.hashType(t[i])
     md5Update(c, cast[cstring](addr(t.callConv)), 1)
 
     if tfNoSideEffect in t.flags: c &= ".noSideEffect"
     if tfThread in t.flags: c &= ".thread"
   else:
-    for i in 0..<t.len: c.hashType(t.sons[i])
+    for i in 0..<t.len: c.hashType(t[i])
   if tfNotNil in t.flags: c &= "not nil"
 
 proc canonConst(n: PNode): TUid =
@@ -238,35 +238,35 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
     encodeVInt(n.sym.id, result)
     pushSym(w, n.sym)
   else:
-    for i in 0 ..< sonsLen(n):
-      encodeNode(w, n.info, n.sons[i], result)
-  add(result, ')')
+    for i in 0..<n.len:
+      encodeNode(w, n.info, n[i], result)
+  result.add(')')
 
 proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
   var oldLen = result.len
   result.add('<')
   if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
   if loc.s != low(loc.s):
-    add(result, '*')
+    result.add('*')
     encodeVInt(ord(loc.s), result)
   if loc.flags != {}:
-    add(result, '$')
+    result.add('$')
     encodeVInt(cast[int32](loc.flags), result)
   if loc.t != nil:
-    add(result, '^')
+    result.add('^')
     encodeVInt(cast[int32](loc.t.id), result)
     pushType(w, loc.t)
   if loc.r != nil:
-    add(result, '!')
+    result.add('!')
     encodeStr($loc.r, result)
   if loc.a != 0:
-    add(result, '?')
+    result.add('?')
     encodeVInt(loc.a, result)
   if oldLen + 1 == result.len:
     # no data was necessary, so remove the '<' again:
     setLen(result, oldLen)
   else:
-    add(result, '>')
+    result.add('>')
 
 proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t == nil:
@@ -277,47 +277,47 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t.kind == tyForward: internalError("encodeType: tyForward")
   # for the new rodfile viewer we use a preceding [ so that the data section
   # can easily be disambiguated:
-  add(result, '[')
+  result.add('[')
   encodeVInt(ord(t.kind), result)
-  add(result, '+')
+  result.add('+')
   encodeVInt(t.id, result)
   if t.n != nil:
-    encodeNode(w, unknownLineInfo(), t.n, result)
+    encodeNode(w, unknownLineInfo, t.n, result)
   if t.flags != {}:
-    add(result, '$')
+    result.add('$')
     encodeVInt(cast[int32](t.flags), result)
   if t.callConv != low(t.callConv):
-    add(result, '?')
+    result.add('?')
     encodeVInt(ord(t.callConv), result)
   if t.owner != nil:
-    add(result, '*')
+    result.add('*')
     encodeVInt(t.owner.id, result)
     pushSym(w, t.owner)
   if t.sym != nil:
-    add(result, '&')
+    result.add('&')
     encodeVInt(t.sym.id, result)
     pushSym(w, t.sym)
   if t.size != - 1:
-    add(result, '/')
+    result.add('/')
     encodeVBiggestInt(t.size, result)
   if t.align != - 1:
-    add(result, '=')
+    result.add('=')
     encodeVInt(t.align, result)
   encodeLoc(w, t.loc, result)
-  for i in 0 ..< sonsLen(t):
-    if t.sons[i] == nil:
-      add(result, "^()")
+  for i in 0..<t.len:
+    if t[i] == nil:
+      result.add("^()")
     else:
-      add(result, '^')
-      encodeVInt(t.sons[i].id, result)
-      pushType(w, t.sons[i])
+      result.add('^')
+      encodeVInt(t[i].id, result)
+      pushType(w, t[i])
 
 proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) =
-  add(result, '|')
+  result.add('|')
   encodeVInt(ord(lib.kind), result)
-  add(result, '|')
+  result.add('|')
   encodeStr($lib.name, result)
-  add(result, '|')
+  result.add('|')
   encodeNode(w, info, lib.path, result)
 
 proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
@@ -363,8 +363,8 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
   encodeLoc(w, s.loc, result)
   if s.annex != nil: encodeLib(w, s.annex, s.info, result)
   if s.constraint != nil:
-    add(result, '#')
-    encodeNode(w, unknownLineInfo(), s.constraint, result)
+    result.add('#')
+    encodeNode(w, unknownLineInfo, s.constraint, result)
   # lazy loading will soon reload the ast lazily, so the ast needs to be
   # the last entry of a symbol:
   if s.ast != nil:

compiler/ccgcalls.nim

@@ -9,43 +9,95 @@
 #
 # included from cgen.nim
 
-proc leftAppearsOnRightSide(le, ri: PNode): bool =
+proc canRaiseDisp(p: BProc; n: PNode): bool =
+  # we assume things like sysFatal cannot raise themselves
+  if n.kind == nkSym and {sfNeverRaises, sfImportc, sfCompilerProc} * n.sym.flags != {}:
+    result = false
+  elif optPanics in p.config.globalOptions or
+      (n.kind == nkSym and sfSystemModule in getModule(n.sym).flags):
+    # we know we can be strict:
+    result = canRaise(n)
+  else:
+    # we have to be *very* conservative:
+    result = canRaiseConservative(n)
+
+proc preventNrvo(p: BProc; le, ri: PNode): bool =
+  proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
+    var n = le
+    while true:
+      # do NOT follow nkHiddenDeref here!
+      case n.kind
+      of nkSym:
+        # we don't own the location so it escapes:
+        if n.sym.owner != p.prc:
+          return true
+        elif inTryStmt and sfUsedInFinallyOrExcept in n.sym.flags:
+          # it is also an observable store if the location is used
+          # in 'except' or 'finally'
+          return true
+        return false
+      of nkDotExpr, nkBracketExpr, nkObjUpConv, nkObjDownConv,
+          nkCheckedFieldExpr:
+        n = n[0]
+      of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+        n = n[1]
+      else:
+        # cannot analyse the location; assume the worst
+        return true
+
   if le != nil:
-    for i in 1 ..< ri.len:
+    for i in 1..<ri.len:
       let r = ri[i]
       if isPartOf(le, r) != arNo: return true
+    # we use the weaker 'canRaise' here in order to prevent too many
+    # annoying warnings, see #14514
+    if canRaise(ri[0]) and
+        locationEscapes(p, le, p.nestedTryStmts.len > 0):
+      message(p.config, le.info, warnObservableStores, $le)
 
 proc hasNoInit(call: PNode): bool {.inline.} =
-  result = call.sons[0].kind == nkSym and sfNoInit in call.sons[0].sym.flags
+  result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
+
+proc isHarmlessStore(p: BProc; canRaise: bool; d: TLoc): bool =
+  if d.k in {locTemp, locNone} or not canRaise:
+    result = true
+  elif d.k == locLocalVar and p.withinTryWithExcept == 0:
+    # we cannot observe a store to a local variable if the current proc
+    # has no error handler:
+    result = true
+  else:
+    result = false
 
 proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
                callee, params: Rope) =
+  let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
   genLineDir(p, ri)
   var pl = callee & ~"(" & params
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
+  var typ = skipTypes(ri[0].typ, abstractInst)
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
       if params != nil: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
-        if d.k == locNone: getTemp(p, typ.sons[0], d, needsInit=true)
+        if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
           discard "resetLoc(p, d)"
-        add(pl, addrLoc(p.config, d))
-        add(pl, ~");$n")
+        pl.add(addrLoc(p.config, d))
+        pl.add(~");$n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
-        add(pl, ~");$n")
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
+        pl.add(~");$n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
+      if canRaise: raiseExit(p)
     else:
-      add(pl, ~")")
+      pl.add(~")")
       if p.module.compileToCpp:
         if lfSingleUse in d.flags:
           # do not generate spurious temporaries for C++! For C we're better off
@@ -56,32 +108,55 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
           excl d.flags, lfSingleUse
         else:
           if d.k == locNone and p.splitDecls == 0:
-            getTempCpp(p, typ.sons[0], d, pl)
+            getTempCpp(p, typ[0], d, pl)
           else:
-            if d.k == locNone: getTemp(p, typ.sons[0], d)
+            if d.k == locNone: getTemp(p, typ[0], d)
             var list: TLoc
             initLoc(list, locCall, d.lode, OnUnknown)
             list.r = pl
             genAssignment(p, d, list, {}) # no need for deep copying
-      else:
-        if d.k == locNone: getTemp(p, typ.sons[0], d)
+            if canRaise: raiseExit(p)
+
+      elif isHarmlessStore(p, canRaise, d):
+        if d.k == locNone: getTemp(p, typ[0], d)
         assert(d.t != nil)        # generate an assignment to d:
         var list: TLoc
         initLoc(list, locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
+        if canRaise: raiseExit(p)
+      else:
+        var tmp: TLoc
+        getTemp(p, typ[0], tmp, needsInit=true)
+        var list: TLoc
+        initLoc(list, locCall, d.lode, OnUnknown)
+        list.r = pl
+        genAssignment(p, tmp, list, {}) # no need for deep copying
+        if canRaise: raiseExit(p)
+        genAssignment(p, d, tmp, {})
   else:
-    add(pl, ~");$n")
+    pl.add(~");$n")
     line(p, cpsStmts, pl)
+    if canRaise: raiseExit(p)
 
 proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
 
-proc openArrayLoc(p: BProc, n: PNode): Rope =
+proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
   var a: TLoc
 
-  let q = skipConv(n)
+  var q = skipConv(n)
+  var skipped = false
+  while q.kind == nkStmtListExpr and q.len > 0:
+    skipped = true
+    q = q.lastSon
   if getMagic(q) == mSlice:
     # magic: pass slice to openArray:
+    if skipped:
+      q = skipConv(n)
+      while q.kind == nkStmtListExpr and q.len > 0:
+        for i in 0..<q.len-1:
+          genStmts(p, q[i])
+        q = q.lastSon
     var b, c: TLoc
     initLocExpr(p, q[1], a)
     initLocExpr(p, q[2], b)
@@ -90,21 +165,26 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
     if optBoundsCheck in p.options:
       genBoundsCheck(p, a, b, c)
     let ty = skipTypes(a.t, abstractVar+{tyPtr})
+    let dest = getTypeDesc(p.module, n.typ[0])
     case ty.kind
     of tyArray:
-      let first = firstOrd(p.config, ty)
+      let first = toInt64(firstOrd(p.config, ty))
       if first == 0:
-        result = "($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c)]
+        result = "($4*)(($1)+($2)), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dest]
       else:
-        result = "($1)+(($2)-($4)), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), intLiteral(first)]
-    of tyOpenArray, tyVarargs, tyUncheckedArray:
-      result = "($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c)]
+        result = "($5*)($1)+(($2)-($4)), ($3)-($2)+1" %
+          [rdLoc(a), rdLoc(b), rdLoc(c), intLiteral(first), dest]
+    of tyOpenArray, tyVarargs, tyUncheckedArray, tyCString:
+      result = "($4*)($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dest]
     of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-          not compileToCpp(p.module):
-        result = "(*$1)$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p)]
+      let atyp = skipTypes(a.t, abstractInst)
+      if formalType.skipTypes(abstractInst).kind == tyVar and atyp.kind == tyString and
+          optSeqDestructors in p.config.globalOptions:
+        linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+      if atyp.kind == tyVar and not compileToCpp(p.module):
+        result = "($5*)(*$1)$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
       else:
-        result = "$1$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p)]
+        result = "($5*)$1$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
     else:
       internalError(p.config, "openArrayLoc: " & typeToString(a.t))
   else:
@@ -113,8 +193,11 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
     of tyOpenArray, tyVarargs:
       result = "$1, $1Len_0" % [rdLoc(a)]
     of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-            not compileToCpp(p.module):
+      let ntyp = skipTypes(n.typ, abstractInst)
+      if formalType.skipTypes(abstractInst).kind == tyVar and ntyp.kind == tyString and
+          optSeqDestructors in p.config.globalOptions:
+        linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+      if ntyp.kind == tyVar and not compileToCpp(p.module):
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
         result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
@@ -136,7 +219,7 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
 
 proc genArgStringToCString(p: BProc, n: PNode): Rope {.inline.} =
   var a: TLoc
-  initLocExpr(p, n.sons[0], a)
+  initLocExpr(p, n[0], a)
   result = ropecg(p.module, "#nimToCStringConv($1)", [a.rdLoc])
 
 proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
@@ -144,19 +227,19 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
   if n.kind == nkStringToCString:
     result = genArgStringToCString(p, n)
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
-    var n = if n.kind != nkHiddenAddr: n else: n.sons[0]
-    result = openArrayLoc(p, n)
+    var n = if n.kind != nkHiddenAddr: n else: n[0]
+    result = openArrayLoc(p, param.typ, n)
   elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
     result = addrLoc(p.config, a)
   elif p.module.compileToCpp and param.typ.kind == tyVar and
       n.kind == nkHiddenAddr:
-    initLocExprSingleUse(p, n.sons[0], a)
+    initLocExprSingleUse(p, n[0], a)
     # if the proc is 'importc'ed but not 'importcpp'ed then 'var T' still
     # means '*T'. See posix.nim for lots of examples that do that in the wild.
-    let callee = call.sons[0]
+    let callee = call[0]
     if callee.kind == nkSym and
-        {sfImportC, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportC} and
+        {sfImportc, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportc} and
         {lfHeader, lfNoDecl} * callee.sym.loc.flags != {}:
       result = addrLoc(p.config, a)
     else:
@@ -174,36 +257,35 @@ proc genArgNoParam(p: BProc, n: PNode): Rope =
     result = rdLoc(a)
 
 template genParamLoop(params) {.dirty.} =
-  if i < sonsLen(typ):
-    assert(typ.n.sons[i].kind == nkSym)
-    let paramType = typ.n.sons[i]
+  if i < typ.len:
+    assert(typ.n[i].kind == nkSym)
+    let paramType = typ.n[i]
     if not paramType.typ.isCompileTimeOnly:
-      if params != nil: add(params, ~", ")
-      add(params, genArg(p, ri.sons[i], paramType.sym, ri))
+      if params != nil: params.add(~", ")
+      params.add(genArg(p, ri[i], paramType.sym, ri))
   else:
-    if params != nil: add(params, ~", ")
-    add(params, genArgNoParam(p, ri.sons[i]))
+    if params != nil: params.add(~", ")
+    params.add(genArgNoParam(p, ri[i]))
 
-proc addActualPrefixForHCR(res: var Rope, module: PSym, sym: PSym) =
-  if sym.flags * {sfImportc, sfNonReloadable} == {} and
+proc addActualSuffixForHCR(res: var Rope, module: PSym, sym: PSym) =
+  if sym.flags * {sfImportc, sfNonReloadable} == {} and sym.loc.k == locProc and
       (sym.typ.callConv == ccInline or sym.owner.id == module.id):
     res = res & "_actual".rope
 
 proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var op: TLoc
   # this is a hotspot in the compiler
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   var params: Rope
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInstOwned)
+  var typ = skipTypes(ri[0].typ, abstractInstOwned)
   assert(typ.kind == tyProc)
-  assert(sonsLen(typ) == sonsLen(typ.n))
-  var length = sonsLen(ri)
-  for i in 1 ..< length:
+  assert(typ.len == typ.n.len)
+  for i in 1..<ri.len:
     genParamLoop(params)
   var callee = rdLoc(op)
-  if p.hcrOn and ri.sons[0].kind == nkSym:
-    callee.addActualPrefixForHCR(p.module.module, ri.sons[0].sym)
+  if p.hcrOn and ri[0].kind == nkSym:
+    callee.addActualSuffixForHCR(p.module.module, ri[0].sym)
   fixupCall(p, le, ri, d, callee, params)
 
 proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
@@ -217,14 +299,13 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
   const PatProc = "$1.ClE_0? $1.ClP_0($3$1.ClE_0):(($4)($1.ClP_0))($2)"
   const PatIter = "$1.ClP_0($3$1.ClE_0)" # we know the env exists
   var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   var pl: Rope
 
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
-  var length = sonsLen(ri)
-  for i in 1 ..< length:
-    assert(sonsLen(typ) == sonsLen(typ.n))
+  for i in 1..<ri.len:
+    assert(typ.len == typ.n.len)
     genParamLoop(pl)
 
   template genCallPattern {.dirty.} =
@@ -234,27 +315,29 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
       lineF(p, cpsStmts, PatProc & ";$n", [rdLoc(op), pl, pl.addComma, rawProc])
 
   let rawProc = getRawProcType(p, typ)
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
-      if sonsLen(ri) > 1: add(pl, ~", ")
+  let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
+      if ri.len > 1: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
-          getTemp(p, typ.sons[0], d, needsInit=true)
+          getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
           discard "resetLoc(p, d)"
-        add(pl, addrLoc(p.config, d))
+        pl.add(addrLoc(p.config, d))
         genCallPattern()
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
         genCallPattern()
+        if canRaise: raiseExit(p)
         genAssignment(p, d, tmp, {}) # no need for deep copying
-    else:
-      if d.k == locNone: getTemp(p, typ.sons[0], d)
+    elif isHarmlessStore(p, canRaise, d):
+      if d.k == locNone: getTemp(p, typ[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
       initLoc(list, locCall, d.lode, OnUnknown)
@@ -262,29 +345,43 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
         list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
       else:
         list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
-
       genAssignment(p, d, list, {}) # no need for deep copying
+      if canRaise: raiseExit(p)
+    else:
+      var tmp: TLoc
+      getTemp(p, typ[0], tmp)
+      assert(d.t != nil)        # generate an assignment to d:
+      var list: TLoc
+      initLoc(list, locCall, d.lode, OnUnknown)
+      if tfIterator in typ.flags:
+        list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
+      else:
+        list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
+      genAssignment(p, tmp, list, {})
+      if canRaise: raiseExit(p)
+      genAssignment(p, d, tmp, {})
   else:
     genCallPattern()
+    if canRaise: raiseExit(p)
 
 proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
-  if i < sonsLen(typ):
+  if i < typ.len:
     # 'var T' is 'T&' in C++. This means we ignore the request of
     # any nkHiddenAddr when it's a 'var T'.
-    let paramType = typ.n.sons[i]
+    let paramType = typ.n[i]
     assert(paramType.kind == nkSym)
     if paramType.typ.isCompileTimeOnly:
       result = nil
-    elif typ.sons[i].kind == tyVar and ri.sons[i].kind == nkHiddenAddr:
-      result = genArgNoParam(p, ri.sons[i][0])
+    elif typ[i].kind == tyVar and ri[i].kind == nkHiddenAddr:
+      result = genArgNoParam(p, ri[i][0])
     else:
-      result = genArgNoParam(p, ri.sons[i]) #, typ.n.sons[i].sym)
+      result = genArgNoParam(p, ri[i]) #, typ.n[i].sym)
   else:
     if tfVarargs notin typ.flags:
       localError(p.config, ri.info, "wrong argument count")
       result = nil
     else:
-      result = genArgNoParam(p, ri.sons[i])
+      result = genArgNoParam(p, ri[i])
 
 discard """
 Dot call syntax in C++
@@ -328,16 +425,16 @@ proc skipAddrDeref(node: PNode): PNode =
   var isAddr = false
   case n.kind
   of nkAddr, nkHiddenAddr:
-    n = n.sons[0]
+    n = n[0]
     isAddr = true
   of nkDerefExpr, nkHiddenDeref:
-    n = n.sons[0]
+    n = n[0]
   else: return n
-  if n.kind == nkObjDownConv: n = n.sons[0]
+  if n.kind == nkObjDownConv: n = n[0]
   if isAddr and n.kind in {nkDerefExpr, nkHiddenDeref}:
-    result = n.sons[0]
+    result = n[0]
   elif n.kind in {nkAddr, nkHiddenAddr}:
-    result = n.sons[0]
+    result = n[0]
   else:
     result = node
 
@@ -345,13 +442,13 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   # for better or worse c2nim translates the 'this' argument to a 'var T'.
   # However manual wrappers may also use 'ptr T'. In any case we support both
   # for convenience.
-  internalAssert p.config, i < sonsLen(typ)
-  assert(typ.n.sons[i].kind == nkSym)
+  internalAssert p.config, i < typ.len
+  assert(typ.n[i].kind == nkSym)
   # if the parameter is lying (tyVar) and thus we required an additional deref,
   # skip the deref:
   var ri = ri[i]
   while ri.kind == nkObjDownConv: ri = ri[0]
-  let t = typ.sons[i].skipTypes({tyGenericInst, tyAlias, tySink})
+  let t = typ[i].skipTypes({tyGenericInst, tyAlias, tySink})
   if t.kind == tyVar:
     let x = if ri.kind == nkHiddenAddr: ri[0] else: ri
     if x.typ.kind == tyPtr:
@@ -373,7 +470,7 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   else:
     ri = skipAddrDeref(ri)
     if ri.kind in {nkAddr, nkHiddenAddr}: ri = ri[0]
-    result = genArgNoParam(p, ri) #, typ.n.sons[i].sym)
+    result = genArgNoParam(p, ri) #, typ.n[i].sym)
     result.add(".")
 
 proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
@@ -382,22 +479,25 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
   while i < pat.len:
     case pat[i]
     of '@':
-      if j < ri.len:
-        result.add genOtherArg(p, ri, j, typ)
-        for k in j+1 ..< ri.len:
-          result.add(~", ")
-          result.add genOtherArg(p, ri, k, typ)
+      var first = true
+      for k in j..<ri.len:
+        let arg = genOtherArg(p, ri, k, typ)
+        if arg.len > 0:
+          if not first:
+            result.add(~", ")
+          first = false
+          result.add arg
       inc i
     of '#':
       if i+1 < pat.len and pat[i+1] in {'+', '@'}:
         let ri = ri[j]
         if ri.kind in nkCallKinds:
-          let typ = skipTypes(ri.sons[0].typ, abstractInst)
-          if pat[i+1] == '+': result.add genArgNoParam(p, ri.sons[0])
+          let typ = skipTypes(ri[0].typ, abstractInst)
+          if pat[i+1] == '+': result.add genArgNoParam(p, ri[0])
           result.add(~"(")
           if 1 < ri.len:
             result.add genOtherArg(p, ri, 1, typ)
-          for k in j+1 ..< ri.len:
+          for k in j+1..<ri.len:
             result.add(~", ")
             result.add genOtherArg(p, ri, k, typ)
           result.add(~")")
@@ -408,7 +508,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
         result.add genThisArg(p, ri, j, typ)
         inc i
       elif i+1 < pat.len and pat[i+1] == '[':
-        var arg = ri.sons[j].skipAddrDeref
+        var arg = ri[j].skipAddrDeref
         while arg.kind in {nkAddr, nkHiddenAddr, nkObjDownConv}: arg = arg[0]
         result.add genArgNoParam(p, arg)
         #result.add debugTree(arg, 0, 10)
@@ -428,24 +528,23 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
         if pat[i] notin {'@', '#', '\''}: inc(i)
         else: break
       if i - 1 >= start:
-        add(result, substr(pat, start, i - 1))
+        result.add(substr(pat, start, i - 1))
 
 proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
-  var length = sonsLen(ri)
-  assert(sonsLen(typ) == sonsLen(typ.n))
+  assert(typ.len == typ.n.len)
   # don't call '$' here for efficiency:
-  let pat = ri.sons[0].sym.loc.r.data
+  let pat = ri[0].sym.loc.r.data
   internalAssert p.config, pat.len > 0
   if pat.contains({'#', '(', '@', '\''}):
     var pl = genPatternCall(p, ri, pat, typ)
     # simpler version of 'fixupCall' that works with the pl+params combination:
-    var typ = skipTypes(ri.sons[0].typ, abstractInst)
-    if typ.sons[0] != nil:
+    var typ = skipTypes(ri[0].typ, abstractInst)
+    if typ[0] != nil:
       if p.module.compileToCpp and lfSingleUse in d.flags:
         # do not generate spurious temporaries for C++! For C we're better off
         # with them to prevent undefined behaviour and because the codegen
@@ -454,115 +553,114 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
         d.r = pl
         excl d.flags, lfSingleUse
       else:
-        if d.k == locNone: getTemp(p, typ.sons[0], d)
+        if d.k == locNone: getTemp(p, typ[0], d)
         assert(d.t != nil)        # generate an assignment to d:
         var list: TLoc
         initLoc(list, locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
     else:
-      add(pl, ~";$n")
+      pl.add(~";$n")
       line(p, cpsStmts, pl)
   else:
     var pl: Rope = nil
-    #var param = typ.n.sons[1].sym
+    #var param = typ.n[1].sym
     if 1 < ri.len:
-      add(pl, genThisArg(p, ri, 1, typ))
-    add(pl, op.r)
+      pl.add(genThisArg(p, ri, 1, typ))
+    pl.add(op.r)
     var params: Rope
-    for i in 2 ..< length:
+    for i in 2..<ri.len:
       if params != nil: params.add(~", ")
-      assert(sonsLen(typ) == sonsLen(typ.n))
-      add(params, genOtherArg(p, ri, i, typ))
+      assert(typ.len == typ.n.len)
+      params.add(genOtherArg(p, ri, i, typ))
     fixupCall(p, le, ri, d, pl, params)
 
 proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
   # generates a crappy ObjC call
   var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   var pl = ~"["
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
-  var length = sonsLen(ri)
-  assert(sonsLen(typ) == sonsLen(typ.n))
+  assert(typ.len == typ.n.len)
 
   # don't call '$' here for efficiency:
-  let pat = ri.sons[0].sym.loc.r.data
+  let pat = ri[0].sym.loc.r.data
   internalAssert p.config, pat.len > 0
   var start = 3
   if ' ' in pat:
     start = 1
-    add(pl, op.r)
-    if length > 1:
-      add(pl, ~": ")
-      add(pl, genArg(p, ri.sons[1], typ.n.sons[1].sym, ri))
+    pl.add(op.r)
+    if ri.len > 1:
+      pl.add(~": ")
+      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
       start = 2
   else:
-    if length > 1:
-      add(pl, genArg(p, ri.sons[1], typ.n.sons[1].sym, ri))
-      add(pl, ~" ")
-    add(pl, op.r)
-    if length > 2:
-      add(pl, ~": ")
-      add(pl, genArg(p, ri.sons[2], typ.n.sons[2].sym, ri))
-  for i in start ..< length:
-    assert(sonsLen(typ) == sonsLen(typ.n))
-    if i >= sonsLen(typ):
+    if ri.len > 1:
+      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
+      pl.add(~" ")
+    pl.add(op.r)
+    if ri.len > 2:
+      pl.add(~": ")
+      pl.add(genArg(p, ri[2], typ.n[2].sym, ri))
+  for i in start..<ri.len:
+    assert(typ.len == typ.n.len)
+    if i >= typ.len:
       internalError(p.config, ri.info, "varargs for objective C method?")
-    assert(typ.n.sons[i].kind == nkSym)
-    var param = typ.n.sons[i].sym
-    add(pl, ~" ")
-    add(pl, param.name.s)
-    add(pl, ~": ")
-    add(pl, genArg(p, ri.sons[i], param, ri))
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
-      if sonsLen(ri) > 1: add(pl, ~" ")
+    assert(typ.n[i].kind == nkSym)
+    var param = typ.n[i].sym
+    pl.add(~" ")
+    pl.add(param.name.s)
+    pl.add(~": ")
+    pl.add(genArg(p, ri[i], param, ri))
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
+      if ri.len > 1: pl.add(~" ")
       # beware of 'result = p(result)'. We always allocate a temporary:
       if d.k in {locTemp, locNone}:
         # We already got a temp. Great, special case it:
-        if d.k == locNone: getTemp(p, typ.sons[0], d, needsInit=true)
-        add(pl, ~"Result: ")
-        add(pl, addrLoc(p.config, d))
-        add(pl, ~"];$n")
+        if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
+        pl.add(~"Result: ")
+        pl.add(addrLoc(p.config, d))
+        pl.add(~"];$n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
-        add(pl, ~"];$n")
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
+        pl.add(~"];$n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
     else:
-      add(pl, ~"]")
-      if d.k == locNone: getTemp(p, typ.sons[0], d)
+      pl.add(~"]")
+      if d.k == locNone: getTemp(p, typ[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
       initLoc(list, locCall, ri, OnUnknown)
       list.r = pl
       genAssignment(p, d, list, {}) # no need for deep copying
   else:
-    add(pl, ~"];$n")
+    pl.add(~"];$n")
     line(p, cpsStmts, pl)
 
 proc genCall(p: BProc, e: PNode, d: var TLoc) =
-  if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
+  if e[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
     genClosureCall(p, nil, e, d)
-  elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags:
+  elif e[0].kind == nkSym and sfInfixCall in e[0].sym.flags:
     genInfixCall(p, nil, e, d)
-  elif e.sons[0].kind == nkSym and sfNamedParamCall in e.sons[0].sym.flags:
+  elif e[0].kind == nkSym and sfNamedParamCall in e[0].sym.flags:
     genNamedParamCall(p, e, d)
   else:
     genPrefixCall(p, nil, e, d)
   postStmtActions(p)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
-  if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
+  if ri[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
-  elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags:
+  elif ri[0].kind == nkSym and sfInfixCall in ri[0].sym.flags:
     genInfixCall(p, le, ri, d)
-  elif ri.sons[0].kind == nkSym and sfNamedParamCall in ri.sons[0].sym.flags:
+  elif ri[0].kind == nkSym and sfNamedParamCall in ri[0].sym.flags:
     genNamedParamCall(p, ri, d)
   else:
     genPrefixCall(p, le, ri, d)

compiler/ccgliterals.nim

@@ -7,6 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
+# included from cgen.nim
+
 ## This include file contains the logic to produce constant string
 ## and seq literals. The code here is responsible that
 ## ``const x = ["a", "b"]`` works without hidden runtime creation code.
@@ -19,7 +21,7 @@ template detectVersion(field, corename) =
     if core == nil or core.kind != skConst:
       m.g.field = 1
     else:
-      m.g.field = int ast.getInt(core.ast)
+      m.g.field = toInt(ast.getInt(core.ast))
   result = m.g.field
 
 proc detectStrVersion(m: BModule): int =
@@ -33,8 +35,8 @@ proc detectSeqVersion(m: BModule): int =
 proc genStringLiteralDataOnlyV1(m: BModule, s: string): Rope =
   discard cgsym(m, "TGenericSeq")
   result = getTempName(m)
-  addf(m.s[cfsData], "STRING_LITERAL($1, $2, $3);$n",
-       [result, makeCString(s), rope(len(s))])
+  m.s[cfsData].addf("STRING_LITERAL($1, $2, $3);$n",
+       [result, makeCString(s), rope(s.len)])
 
 proc genStringLiteralV1(m: BModule; n: PNode): Rope =
   if s.isNil:
@@ -51,29 +53,31 @@ proc genStringLiteralV1(m: BModule; n: PNode): Rope =
 
 # ------ Version 2: destructor based strings and seqs -----------------------
 
-proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope) =
-  addf(m.s[cfsData], "static const struct {$n" &
-       "  NI cap; void* allocator; NIM_CHAR data[$2+1];$n" &
-       "} $1 = { $2, NIM_NIL, $3 };$n",
-       [result, rope(len(s)), makeCString(s)])
+proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope; isConst: bool) =
+  m.s[cfsData].addf("static $4 struct {$n" &
+       "  NI cap; NIM_CHAR data[$2+1];$n" &
+       "} $1 = { $2 | NIM_STRLIT_FLAG, $3 };$n",
+       [result, rope(s.len), makeCString(s),
+       rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2(m: BModule; n: PNode): Rope =
+proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   if id == m.labels:
     let pureLit = getTempName(m)
-    genStringLiteralDataOnlyV2(m, n.strVal, pureLit)
+    genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
     result = getTempName(m)
     discard cgsym(m, "NimStrPayload")
     discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    addf(m.s[cfsData], "static const NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [result, rope(len(n.strVal)), pureLit])
+    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [result, rope(n.strVal.len), pureLit, rope(if isConst: "const" else: "")])
   else:
     result = getTempName(m)
-    addf(m.s[cfsData], "static const NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [result, rope(len(n.strVal)), m.tmpBase & rope(id)])
+    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [result, rope(n.strVal.len), m.tmpBase & rope(id),
+          rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2Const(m: BModule; n: PNode): Rope =
+proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   var pureLit: Rope
   if id == m.labels:
@@ -81,19 +85,20 @@ proc genStringLiteralV2Const(m: BModule; n: PNode): Rope =
     discard cgsym(m, "NimStrPayload")
     discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    genStringLiteralDataOnlyV2(m, n.strVal, pureLit)
+    genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
   else:
     pureLit = m.tmpBase & rope(id)
-  result = "{$1, (NimStrPayload*)&$2}" % [rope(len(n.strVal)), pureLit]
+  result = "{$1, (NimStrPayload*)&$2}" % [rope(n.strVal.len), pureLit]
 
 # ------ Version selector ---------------------------------------------------
 
-proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo): Rope =
+proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo;
+                              isConst: bool): Rope =
   case detectStrVersion(m)
   of 0, 1: result = genStringLiteralDataOnlyV1(m, s)
   of 2:
     result = getTempName(m)
-    genStringLiteralDataOnlyV2(m, s, result)
+    genStringLiteralDataOnlyV2(m, s, result, isConst)
   else:
     localError(m.config, info, "cannot determine how to produce code for string literal")
 
@@ -103,6 +108,6 @@ proc genNilStringLiteral(m: BModule; info: TLineInfo): Rope =
 proc genStringLiteral(m: BModule; n: PNode): Rope =
   case detectStrVersion(m)
   of 0, 1: result = genStringLiteralV1(m, n)
-  of 2: result = genStringLiteralV2(m, n)
+  of 2: result = genStringLiteralV2(m, n, isConst = true)
   else:
     localError(m.config, n.info, "cannot determine how to produce code for string literal")

compiler/ccgmerge.nim

@@ -11,8 +11,8 @@
 ## is needed for incremental compilation.
 
 import
-  ast, astalgo, ropes, options, strutils, nimlexbase, msgs, cgendata, rodutils,
-  intsets, platform, llstream, tables, sighashes, modulegraphs, pathutils
+  ast, ropes, options, strutils, nimlexbase, cgendata, rodutils,
+  intsets, llstream, tables, modulegraphs, pathutils
 
 # Careful! Section marks need to contain a tabulator so that they cannot
 # be part of C string literals.
@@ -28,8 +28,8 @@ const
     cfsFieldInfo: "NIM_merge_FIELD_INFO",
     cfsTypeInfo: "NIM_merge_TYPE_INFO",
     cfsProcHeaders: "NIM_merge_PROC_HEADERS",
-    cfsVars: "NIM_merge_VARS",
     cfsData: "NIM_merge_DATA",
+    cfsVars: "NIM_merge_VARS",
     cfsProcs: "NIM_merge_PROCS",
     cfsInitProc: "NIM_merge_INIT_PROC",
     cfsDatInitProc: "NIM_merge_DATINIT_PROC",
@@ -50,9 +50,9 @@ const
 proc genSectionStart*(fs: TCFileSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
     result = nil
-    add(result, "\n/*\t")
-    add(result, CFileSectionNames[fs])
-    add(result, ":*/\n")
+    result.add("\n/*\t")
+    result.add(CFileSectionNames[fs])
+    result.add(":*/\n")
 
 proc genSectionEnd*(fs: TCFileSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
@@ -61,9 +61,9 @@ proc genSectionEnd*(fs: TCFileSection; conf: ConfigRef): Rope =
 proc genSectionStart*(ps: TCProcSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
     result = rope("")
-    add(result, "\n/*\t")
-    add(result, CProcSectionNames[ps])
-    add(result, ":*/\n")
+    result.add("\n/*\t")
+    result.add(CProcSectionNames[ps])
+    result.add(":*/\n")
 
 proc genSectionEnd*(ps: TCProcSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):

compiler/ccgreset.nim

@@ -0,0 +1,94 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# included from cgen.nim
+
+## Code specialization instead of the old, incredibly slow 'genericReset'
+## implementation.
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType)
+
+proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
+                     typ: PType) =
+  if n == nil: return
+  case n.kind
+  of nkRecList:
+    for i in 0..<n.len:
+      specializeResetN(p, accessor, n[i], typ)
+  of nkRecCase:
+    if (n[0].kind != nkSym): internalError(p.config, n.info, "specializeResetN")
+    let disc = n[0].sym
+    if disc.loc.r == nil: fillObjectFields(p.module, typ)
+    if disc.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
+    for i in 1..<n.len:
+      let branch = n[i]
+      assert branch.kind in {nkOfBranch, nkElse}
+      if branch.kind == nkOfBranch:
+        genCaseRange(p, branch)
+      else:
+        lineF(p, cpsStmts, "default:$n", [])
+      specializeResetN(p, accessor, lastSon(branch), typ)
+      lineF(p, cpsStmts, "break;$n", [])
+    lineF(p, cpsStmts, "} $n", [])
+    specializeResetT(p, "$1.$2" % [accessor, disc.loc.r], disc.loc.t)
+  of nkSym:
+    let field = n.sym
+    if field.typ.kind == tyVoid: return
+    if field.loc.r == nil: fillObjectFields(p.module, typ)
+    if field.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    specializeResetT(p, "$1.$2" % [accessor, field.loc.r], field.loc.t)
+  else: internalError(p.config, n.info, "specializeResetN()")
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
+  if typ == nil: return
+
+  case typ.kind
+  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred,
+     tySink, tyOwned:
+    specializeResetT(p, accessor, lastSon(typ))
+  of tyArray:
+    let arraySize = lengthOrd(p.config, typ[0])
+    var i: TLoc
+    getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i)
+    linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
+            [i.r, arraySize])
+    specializeResetT(p, ropecg(p.module, "$1[$2]", [accessor, i.r]), typ[1])
+    lineF(p, cpsStmts, "}$n", [])
+  of tyObject:
+    for i in 0..<typ.len:
+      var x = typ[i]
+      if x != nil: x = x.skipTypes(skipPtrs)
+      specializeResetT(p, accessor.parentObj(p.module), x)
+    if typ.n != nil: specializeResetN(p, accessor, typ.n, typ)
+  of tyTuple:
+    let typ = getUniqueType(typ)
+    for i in 0..<typ.len:
+      specializeResetT(p, ropecg(p.module, "$1.Field$2", [accessor, i]), typ[i])
+
+  of tyString, tyRef, tySequence:
+    lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1, NIM_NIL);$n", [accessor])
+
+  of tyProc:
+    if typ.callConv == ccClosure:
+      lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1.ClE_0, NIM_NIL);$n", [accessor])
+      lineCg(p, cpsStmts, "$1.ClP_0 = NIM_NIL;$n", [accessor])
+    else:
+      lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  of tyChar, tyBool, tyEnum, tyInt..tyUInt64:
+    lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
+  of tyCString, tyPointer, tyPtr, tyVar, tyLent:
+    lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  else:
+    discard
+
+proc specializeReset(p: BProc, a: TLoc) =
+  specializeResetT(p, rdLoc(a), a.t)

compiler/ccgthreadvars.nim

@@ -16,11 +16,11 @@ proc emulatedThreadVars(conf: ConfigRef): bool =
   result = {optThreads, optTlsEmulation} <= conf.globalOptions
 
 proc accessThreadLocalVar(p: BProc, s: PSym) =
-  if emulatedThreadVars(p.config) and not p.threadVarAccessed:
-    p.threadVarAccessed = true
+  if emulatedThreadVars(p.config) and threadVarAccessed notin p.flags:
+    p.flags.incl threadVarAccessed
     incl p.module.flags, usesThreadVars
-    addf(p.procSec(cpsLocals), "\tNimThreadVars* NimTV_;$n", [])
-    add(p.procSec(cpsInit),
+    p.procSec(cpsLocals).addf("\tNimThreadVars* NimTV_;$n", [])
+    p.procSec(cpsInit).add(
       ropecg(p.module, "\tNimTV_ = (NimThreadVars*) #GetThreadLocalVars();$n", []))
 
 proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
@@ -30,23 +30,25 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
     # allocator for it :-(
     if not containsOrIncl(m.g.nimtvDeclared, s.id):
       m.g.nimtvDeps.add(s.loc.t)
-      addf(m.g.nimtv, "$1 $2;$n", [getTypeDesc(m, s.loc.t), s.loc.r])
+      m.g.nimtv.addf("$1 $2;$n", [getTypeDesc(m, s.loc.t), s.loc.r])
   else:
-    if isExtern: add(m.s[cfsVars], "extern ")
-    if optThreads in m.config.globalOptions: add(m.s[cfsVars], "NIM_THREADVAR ")
-    add(m.s[cfsVars], getTypeDesc(m, s.loc.t))
-    addf(m.s[cfsVars], " $1;$n", [s.loc.r])
+    if isExtern: m.s[cfsVars].add("extern ")
+    elif lfExportLib in s.loc.flags: m.s[cfsVars].add("N_LIB_EXPORT_VAR ")
+    else: m.s[cfsVars].add("N_LIB_PRIVATE ")
+    if optThreads in m.config.globalOptions: m.s[cfsVars].add("NIM_THREADVAR ")
+    m.s[cfsVars].add(getTypeDesc(m, s.loc.t))
+    m.s[cfsVars].addf(" $1;$n", [s.loc.r])
 
 proc generateThreadLocalStorage(m: BModule) =
   if m.g.nimtv != nil and (usesThreadVars in m.flags or sfMainModule in m.module.flags):
     for t in items(m.g.nimtvDeps): discard getTypeDesc(m, t)
-    addf(m.s[cfsSeqTypes], "typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
+    m.s[cfsSeqTypes].addf("typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
 
 proc generateThreadVarsSize(m: BModule) =
   if m.g.nimtv != nil:
     let externc = if m.config.cmd == cmdCompileToCpp or
                        sfCompileToCpp in m.module.flags: "extern \"C\" "
                   else: ""
-    addf(m.s[cfsProcs],
+    m.s[cfsProcs].addf(
       "$#NI NimThreadVarsSize(){return (NI)sizeof(NimThreadVars);}$n",
       [externc.rope])

compiler/ccgtrav.nim

@@ -17,7 +17,7 @@ type
     visitorFrmt: string
 
 const
-  visitorFrmt  = "#nimGCvisit((void*)$1, $2);$n"
+  visitorFrmt = "#nimGCvisit((void*)$1, $2);$n"
 
 proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType)
 proc genCaseRange(p: BProc, branch: PNode)
@@ -28,18 +28,18 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
   if n == nil: return
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      genTraverseProc(c, accessor, n.sons[i], typ)
+    for i in 0..<n.len:
+      genTraverseProc(c, accessor, n[i], typ)
   of nkRecCase:
-    if (n.sons[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
+    if (n[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
     var p = c.p
-    let disc = n.sons[0].sym
+    let disc = n[0].sym
     if disc.loc.r == nil: fillObjectFields(c.p.module, typ)
     if disc.loc.t == nil:
       internalError(c.p.config, n.info, "genTraverseProc()")
     lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
-    for i in 1 ..< sonsLen(n):
-      let branch = n.sons[i]
+    for i in 1..<n.len:
+      let branch = n[i]
       assert branch.kind in {nkOfBranch, nkElse}
       if branch.kind == nkOfBranch:
         genCaseRange(c.p, branch)
@@ -73,38 +73,39 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType) =
      tySink, tyOwned:
     genTraverseProc(c, accessor, lastSon(typ))
   of tyArray:
-    let arraySize = lengthOrd(c.p.config, typ.sons[0])
+    let arraySize = lengthOrd(c.p.config, typ[0])
     var i: TLoc
-    getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo(), tyInt), i)
+    getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
     let oldCode = p.s(cpsStmts)
     linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
             [i.r, arraySize])
     let oldLen = p.s(cpsStmts).len
-    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", [accessor, i.r]), typ.sons[1])
+    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", [accessor, i.r]), typ[1])
     if p.s(cpsStmts).len == oldLen:
       # do not emit dummy long loops for faster debug builds:
       p.s(cpsStmts) = oldCode
     else:
       lineF(p, cpsStmts, "}$n", [])
   of tyObject:
-    for i in 0 ..< sonsLen(typ):
-      var x = typ.sons[i]
+    for i in 0..<typ.len:
+      var x = typ[i]
       if x != nil: x = x.skipTypes(skipPtrs)
       genTraverseProc(c, accessor.parentObj(c.p.module), x)
     if typ.n != nil: genTraverseProc(c, accessor, typ.n, typ)
   of tyTuple:
     let typ = getUniqueType(typ)
-    for i in 0 ..< sonsLen(typ):
-      genTraverseProc(c, ropecg(c.p.module, "$1.Field$2", [accessor, i]), typ.sons[i])
+    for i in 0..<typ.len:
+      genTraverseProc(c, ropecg(c.p.module, "$1.Field$2", [accessor, i]), typ[i])
   of tyRef:
     lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
   of tySequence:
-    if tfHasAsgn notin typ.flags:
+    if optSeqDestructors notin c.p.module.config.globalOptions:
       lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
     elif containsGarbageCollectedRef(typ.lastSon):
       # destructor based seqs are themselves not traced but their data is, if
       # they contain a GC'ed type:
-      genTraverseProcSeq(c, accessor, typ)
+      lineCg(p, cpsStmts, "#nimGCvisitSeq((void*)$1, $2);$n", [accessor, c.visitorFrmt])
+      #genTraverseProcSeq(c, accessor, typ)
   of tyString:
     if tfHasAsgn notin typ.flags:
       lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
@@ -118,7 +119,7 @@ proc genTraverseProcSeq(c: TTraversalClosure, accessor: Rope, typ: PType) =
   var p = c.p
   assert typ.kind == tySequence
   var i: TLoc
-  getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo(), tyInt), i)
+  getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
   let oldCode = p.s(cpsStmts)
   var a: TLoc
   a.r = accessor
@@ -126,7 +127,7 @@ proc genTraverseProcSeq(c: TTraversalClosure, accessor: Rope, typ: PType) =
   lineF(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
       [i.r, lenExpr(c.p, a)])
   let oldLen = p.s(cpsStmts).len
-  genTraverseProc(c, "$1$3[$2]" % [accessor, i.r, dataField(c.p)], typ.sons[0])
+  genTraverseProc(c, "$1$3[$2]" % [accessor, i.r, dataField(c.p)], typ[0])
   if p.s(cpsStmts).len == oldLen:
     # do not emit dummy long loops for faster debug builds:
     p.s(cpsStmts) = oldCode
@@ -154,11 +155,11 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope =
   if typ.kind == tySequence:
     genTraverseProcSeq(c, "a".rope, typ)
   else:
-    if skipTypes(typ.sons[0], typedescInst+{tyOwned}).kind == tyArray:
+    if skipTypes(typ[0], typedescInst+{tyOwned}).kind == tyArray:
       # C's arrays are broken beyond repair:
-      genTraverseProc(c, "a".rope, typ.sons[0])
+      genTraverseProc(c, "a".rope, typ[0])
     else:
-      genTraverseProc(c, "(*a)".rope, typ.sons[0])
+      genTraverseProc(c, "(*a)".rope, typ[0])
 
   let generatedProc = "$1 {$n$2$3$4}\n" %
         [header, p.s(cpsLocals), p.s(cpsInit), p.s(cpsStmts)]
@@ -167,8 +168,8 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope =
   m.s[cfsProcs].add(generatedProc)
 
   if hcrOn:
-    addf(m.s[cfsProcHeaders], "N_NIMCALL_PTR(void, $1)(void*, NI);\n", [result])
-    addf(m.s[cfsDynLibInit], "\t$1 = (N_NIMCALL_PTR(void, )(void*, NI)) hcrRegisterProc($3, \"$1\", (void*)$2);\n",
+    m.s[cfsProcHeaders].addf("N_NIMCALL_PTR(void, $1)(void*, NI);\n", [result])
+    m.s[cfsDynLibInit].addf("\t$1 = (N_NIMCALL_PTR(void, )(void*, NI)) hcrRegisterProc($3, \"$1\", (void*)$2);\n",
          [result, markerName, getModuleDllPath(m)])
 
 proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope =

compiler/ccgutils.nim

@@ -10,17 +10,17 @@
 # This module declares some helpers for the C code generator.
 
 import
-  ast, astalgo, ropes, hashes, strutils, types, msgs, wordrecg,
+  ast, hashes, strutils, msgs, wordrecg,
   platform, trees, options
 
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
   case n.kind
   of nkStmtList:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       result = getPragmaStmt(n[i], w)
       if result != nil: break
   of nkPragma:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       if whichPragma(n[i]) == w: return n[i]
   else: discard
 
@@ -33,7 +33,7 @@ proc hashString*(conf: ConfigRef; s: string): BiggestInt =
     # we have to use the same bitwidth
     # as the target CPU
     var b = 0'u64
-    for i in 0 ..< len(s):
+    for i in 0..<s.len:
       b = b + uint(s[i])
       b = b + (b shl 10)
       b = b xor (b shr 6)
@@ -43,7 +43,7 @@ proc hashString*(conf: ConfigRef; s: string): BiggestInt =
     result = cast[Hash](b)
   else:
     var a = 0'u32
-    for i in 0 ..< len(s):
+    for i in 0..<s.len:
       a = a + uint32(s[i])
       a = a + (a shl 10)
       a = a xor (a shr 6)
@@ -70,18 +70,18 @@ proc mangle*(name: string): string =
   template special(x) =
     result.add x
     requiresUnderscore = true
-  for i in start..(name.len-1):
+  for i in start..<name.len:
     let c = name[i]
     case c
     of 'a'..'z', '0'..'9', 'A'..'Z':
-      add(result, c)
+      result.add(c)
     of '_':
       # we generate names like 'foo_9' for scope disambiguations and so
       # disallow this here:
       if i > 0 and i < name.len-1 and name[i+1] in Digits:
         discard
       else:
-        add(result, c)
+        result.add(c)
     of '$': special "dollar"
     of '%': special "percent"
     of '&': special "amp"
@@ -102,7 +102,7 @@ proc mangle*(name: string): string =
     of '@': special "at"
     of '|': special "bar"
     else:
-      add(result, "X" & toHex(ord(c), 2))
+      result.add("X" & toHex(ord(c), 2))
       requiresUnderscore = true
   if requiresUnderscore:
     result.add "_"

compiler/cgendata.nim

@@ -10,7 +10,7 @@
 ## This module contains the data structures for the C code generation phase.
 
 import
-  ast, astalgo, ropes, passes, options, intsets, platform, sighashes,
+  ast, ropes, options, intsets,
   tables, ndi, lineinfos, pathutils, modulegraphs
 
 type
@@ -27,8 +27,8 @@ type
     cfsFieldInfo,             # section for field information
     cfsTypeInfo,              # section for type information
     cfsProcHeaders,           # section for C procs prototypes
-    cfsVars,                  # section for C variable declarations
     cfsData,                  # section for C constant data
+    cfsVars,                  # section for C variable declarations
     cfsProcs,                 # section for C procs that are not inline
     cfsInitProc,              # section for the C init proc
     cfsDatInitProc,           # section for the C datInit proc
@@ -58,22 +58,27 @@ type
     id*: int                  # the ID of the label; positive means that it
     label*: Rope              # generated text for the label
                               # nil if label is not used
-    sections*: TCProcSections # the code beloging
+    sections*: TCProcSections # the code belonging
     isLoop*: bool             # whether block is a loop
     nestedTryStmts*: int16    # how many try statements is it nested into
     nestedExceptStmts*: int16 # how many except statements is it nested into
     frameLen*: int16
 
+  TCProcFlag* = enum
+    beforeRetNeeded,
+    threadVarAccessed,
+    hasCurFramePointer,
+    noSafePoints,
+    nimErrorFlagAccessed,
+    nimErrorFlagDeclared,
+    nimErrorFlagDisabled
+
   TCProc = object             # represents C proc that is currently generated
     prc*: PSym                # the Nim proc that this C proc belongs to
-    beforeRetNeeded*: bool    # true iff 'BeforeRet' label for proc is needed
-    threadVarAccessed*: bool  # true if the proc already accessed some threadvar
-    hasCurFramePointer*: bool # true if _nimCurFrame var needed to recover after
-                              # exception is generated
-    noSafePoints*: bool       # the proc doesn't use safe points in exception handling
+    flags*: set[TCProcFlag]
     lastLineInfo*: TLineInfo  # to avoid generating excessive 'nimln' statements
     currLineInfo*: TLineInfo  # AST codegen will make this superfluous
-    nestedTryStmts*: seq[tuple[n: PNode, inExcept: bool]]
+    nestedTryStmts*: seq[tuple[fin: PNode, inExcept: bool, label: Natural]]
                               # in how many nested try statements we are
                               # (the vars must be volatile then)
                               # bool is true when are in the except part of a try block
@@ -86,21 +91,19 @@ type
     options*: TOptions        # options that should be used for code
                               # generation; this is the same as prc.options
                               # unless prc == nil
-    maxFrameLen*: int         # max length of frame descriptor
     module*: BModule          # used to prevent excessive parameter passing
     withinLoop*: int          # > 0 if we are within a loop
     splitDecls*: int          # > 0 if we are in some context for C++ that
                               # requires 'T x = T()' to become 'T x; x = T()'
                               # (yes, C++ is weird like that)
-    gcFrameId*: Natural       # for the GC stack marking
-    gcFrameType*: Rope        # the struct {} we put the GC markers into
+    withinTryWithExcept*: int # required for goto based exception handling
     sigConflicts*: CountTable[string]
 
   TTypeSeq* = seq[PType]
   TypeCache* = Table[SigHash, Rope]
   TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: PSym]]
 
-  Codegenflag* = enum
+  CodegenFlag* = enum
     preventStackTrace,  # true if stack traces need to be prevented
     usesThreadVars,     # true if the module uses a thread var
     frameDeclared,      # hack for ROD support so that we don't declare
@@ -114,11 +117,9 @@ type
     mainModProcs*, mainModInit*, otherModsInit*, mainDatInit*: Rope
     mapping*: Rope             # the generated mapping file (if requested)
     modules*: seq[BModule]     # list of all compiled modules
-    modules_closed*: seq[BModule] # list of the same compiled modules, but in the order they were closed
+    modulesClosed*: seq[BModule] # list of the same compiled modules, but in the order they were closed
     forwardedProcs*: seq[PSym] # proc:s that did not yet have a body
     generatedHeader*: BModule
-    breakPointId*: int
-    breakpoints*: Rope # later the breakpoints are inserted into the main proc
     typeInfoMarker*: TypeCacheWithOwner
     config*: ConfigRef
     graph*: ModuleGraph
@@ -137,7 +138,7 @@ type
 
   TCGen = object of PPassContext # represents a C source file
     s*: TCFileSections        # sections of the C file
-    flags*: set[Codegenflag]
+    flags*: set[CodegenFlag]
     module*: PSym
     filename*: AbsoluteFile
     cfilename*: AbsoluteFile  # filename of the module (including path,
@@ -152,7 +153,7 @@ type
     initProc*: BProc          # code for init procedure
     preInitProc*: BProc       # code executed before the init proc
     hcrCreateTypeInfosProc*: Rope # type info globals are in here when HCR=on
-    inHcrInitGuard*: bool     # We are currently withing a HCR reloading guard.
+    inHcrInitGuard*: bool     # We are currently within a HCR reloading guard.
     typeStack*: TTypeSeq      # used for type generation
     dataCache*: TNodeTable
     typeNodes*, nimTypes*: int # used for type info generation
@@ -174,7 +175,7 @@ proc includeHeader*(this: BModule; header: string) =
 
 proc s*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # section in the current block
-  result = p.blocks[p.blocks.len-1].sections[s]
+  result = p.blocks[^1].sections[s]
 
 proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # top level proc sections
@@ -196,6 +197,6 @@ proc newModuleList*(g: ModuleGraph): BModuleList =
     config: g.config, graph: g, nimtvDeclared: initIntSet())
 
 iterator cgenModules*(g: BModuleList): BModule =
-  for m in g.modules_closed:
+  for m in g.modulesClosed:
     # iterate modules in the order they were closed
     yield m

compiler/cgmeth.nim

@@ -7,10 +7,10 @@
 #    distribution, for details about the copyright.
 #
 
-## This module implements code generation for multi methods.
+## This module implements code generation for methods.
 
 import
-  intsets, options, ast, astalgo, msgs, idents, renderer, types, magicsys,
+  intsets, options, ast, msgs, idents, renderer, types, magicsys,
   sempass2, strutils, modulegraphs, lineinfos
 
 proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
@@ -23,11 +23,11 @@ proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
       result = n
     elif diff < 0:
       result = newNodeIT(nkObjUpConv, n.info, d)
-      addSon(result, n)
+      result.add n
       if downcast: internalError(conf, n.info, "cgmeth.genConv: no upcast allowed")
     elif diff > 0:
       result = newNodeIT(nkObjDownConv, n.info, d)
-      addSon(result, n)
+      result.add n
       if not downcast:
         internalError(conf, n.info, "cgmeth.genConv: no downcast allowed")
     else:
@@ -44,26 +44,26 @@ proc getDispatcher*(s: PSym): PSym =
 proc methodCall*(n: PNode; conf: ConfigRef): PNode =
   result = n
   # replace ordinary method by dispatcher method:
-  let disp = getDispatcher(result.sons[0].sym)
+  let disp = getDispatcher(result[0].sym)
   if disp != nil:
-    result.sons[0].sym = disp
+    result[0].sym = disp
     # change the arguments to up/downcasts to fit the dispatcher's parameters:
-    for i in 1 ..< sonsLen(result):
-      result.sons[i] = genConv(result.sons[i], disp.typ.sons[i], true, conf)
+    for i in 1..<result.len:
+      result[i] = genConv(result[i], disp.typ[i], true, conf)
   else:
-    localError(conf, n.info, "'" & $result.sons[0] & "' lacks a dispatcher")
+    localError(conf, n.info, "'" & $result[0] & "' lacks a dispatcher")
 
 type
   MethodResult = enum No, Invalid, Yes
 
 proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
   if a.name.id != b.name.id: return
-  if sonsLen(a.typ) != sonsLen(b.typ):
+  if a.typ.len != b.typ.len:
     return
 
-  for i in 1 ..< sonsLen(a.typ):
-    var aa = a.typ.sons[i]
-    var bb = b.typ.sons[i]
+  for i in 1..<a.typ.len:
+    var aa = a.typ[i]
+    var bb = b.typ[i]
     while true:
       aa = skipTypes(aa, {tyGenericInst, tyAlias})
       bb = skipTypes(bb, {tyGenericInst, tyAlias})
@@ -72,7 +72,7 @@ proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
         bb = bb.lastSon
       else:
         break
-    if sameType(a.typ.sons[i], b.typ.sons[i]):
+    if sameType(a.typ[i], b.typ[i]):
       if aa.kind == tyObject and result != Invalid:
         result = Yes
     elif aa.kind == tyObject and bb.kind == tyObject and (i == 1 or multiMethods):
@@ -90,22 +90,22 @@ proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
       return No
   if result == Yes:
     # check for return type:
-    if not sameTypeOrNil(a.typ.sons[0], b.typ.sons[0]):
-      if b.typ.sons[0] != nil and b.typ.sons[0].kind == tyUntyped:
+    if not sameTypeOrNil(a.typ[0], b.typ[0]):
+      if b.typ[0] != nil and b.typ[0].kind == tyUntyped:
         # infer 'auto' from the base to make it consistent:
-        b.typ.sons[0] = a.typ.sons[0]
+        b.typ[0] = a.typ[0]
       else:
         return No
 
 proc attachDispatcher(s: PSym, dispatcher: PNode) =
   if dispatcherPos < s.ast.len:
     # we've added a dispatcher already, so overwrite it
-    s.ast.sons[dispatcherPos] = dispatcher
+    s.ast[dispatcherPos] = dispatcher
   else:
     setLen(s.ast.sons, dispatcherPos+1)
     if s.ast[resultPos] == nil:
       s.ast[resultPos] = newNodeI(nkEmpty, s.info)
-    s.ast.sons[dispatcherPos] = dispatcher
+    s.ast[dispatcherPos] = dispatcher
 
 proc createDispatcher(s: PSym): PSym =
   var disp = copySym(s)
@@ -115,16 +115,16 @@ proc createDispatcher(s: PSym): PSym =
   # we can't inline the dispatcher itself (for now):
   if disp.typ.callConv == ccInline: disp.typ.callConv = ccDefault
   disp.ast = copyTree(s.ast)
-  disp.ast.sons[bodyPos] = newNodeI(nkEmpty, s.info)
+  disp.ast[bodyPos] = newNodeI(nkEmpty, s.info)
   disp.loc.r = nil
-  if s.typ.sons[0] != nil:
-    if disp.ast.sonsLen > resultPos:
-      disp.ast.sons[resultPos].sym = copySym(s.ast.sons[resultPos].sym)
+  if s.typ[0] != nil:
+    if disp.ast.len > resultPos:
+      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym)
     else:
       # We've encountered a method prototype without a filled-in
       # resultPos slot. We put a placeholder in there that will
       # be updated in fixupDispatcher().
-      disp.ast.addSon(newNodeI(nkEmpty, s.info))
+      disp.ast.add newNodeI(nkEmpty, s.info)
   attachDispatcher(s, newSymNode(disp))
   # attach to itself to prevent bugs:
   attachDispatcher(disp, newSymNode(disp))
@@ -136,9 +136,9 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
   # from later definitions, particularly the resultPos slot. Also,
   # the lock level of the dispatcher needs to be updated/checked
   # against that of the method.
-  if disp.ast.sonsLen > resultPos and meth.ast.sonsLen > resultPos and
-     disp.ast.sons[resultPos].kind == nkEmpty:
-    disp.ast.sons[resultPos] = copyTree(meth.ast.sons[resultPos])
+  if disp.ast.len > resultPos and meth.ast.len > resultPos and
+     disp.ast[resultPos].kind == nkEmpty:
+    disp.ast[resultPos] = copyTree(meth.ast[resultPos])
 
   # The following code works only with lock levels, so we disable
   # it when they're not available.
@@ -158,13 +158,12 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
         disp.typ.lockLevel = meth.typ.lockLevel
 
 proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
-  let L = len(g.methods)
   var witness: PSym
-  for i in 0 ..< L:
+  for i in 0..<g.methods.len:
     let disp = g.methods[i].dispatcher
     case sameMethodBucket(disp, s, multimethods = optMultiMethods in g.config.globalOptions)
     of Yes:
-      add(g.methods[i].methods, s)
+      g.methods[i].methods.add(s)
       attachDispatcher(s, disp.ast[dispatcherPos])
       fixupDispatcher(s, disp, g.config)
       #echo "fixup ", disp.name.s, " ", disp.id
@@ -178,7 +177,7 @@ proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
     of Invalid:
       if witness.isNil: witness = g.methods[i].methods[0]
   # create a new dispatcher:
-  add(g.methods, (methods: @[s], dispatcher: createDispatcher(s)))
+  g.methods.add((methods: @[s], dispatcher: createDispatcher(s)))
   #echo "adding ", s.info
   #if fromCache:
   #  internalError(s.info, "no method dispatcher found")
@@ -190,32 +189,32 @@ proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
 
 proc relevantCol(methods: seq[PSym], col: int): bool =
   # returns true iff the position is relevant
-  var t = methods[0].typ.sons[col].skipTypes(skipPtrs)
+  var t = methods[0].typ[col].skipTypes(skipPtrs)
   if t.kind == tyObject:
-    for i in 1 .. high(methods):
-      let t2 = skipTypes(methods[i].typ.sons[col], skipPtrs)
+    for i in 1..high(methods):
+      let t2 = skipTypes(methods[i].typ[col], skipPtrs)
       if not sameType(t2, t):
         return true
 
 proc cmpSignatures(a, b: PSym, relevantCols: IntSet): int =
-  for col in 1 ..< sonsLen(a.typ):
+  for col in 1..<a.typ.len:
     if contains(relevantCols, col):
-      var aa = skipTypes(a.typ.sons[col], skipPtrs)
-      var bb = skipTypes(b.typ.sons[col], skipPtrs)
+      var aa = skipTypes(a.typ[col], skipPtrs)
+      var bb = skipTypes(b.typ[col], skipPtrs)
       var d = inheritanceDiff(aa, bb)
       if (d != high(int)) and d != 0:
         return d
 
 proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
   # we use shellsort here; fast and simple
-  var n = len(a)
+  var n = a.len
   var h = 1
   while true:
     h = 3 * h + 1
     if h > n: break
   while true:
     h = h div 3
-    for i in h ..< n:
+    for i in h..<n:
       var v = a[i]
       var j = i
       while cmpSignatures(a[j - h], v, relevantCols) >= 0:
@@ -228,71 +227,70 @@ proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
 proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet): PSym =
   var base = methods[0].ast[dispatcherPos].sym
   result = base
-  var paramLen = sonsLen(base.typ)
+  var paramLen = base.typ.len
   var nilchecks = newNodeI(nkStmtList, base.info)
   var disp = newNodeI(nkIfStmt, base.info)
-  var ands = getSysMagic(g, unknownLineInfo(), "and", mAnd)
-  var iss = getSysMagic(g, unknownLineInfo(), "of", mOf)
-  let boolType = getSysType(g, unknownLineInfo(), tyBool)
-  for col in 1 ..< paramLen:
+  var ands = getSysMagic(g, unknownLineInfo, "and", mAnd)
+  var iss = getSysMagic(g, unknownLineInfo, "of", mOf)
+  let boolType = getSysType(g, unknownLineInfo, tyBool)
+  for col in 1..<paramLen:
     if contains(relevantCols, col):
-      let param = base.typ.n.sons[col].sym
+      let param = base.typ.n[col].sym
       if param.typ.skipTypes(abstractInst).kind in {tyRef, tyPtr}:
-        addSon(nilchecks, newTree(nkCall,
-            newSymNode(getCompilerProc(g, "chckNilDisp")), newSymNode(param)))
-  for meth in 0 .. high(methods):
+        nilchecks.add newTree(nkCall,
+            newSymNode(getCompilerProc(g, "chckNilDisp")), newSymNode(param))
+  for meth in 0..high(methods):
     var curr = methods[meth]      # generate condition:
     var cond: PNode = nil
-    for col in 1 ..< paramLen:
+    for col in 1..<paramLen:
       if contains(relevantCols, col):
         var isn = newNodeIT(nkCall, base.info, boolType)
-        addSon(isn, newSymNode(iss))
-        let param = base.typ.n.sons[col].sym
-        addSon(isn, newSymNode(param))
-        addSon(isn, newNodeIT(nkType, base.info, curr.typ.sons[col]))
+        isn.add newSymNode(iss)
+        let param = base.typ.n[col].sym
+        isn.add newSymNode(param)
+        isn.add newNodeIT(nkType, base.info, curr.typ[col])
         if cond != nil:
           var a = newNodeIT(nkCall, base.info, boolType)
-          addSon(a, newSymNode(ands))
-          addSon(a, cond)
-          addSon(a, isn)
+          a.add newSymNode(ands)
+          a.add cond
+          a.add isn
           cond = a
         else:
           cond = isn
-    let retTyp = base.typ.sons[0]
+    let retTyp = base.typ[0]
     let call = newNodeIT(nkCall, base.info, retTyp)
-    addSon(call, newSymNode(curr))
-    for col in 1 ..< paramLen:
-      addSon(call, genConv(newSymNode(base.typ.n.sons[col].sym),
-                           curr.typ.sons[col], false, g.config))
+    call.add newSymNode(curr)
+    for col in 1..<paramLen:
+      call.add genConv(newSymNode(base.typ.n[col].sym),
+                           curr.typ[col], false, g.config)
     var ret: PNode
     if retTyp != nil:
       var a = newNodeI(nkFastAsgn, base.info)
-      addSon(a, newSymNode(base.ast.sons[resultPos].sym))
-      addSon(a, call)
+      a.add newSymNode(base.ast[resultPos].sym)
+      a.add call
       ret = newNodeI(nkReturnStmt, base.info)
-      addSon(ret, a)
+      ret.add a
     else:
       ret = call
     if cond != nil:
       var a = newNodeI(nkElifBranch, base.info)
-      addSon(a, cond)
-      addSon(a, ret)
-      addSon(disp, a)
+      a.add cond
+      a.add ret
+      disp.add a
     else:
       disp = ret
   nilchecks.add disp
   nilchecks.flags.incl nfTransf # should not be further transformed
-  result.ast.sons[bodyPos] = nilchecks
+  result.ast[bodyPos] = nilchecks
 
 proc generateMethodDispatchers*(g: ModuleGraph): PNode =
   result = newNode(nkStmtList)
-  for bucket in 0 ..< len(g.methods):
+  for bucket in 0..<g.methods.len:
     var relevantCols = initIntSet()
-    for col in 1 ..< sonsLen(g.methods[bucket].methods[0].typ):
+    for col in 1..<g.methods[bucket].methods[0].typ.len:
       if relevantCol(g.methods[bucket].methods, col): incl(relevantCols, col)
       if optMultiMethods notin g.config.globalOptions:
         # if multi-methods are not enabled, we are interested only in the first field
         break
     sortBucket(g.methods[bucket].methods, relevantCols)
-    addSon(result,
-           newSymNode(genDispatcher(g, g.methods[bucket].methods, relevantCols)))
+    result.add newSymNode(genDispatcher(g, g.methods[bucket].methods, relevantCols))

compiler/closureiters.nim

@@ -90,9 +90,6 @@
 #  :unrollFinally = true
 #  goto nearestFinally (or -1 if not exists)
 #
-# Every finally block calls closureIterEndFinally() upon its successful
-# completion.
-#
 # Example:
 #
 # try:
@@ -126,13 +123,15 @@
 #     if :curExc.isNil:
 #       return :tmpResult
 #     else:
+#       closureIterSetupExc(nil)
 #       raise
 #   state = -1 # Goto next state. In this case we just exit
 #   break :stateLoop
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, idents,
-  renderer, types, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
+  ast, msgs, idents,
+  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos,
+  tables, options
 
 type
   Ctx = object
@@ -156,8 +155,8 @@ type
                     # is their finally. For finally it is parent finally. Otherwise -1
 
 const
-  nkSkip = { nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
-            nkCommentStmt } + procDefs
+  nkSkip = {nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
+            nkCommentStmt} + procDefs
 
 proc newStateAccess(ctx: var Ctx): PNode =
   if ctx.stateVarSym.isNil:
@@ -174,7 +173,7 @@ proc newStateAssgn(ctx: var Ctx, toValue: PNode): PNode =
 proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
   # Creates state assignment:
   #   :state = stateNo
-  ctx.newStateAssgn(newIntTypeNode(nkIntLit, stateNo, ctx.g.getSysType(TLineInfo(), tyInt)))
+  ctx.newStateAssgn(newIntTypeNode(stateNo, ctx.g.getSysType(TLineInfo(), tyInt)))
 
 proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
   result = newSym(skVar, getIdent(ctx.g.cache, name), ctx.fn, ctx.fn.info)
@@ -239,7 +238,7 @@ proc toStmtList(n: PNode): PNode =
 proc addGotoOut(n: PNode, gotoOut: PNode): PNode =
   # Make sure `n` is a stmtlist, and ends with `gotoOut`
   result = toStmtList(n)
-  if result.len == 0 or result.sons[^1].kind != nkGotoState:
+  if result.len == 0 or result[^1].kind != nkGotoState:
     result.add(gotoOut)
 
 proc newTempVar(ctx: var Ctx, typ: PType): PSym =
@@ -275,7 +274,7 @@ proc transformBreaksAndContinuesInWhile(ctx: var Ctx, n: PNode, before, after: P
     if ctx.blockLevel == 0:
       result = after
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformBreaksAndContinuesInWhile(n[i], before, after)
 
 proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode =
@@ -295,7 +294,7 @@ proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode
       if label.kind == nkSym and n[0].sym == label.sym:
         result = after
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformBreaksInBlock(n[i], label, after)
 
 proc newNullifyCurExc(ctx: var Ctx, info: TLineInfo): PNode =
@@ -320,7 +319,7 @@ proc collectExceptState(ctx: var Ctx, n: PNode): PNode {.inline.} =
 
       if c.len > 1:
         var cond: PNode
-        for i in 0 .. c.len - 2:
+        for i in 0..<c.len - 1:
           assert(c[i].kind == nkType)
           let nextCond = newTree(nkCall,
             newSymNode(g.getSysMagic(c.info, "of", mOf)),
@@ -359,7 +358,7 @@ proc addElseToExcept(ctx: var Ctx, n: PNode) =
     block: # :unrollFinally = true
       branchBody.add(newTree(nkAsgn,
         ctx.newUnrollFinallyAccess(n.info),
-        newIntTypeNode(nkIntLit, 1, ctx.g.getSysType(n.info, tyBool))))
+        newIntTypeNode(1, ctx.g.getSysType(n.info, tyBool))))
 
     block: # :curExc = getCurrentException()
       branchBody.add(newTree(nkAsgn,
@@ -403,7 +402,7 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
   var n = n
   while n.kind == nkStmtListExpr:
     result.s.sons.add(n.sons)
-    result.s.sons.setLen(result.s.sons.len - 1) # delete last son
+    result.s.sons.setLen(result.s.len - 1) # delete last son
     n = n[^1]
 
   result.res = n
@@ -437,7 +436,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkYieldStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -451,7 +450,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkPar, nkObjConstr, nkTupleConstr, nkBracket:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -461,7 +460,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       if n.typ.isNil: internalError(ctx.g.config, "lowerStmtListExprs: constr typ.isNil")
       result.typ = n.typ
 
-      for i in 0 ..< n.len:
+      for i in 0..<n.len:
         case n[i].kind
         of nkExprColonExpr:
           if n[i][1].kind == nkStmtListExpr:
@@ -477,7 +476,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkIfStmt, nkIfExpr:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -543,7 +542,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkTryStmt, nkHiddenTryStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -556,7 +555,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         let tmp = ctx.newTempVar(n.typ)
 
         n[0] = ctx.convertExprBodyToAsgn(n[0], tmp)
-        for i in 1 ..< n.len:
+        for i in 1..<n.len:
           let branch = n[i]
           case branch.kind
           of nkExceptBranch:
@@ -573,7 +572,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkCaseStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -591,7 +590,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           result.add(st)
           n[0] = ex
 
-        for i in 1 ..< n.len:
+        for i in 1..<n.len:
           let branch = n[i]
           case branch.kind
           of nkOfBranch:
@@ -603,9 +602,9 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         result.add(n)
         result.add(ctx.newEnvVarAccess(tmp))
 
-  of nkCallKinds:
+  of nkCallKinds, nkChckRange, nkChckRangeF, nkChckRange64:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -645,7 +644,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         result.add(ifNode)
         result.add(ctx.newEnvVarAccess(tmp))
       else:
-        for i in 0 ..< n.len:
+        for i in 0..<n.len:
           if n[i].kind == nkStmtListExpr:
             let (st, ex) = exprToStmtList(n[i])
             result.add(st)
@@ -674,7 +673,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkDiscardStmt, nkReturnStmt, nkRaiseStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -685,9 +684,10 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       n[0] = ex
       result.add(n)
 
-  of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv, nkObjDownConv:
+  of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv, nkObjDownConv,
+      nkDerefExpr, nkHiddenDeref:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -701,7 +701,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkAsgn, nkFastAsgn:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -759,7 +759,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         n[0] = newSymNode(ctx.g.getSysSym(n[0].info, "true"))
         n[1] = newBody
 
-  of nkDotExpr:
+  of nkDotExpr, nkCheckedFieldExpr:
     var ns = false
     n[0] = ctx.lowerStmtListExprs(n[0], ns)
     if ns:
@@ -779,14 +779,14 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       result = newNodeI(nkStmtListExpr, n.info)
       result.typ = n.typ
       let (st, ex) = exprToStmtList(n[1])
-      n.kind = nkBlockStmt
+      n.transitionSonsKind(nkBlockStmt)
       n.typ = nil
       n[1] = st
       result.add(n)
       result.add(ex)
 
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], needsSplit)
 
 proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
@@ -809,10 +809,11 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   let retStmt = newTree(nkReturnStmt, asgn)
   let branch = newTree(nkElifBranch, cmp, retStmt)
 
-  # The C++ backend requires `getCurrentException` here.
-  let raiseStmt = newTree(nkRaiseStmt, ctx.g.callCodegenProc("getCurrentException"))
+  let nullifyExc = newTree(nkCall, newSymNode(ctx.g.getCompilerProc("closureIterSetupExc")), nilnode)
+  nullifyExc.info = info
+  let raiseStmt = newTree(nkRaiseStmt, curExc)
   raiseStmt.info = info
-  let elseBranch = newTree(nkElse, raiseStmt)
+  let elseBranch = newTree(nkElse, newTree(nkStmtList, nullifyExc, raiseStmt))
 
   let ifBody = newTree(nkIfStmt, branch, elseBranch)
   let elifBranch = newTree(nkElifBranch, ctx.newUnrollFinallyAccess(info), ifBody)
@@ -832,13 +833,14 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
     block: # :unrollFinally = true
       let asgn = newNodeI(nkAsgn, n.info)
       asgn.add(ctx.newUnrollFinallyAccess(n.info))
-      asgn.add(newIntTypeNode(nkIntLit, 1, ctx.g.getSysType(n.info, tyBool)))
+      asgn.add(newIntTypeNode(1, ctx.g.getSysType(n.info, tyBool)))
       result.add(asgn)
 
     if n[0].kind != nkEmpty:
       let asgnTmpResult = newNodeI(nkAsgn, n.info)
       asgnTmpResult.add(ctx.newTmpResultAccess())
-      asgnTmpResult.add(n[0])
+      let x = if n[0].kind in {nkAsgn, nkFastAsgn}: n[0][1] else: n[0]
+      asgnTmpResult.add(x)
       result.add(asgnTmpResult)
 
     result.add(ctx.newNullifyCurExc(n.info))
@@ -849,161 +851,160 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
   of nkSkip:
     discard
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformReturnsInTry(n[i])
 
 proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode =
   result = n
-  case n.kind:
-    of nkSkip:
-      discard
+  case n.kind
+  of nkSkip: discard
 
-    of nkStmtList, nkStmtListExpr:
-      result = addGotoOut(result, gotoOut)
-      for i in 0 ..< n.len:
-        if n[i].hasYields:
-          # Create a new split
-          let go = newNodeI(nkGotoState, n[i].info)
-          n[i] = ctx.transformClosureIteratorBody(n[i], go)
+  of nkStmtList, nkStmtListExpr:
+    result = addGotoOut(result, gotoOut)
+    for i in 0..<n.len:
+      if n[i].hasYields:
+        # Create a new split
+        let go = newNodeI(nkGotoState, n[i].info)
+        n[i] = ctx.transformClosureIteratorBody(n[i], go)
 
-          let s = newNodeI(nkStmtList, n[i + 1].info)
-          for j in i + 1 ..< n.len:
-            s.add(n[j])
+        let s = newNodeI(nkStmtList, n[i + 1].info)
+        for j in i + 1..<n.len:
+          s.add(n[j])
 
-          n.sons.setLen(i + 1)
-          discard ctx.newState(s, go)
-          if ctx.transformClosureIteratorBody(s, gotoOut) != s:
-            internalError(ctx.g.config, "transformClosureIteratorBody != s")
-          break
+        n.sons.setLen(i + 1)
+        discard ctx.newState(s, go)
+        if ctx.transformClosureIteratorBody(s, gotoOut) != s:
+          internalError(ctx.g.config, "transformClosureIteratorBody != s")
+        break
 
-    of nkYieldStmt:
-      result = newNodeI(nkStmtList, n.info)
-      result.add(n)
-      result.add(gotoOut)
+  of nkYieldStmt:
+    result = newNodeI(nkStmtList, n.info)
+    result.add(n)
+    result.add(gotoOut)
 
-    of nkElse, nkElseExpr:
-      result[0] = addGotoOut(result[0], gotoOut)
-      result[0] = ctx.transformClosureIteratorBody(result[0], gotoOut)
+  of nkElse, nkElseExpr:
+    result[0] = addGotoOut(result[0], gotoOut)
+    result[0] = ctx.transformClosureIteratorBody(result[0], gotoOut)
 
-    of nkElifBranch, nkElifExpr, nkOfBranch:
-      result[^1] = addGotoOut(result[^1], gotoOut)
-      result[^1] = ctx.transformClosureIteratorBody(result[^1], gotoOut)
-
-    of nkIfStmt, nkCaseStmt:
-      for i in 0 ..< n.len:
-        n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
-      if n[^1].kind != nkElse:
-        # We don't have an else branch, but every possible branch has to end with
-        # gotoOut, so add else here.
-        let elseBranch = newTree(nkElse, gotoOut)
-        n.add(elseBranch)
-
-    of nkWhileStmt:
-      # while e:
-      #   s
-      # ->
-      # BEGIN_STATE:
-      #   if e:
-      #     s
-      #     goto BEGIN_STATE
-      #   else:
-      #     goto OUT
-
-      result = newNodeI(nkGotoState, n.info)
-
-      let s = newNodeI(nkStmtList, n.info)
-      discard ctx.newState(s, result)
-      let ifNode = newNodeI(nkIfStmt, n.info)
-      let elifBranch = newNodeI(nkElifBranch, n.info)
-      elifBranch.add(n[0])
-
-      var body = addGotoOut(n[1], result)
-
-      body = ctx.transformBreaksAndContinuesInWhile(body, result, gotoOut)
-      body = ctx.transformClosureIteratorBody(body, result)
-
-      elifBranch.add(body)
-      ifNode.add(elifBranch)
+  of nkElifBranch, nkElifExpr, nkOfBranch:
+    result[^1] = addGotoOut(result[^1], gotoOut)
+    result[^1] = ctx.transformClosureIteratorBody(result[^1], gotoOut)
 
+  of nkIfStmt, nkCaseStmt:
+    for i in 0..<n.len:
+      n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
+    if n[^1].kind != nkElse:
+      # We don't have an else branch, but every possible branch has to end with
+      # gotoOut, so add else here.
       let elseBranch = newTree(nkElse, gotoOut)
-      ifNode.add(elseBranch)
-      s.add(ifNode)
+      n.add(elseBranch)
 
-    of nkBlockStmt:
-      result[1] = addGotoOut(result[1], gotoOut)
-      result[1] = ctx.transformBreaksInBlock(result[1], result[0], gotoOut)
-      result[1] = ctx.transformClosureIteratorBody(result[1], gotoOut)
+  of nkWhileStmt:
+    # while e:
+    #   s
+    # ->
+    # BEGIN_STATE:
+    #   if e:
+    #     s
+    #     goto BEGIN_STATE
+    #   else:
+    #     goto OUT
 
-    of nkTryStmt, nkHiddenTryStmt:
-      # See explanation above about how this works
-      ctx.hasExceptions = true
+    result = newNodeI(nkGotoState, n.info)
 
-      result = newNodeI(nkGotoState, n.info)
-      var tryBody = toStmtList(n[0])
-      var exceptBody = ctx.collectExceptState(n)
-      var finallyBody = newTree(nkStmtList, getFinallyNode(ctx, n))
-      finallyBody = ctx.transformReturnsInTry(finallyBody)
-      finallyBody.add(ctx.newEndFinallyNode(finallyBody.info))
+    let s = newNodeI(nkStmtList, n.info)
+    discard ctx.newState(s, result)
+    let ifNode = newNodeI(nkIfStmt, n.info)
+    let elifBranch = newNodeI(nkElifBranch, n.info)
+    elifBranch.add(n[0])
 
-      # The following index calculation is based on the knowledge how state
-      # indexes are assigned
-      let tryIdx = ctx.states.len
-      var exceptIdx, finallyIdx: int
-      if exceptBody.kind != nkEmpty:
-        exceptIdx = -(tryIdx + 1)
-        finallyIdx = tryIdx + 2
-      else:
-        exceptIdx = tryIdx + 1
-        finallyIdx = tryIdx + 1
+    var body = addGotoOut(n[1], result)
 
-      let outToFinally = newNodeI(nkGotoState, finallyBody.info)
+    body = ctx.transformBreaksAndContinuesInWhile(body, result, gotoOut)
+    body = ctx.transformClosureIteratorBody(body, result)
 
-      block: # Create initial states.
-        let oldExcHandlingState = ctx.curExcHandlingState
-        ctx.curExcHandlingState = exceptIdx
-        let realTryIdx = ctx.newState(tryBody, result)
-        assert(realTryIdx == tryIdx)
+    elifBranch.add(body)
+    ifNode.add(elifBranch)
 
-        if exceptBody.kind != nkEmpty:
-          ctx.curExcHandlingState = finallyIdx
-          let realExceptIdx = ctx.newState(exceptBody, nil)
-          assert(realExceptIdx == -exceptIdx)
+    let elseBranch = newTree(nkElse, gotoOut)
+    ifNode.add(elseBranch)
+    s.add(ifNode)
 
-        ctx.curExcHandlingState = oldExcHandlingState
-        let realFinallyIdx = ctx.newState(finallyBody, outToFinally)
-        assert(realFinallyIdx == finallyIdx)
+  of nkBlockStmt:
+    result[1] = addGotoOut(result[1], gotoOut)
+    result[1] = ctx.transformBreaksInBlock(result[1], result[0], gotoOut)
+    result[1] = ctx.transformClosureIteratorBody(result[1], gotoOut)
 
-      block: # Subdivide the states
-        let oldNearestFinally = ctx.nearestFinally
-        ctx.nearestFinally = finallyIdx
+  of nkTryStmt, nkHiddenTryStmt:
+    # See explanation above about how this works
+    ctx.hasExceptions = true
 
-        let oldExcHandlingState = ctx.curExcHandlingState
-
-        ctx.curExcHandlingState = exceptIdx
-
-        if ctx.transformReturnsInTry(tryBody) != tryBody:
-          internalError(ctx.g.config, "transformReturnsInTry != tryBody")
-        if ctx.transformClosureIteratorBody(tryBody, outToFinally) != tryBody:
-          internalError(ctx.g.config, "transformClosureIteratorBody != tryBody")
-
-        ctx.curExcHandlingState = finallyIdx
-        ctx.addElseToExcept(exceptBody)
-        if ctx.transformReturnsInTry(exceptBody) != exceptBody:
-          internalError(ctx.g.config, "transformReturnsInTry != exceptBody")
-        if ctx.transformClosureIteratorBody(exceptBody, outToFinally) != exceptBody:
-          internalError(ctx.g.config, "transformClosureIteratorBody != exceptBody")
-
-        ctx.curExcHandlingState = oldExcHandlingState
-        ctx.nearestFinally = oldNearestFinally
-        if ctx.transformClosureIteratorBody(finallyBody, gotoOut) != finallyBody:
-          internalError(ctx.g.config, "transformClosureIteratorBody != finallyBody")
-
-    of nkGotoState, nkForStmt:
-      internalError(ctx.g.config, "closure iter " & $n.kind)
+    result = newNodeI(nkGotoState, n.info)
+    var tryBody = toStmtList(n[0])
+    var exceptBody = ctx.collectExceptState(n)
+    var finallyBody = newTree(nkStmtList, getFinallyNode(ctx, n))
+    finallyBody = ctx.transformReturnsInTry(finallyBody)
+    finallyBody.add(ctx.newEndFinallyNode(finallyBody.info))
 
+    # The following index calculation is based on the knowledge how state
+    # indexes are assigned
+    let tryIdx = ctx.states.len
+    var exceptIdx, finallyIdx: int
+    if exceptBody.kind != nkEmpty:
+      exceptIdx = -(tryIdx + 1)
+      finallyIdx = tryIdx + 2
     else:
-      for i in 0 ..< n.len:
-        n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
+      exceptIdx = tryIdx + 1
+      finallyIdx = tryIdx + 1
+
+    let outToFinally = newNodeI(nkGotoState, finallyBody.info)
+
+    block: # Create initial states.
+      let oldExcHandlingState = ctx.curExcHandlingState
+      ctx.curExcHandlingState = exceptIdx
+      let realTryIdx = ctx.newState(tryBody, result)
+      assert(realTryIdx == tryIdx)
+
+      if exceptBody.kind != nkEmpty:
+        ctx.curExcHandlingState = finallyIdx
+        let realExceptIdx = ctx.newState(exceptBody, nil)
+        assert(realExceptIdx == -exceptIdx)
+
+      ctx.curExcHandlingState = oldExcHandlingState
+      let realFinallyIdx = ctx.newState(finallyBody, outToFinally)
+      assert(realFinallyIdx == finallyIdx)
+
+    block: # Subdivide the states
+      let oldNearestFinally = ctx.nearestFinally
+      ctx.nearestFinally = finallyIdx
+
+      let oldExcHandlingState = ctx.curExcHandlingState
+
+      ctx.curExcHandlingState = exceptIdx
+
+      if ctx.transformReturnsInTry(tryBody) != tryBody:
+        internalError(ctx.g.config, "transformReturnsInTry != tryBody")
+      if ctx.transformClosureIteratorBody(tryBody, outToFinally) != tryBody:
+        internalError(ctx.g.config, "transformClosureIteratorBody != tryBody")
+
+      ctx.curExcHandlingState = finallyIdx
+      ctx.addElseToExcept(exceptBody)
+      if ctx.transformReturnsInTry(exceptBody) != exceptBody:
+        internalError(ctx.g.config, "transformReturnsInTry != exceptBody")
+      if ctx.transformClosureIteratorBody(exceptBody, outToFinally) != exceptBody:
+        internalError(ctx.g.config, "transformClosureIteratorBody != exceptBody")
+
+      ctx.curExcHandlingState = oldExcHandlingState
+      ctx.nearestFinally = oldNearestFinally
+      if ctx.transformClosureIteratorBody(finallyBody, gotoOut) != finallyBody:
+        internalError(ctx.g.config, "transformClosureIteratorBody != finallyBody")
+
+  of nkGotoState, nkForStmt:
+    internalError(ctx.g.config, "closure iter " & $n.kind)
+
+  else:
+    for i in 0..<n.len:
+      n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
 
 proc stateFromGotoState(n: PNode): int =
   assert(n.kind == nkGotoState)
@@ -1039,18 +1040,18 @@ proc transformStateAssignments(ctx: var Ctx, n: PNode): PNode =
       result.add(ctx.newStateAssgn(stateFromGotoState(n[1])))
 
       var retStmt = newNodeI(nkReturnStmt, n.info)
-      if n[0].sons[0].kind != nkEmpty:
-        var a = newNodeI(nkAsgn, n[0].sons[0].info)
-        var retVal = n[0].sons[0] #liftCapturedVars(n.sons[0], owner, d, c)
-        addSon(a, newSymNode(getClosureIterResult(ctx.g, ctx.fn)))
-        addSon(a, retVal)
+      if n[0][0].kind != nkEmpty:
+        var a = newNodeI(nkAsgn, n[0][0].info)
+        var retVal = n[0][0] #liftCapturedVars(n[0], owner, d, c)
+        a.add newSymNode(getClosureIterResult(ctx.g, ctx.fn))
+        a.add retVal
         retStmt.add(a)
       else:
         retStmt.add(ctx.g.emptyNode)
 
       result.add(retStmt)
     else:
-      for i in 0 ..< n.len:
+      for i in 0..<n.len:
         n[i] = ctx.transformStateAssignments(n[i])
 
   of nkSkip:
@@ -1070,7 +1071,7 @@ proc transformStateAssignments(ctx: var Ctx, n: PNode): PNode =
     result.add(breakState)
 
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformStateAssignments(n[i])
 
 proc skipStmtList(ctx: Ctx; n: PNode): PNode =
@@ -1102,7 +1103,7 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
 
   result = ctx.states[stateIdx][0].intVal.int
 
-proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
+proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
   result = n
   case n.kind
   of nkSkip:
@@ -1111,7 +1112,7 @@ proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
     result = copyTree(n)
     result[0].intVal = ctx.skipEmptyStates(result[0].intVal.int)
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.skipThroughEmptyStates(n[i])
 
 proc newArrayType(g: ModuleGraph; n: int, t: PType, owner: PSym): PType =
@@ -1149,7 +1150,6 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
 
   # :state = exceptionTable[:state]
   block:
-
     # exceptionTable[:state]
     let getNextState = newTree(nkBracketExpr,
       ctx.createExceptionTable(),
@@ -1164,7 +1164,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
     let cond = newTree(nkCall,
       ctx.g.getSysMagic(info, "==", mEqI).newSymNode(),
       ctx.newStateAccess(),
-      newIntTypeNode(nkIntLit, 0, intTyp))
+      newIntTypeNode(0, intTyp))
     cond.typ = boolTyp
 
     let raiseStmt = newTree(nkRaiseStmt, ctx.g.emptyNode)
@@ -1176,7 +1176,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
   block:
     let cond = newTree(nkCall,
       ctx.g.getSysMagic(info, "<", mLtI).newSymNode,
-      newIntTypeNode(nkIntLit, 0, intTyp),
+      newIntTypeNode(0, intTyp),
       ctx.newStateAccess())
     cond.typ = boolTyp
 
@@ -1188,7 +1188,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
     let cond = newTree(nkCall,
       ctx.g.getSysMagic(info, "<", mLtI).newSymNode,
       ctx.newStateAccess(),
-      newIntTypeNode(nkIntLit, 0, intTyp))
+      newIntTypeNode(0, intTyp))
     cond.typ = boolTyp
 
     let negateState = newTree(nkCall,
@@ -1283,19 +1283,117 @@ proc deleteEmptyStates(ctx: var Ctx) =
     else:
       inc i
 
+type
+  PreprocessContext = object
+    finallys: seq[PNode]
+    config: ConfigRef
+    blocks: seq[(PNode, int)]
+  FreshVarsContext = object
+    tab: Table[int, PSym]
+    config: ConfigRef
+    info: TLineInfo
+
+proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
+  case n.kind
+  of nkSym:
+    let x = c.tab.getOrDefault(n.sym.id)
+    if x == nil:
+      result = n
+    else:
+      result = newSymNode(x, n.info)
+  of nkSkip - {nkSym}:
+    result = n
+  of nkLetSection, nkVarSection:
+    result = copyNode(n)
+    for it in n:
+      if it.kind in {nkIdentDefs, nkVarTuple}:
+        let idefs = copyNode(it)
+        for v in 0..it.len-3:
+          if it[v].kind == nkSym:
+            let x = copySym(it[v].sym)
+            c.tab[it[v].sym.id] = x
+            idefs.add newSymNode(x)
+          else:
+            idefs.add it[v]
+
+        for rest in it.len-2 ..< it.len: idefs.add it[rest]
+        result.add idefs
+      else:
+        result.add it
+  of nkRaiseStmt:
+    localError(c.config, c.info, "unsupported control flow: 'finally: ... raise' duplicated because of 'break'")
+  else:
+    result = n
+    for i in 0..<n.safeLen:
+      result[i] = freshVars(n[i], c)
+
+proc preprocess(c: var PreprocessContext; n: PNode): PNode =
+  # in order to fix bug #15243 without risking regressions, we preprocess
+  # the AST so that 'break' statements inside a 'try finally' also have the
+  # finally section. We need to duplicate local variables here and also
+  # detect: 'finally: raises X' which is currently not supported. We produce
+  # an error for this case for now. All this will be done properly with Yuriy's
+  # patch.
+  result = n
+  case n.kind
+  of nkTryStmt:
+    let f = n.lastSon
+    if f.kind == nkFinally:
+      c.finallys.add f.lastSon
+
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
+    if f.kind == nkFinally:
+      discard c.finallys.pop()
+
+  of nkWhileStmt, nkBlockStmt:
+    c.blocks.add((n, c.finallys.len))
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+    discard c.blocks.pop()
+
+  of nkBreakStmt:
+    if c.blocks.len == 0:
+      discard
+    else:
+      var fin = -1
+      if n[0].kind == nkEmpty:
+        fin = c.blocks[^1][1]
+      elif n[0].kind == nkSym:
+        for i in countdown(c.blocks.high, 0):
+          if c.blocks[i][0].kind == nkBlockStmt and c.blocks[i][0][0].kind == nkSym and
+              c.blocks[i][0][0].sym == n[0].sym:
+            fin = c.blocks[i][1]
+            break
+
+      if fin >= 0:
+        result = newNodeI(nkStmtList, n.info)
+        for i in countdown(c.finallys.high, fin):
+          var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info)
+          result.add freshVars(preprocess(c, c.finallys[i]), vars)
+        result.add n
+  of nkSkip: discard
+  else:
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
 proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   var ctx: Ctx
   ctx.g = g
   ctx.fn = fn
 
   if getEnvParam(fn).isNil:
-    # Lambda lifting was not done yet. Use temporary :state sym, which
+    # Lambda lifting was not done yet. Use temporary :state sym, which will
     # be handled specially by lambda lifting. Local temp vars (if needed)
-    # should folllow the same logic.
+    # should follow the same logic.
     ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), fn, fn.info)
     ctx.stateVarSym.typ = g.createClosureIterStateType(fn)
   ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), fn, fn.info)
-  var n = n.toStmtList
+  var pc = PreprocessContext(finallys: @[], config: g.config)
+  var n = preprocess(pc, n.toStmtList)
+  #echo "transformed into ", n
+  #var n = n.toStmtList
 
   discard ctx.newState(n, nil)
   let gotoOut = newTree(nkGotoState, g.newIntLit(n.info, -1))
@@ -1312,7 +1410,7 @@ proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   # Optimize empty states away
   ctx.deleteEmptyStates()
 
-  # Make new body by concating the list of states
+  # Make new body by concatenating the list of states
   result = newNodeI(nkStmtList, n.info)
   for s in ctx.states:
     assert(s.len == 2)

compiler/cmdlinehelper.nim

@@ -10,9 +10,11 @@
 ## Helpers for binaries that use compiler passes, eg: nim, nimsuggest, nimfix
 
 import
-  options, idents, nimconf, scriptconfig, extccomp, commands, msgs,
+  options, idents, nimconf, extccomp, commands, msgs,
   lineinfos, modulegraphs, condsyms, os, pathutils
 
+from strutils import normalize
+
 type
   NimProg* = ref object
     suggestMode*: bool
@@ -27,10 +29,7 @@ proc initDefinesProg*(self: NimProg, conf: ConfigRef, name: string) =
 proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   self.processCmdLine(passCmd1, "", conf)
   if self.supportsStdinFile and conf.projectName == "-":
-    conf.projectName = "stdinfile"
-    conf.projectFull = AbsoluteFile "stdinfile"
-    conf.projectPath = AbsoluteDir getCurrentDir()
-    conf.projectIsStdin = true
+    handleStdinInput(conf)
   elif conf.projectName != "":
     try:
       conf.projectFull = canonicalizePath(conf, AbsoluteFile conf.projectName)
@@ -44,45 +43,31 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
     conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir())
 
 proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef): bool =
-  loadConfigs(DefaultConfig, cache, conf) # load all config files
   if self.suggestMode:
     conf.command = "nimsuggest"
+  loadConfigs(DefaultConfig, cache, conf) # load all config files
 
-  template runNimScriptIfExists(path: AbsoluteFile) =
-    let p = path # eval once
-    if fileExists(p):
-      runNimScript(cache, p, freshDefines = false, conf)
-
-  # Caution: make sure this stays in sync with `loadConfigs`
-  if optSkipSystemConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(getSystemConfigPath(conf, DefaultConfigNims))
-
-  if optSkipUserConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(getUserConfigPath(DefaultConfigNims))
-
-  if optSkipParentConfigFiles notin conf.globalOptions:
-    for dir in parentDirs(conf.projectPath.string, fromRoot = true, inclusive = false):
-      runNimScriptIfExists(AbsoluteDir(dir) / DefaultConfigNims)
-
-  if optSkipProjConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(conf.projectPath / DefaultConfigNims)
   block:
     let scriptFile = conf.projectFull.changeFileExt("nims")
     if not self.suggestMode:
-      runNimScriptIfExists(scriptFile)
       # 'nim foo.nims' means to just run the NimScript file and do nothing more:
       if fileExists(scriptFile) and scriptFile == conf.projectFull:
-        return false
-    else:
-      if scriptFile != conf.projectFull:
-        runNimScriptIfExists(scriptFile)
-      else:
-        # 'nimsuggest foo.nims' means to just auto-complete the NimScript file
-        discard
+        if conf.command == "":
+          conf.command = "e"
+          return false
+        elif conf.command.normalize == "e":
+          return false
 
   # now process command line arguments again, because some options in the
-  # command line can overwite the config file's settings
+  # command line can overwrite the config file's settings
   extccomp.initVars(conf)
+  # XXX This is hacky. We need to find a better way.
+  case conf.command
+  of "cpp", "compiletocpp":
+    conf.cmd = cmdCompileToCpp
+  else:
+    discard
+
   self.processCmdLine(passCmd2, "", conf)
   if conf.command == "":
     rawMessage(conf, errGenerated, "command missing")

compiler/commands.nim

@@ -17,7 +17,9 @@ template bootSwitch(name, expr, userString) =
   const name = if expr: " " & userString else: ""
 
 bootSwitch(usedRelease, defined(release), "-d:release")
-bootSwitch(usedGnuReadline, defined(useLinenoise), "-d:useLinenoise")
+bootSwitch(usedDanger, defined(danger), "-d:danger")
+# `useLinenoise` deprecated in favor of `nimUseLinenoise`, kept for backward compatibility
+bootSwitch(useLinenoise, defined(nimUseLinenoise) or defined(useLinenoise), "-d:nimUseLinenoise")
 bootSwitch(usedBoehm, defined(boehmgc), "--gc:boehm")
 bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
 bootSwitch(usedGenerational, defined(gcgenerational), "--gc:generational")
@@ -26,9 +28,12 @@ bootSwitch(usedNoGC, defined(nogc), "--gc:none")
 
 import
   os, msgs, options, nversion, condsyms, strutils, extccomp, platform,
-  wordrecg, parseutils, nimblecmd, idents, parseopt, sequtils, lineinfos,
+  wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos,
   pathutils, strtabs
 
+from incremental import nimIncremental
+from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect
+
 # but some have deps to imported modules. Yay.
 bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
 bootSwitch(usedNativeStacktrace,
@@ -47,15 +52,16 @@ const
       "Compiled at $4\n" &
       "Copyright (c) 2006-" & copyrightYear & " by Andreas Rumpf\n"
 
+proc genFeatureDesc[T: enum](t: typedesc[T]): string {.compileTime.} =
+  var x = ""
+  for f in low(T)..high(T):
+    if x.len > 0: x.add "|"
+    x.add $f
+  x
+
 const
   Usage = slurp"../doc/basicopt.txt".replace(" //", " ")
-  FeatureDesc = block:
-    var x = ""
-    for f in low(Feature)..high(Feature):
-      if x.len > 0: x.add "|"
-      x.add $f
-    x
-  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", " ") % FeatureDesc
+  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", " ") % [genFeatureDesc(Feature), genFeatureDesc(LegacyFeature)]
 
 proc getCommandLineDesc(conf: ConfigRef): string =
   result = (HelpMessage % [VersionAsString, platform.OS[conf.target.hostOS].name,
@@ -96,8 +102,8 @@ proc writeVersionInfo(conf: ConfigRef; pass: TCmdLinePass) =
     when gitHash.len == 40:
       msgWriteln(conf, "git hash: " & gitHash, {msgStdout})
 
-    msgWriteln(conf, "active boot switches:" & usedRelease &
-      usedTinyC & usedGnuReadline & usedNativeStacktrace &
+    msgWriteln(conf, "active boot switches:" & usedRelease & usedDanger &
+      usedTinyC & useLinenoise & usedNativeStacktrace &
       usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC,
                {msgStdout})
     msgQuit(0)
@@ -106,7 +112,7 @@ proc writeCommandLineUsage*(conf: ConfigRef) =
   msgWriteln(conf, getCommandLineDesc(conf), {msgStdout})
 
 proc addPrefix(switch: string): string =
-  if len(switch) <= 1: result = "-" & switch
+  if switch.len <= 1: result = "-" & switch
   else: result = "--" & switch
 
 const
@@ -123,14 +129,14 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
                  info: TLineInfo) =
   cmd = ""
   var i = 0
-  if i < len(switch) and switch[i] == '-': inc(i)
-  if i < len(switch) and switch[i] == '-': inc(i)
-  while i < len(switch):
+  if i < switch.len and switch[i] == '-': inc(i)
+  if i < switch.len and switch[i] == '-': inc(i)
+  while i < switch.len:
     case switch[i]
-    of 'a'..'z', 'A'..'Z', '0'..'9', '_', '.': add(cmd, switch[i])
+    of 'a'..'z', 'A'..'Z', '0'..'9', '_', '.': cmd.add(switch[i])
     else: break
     inc(i)
-  if i >= len(switch): arg = ""
+  if i >= switch.len: arg = ""
   # cmd:arg => (cmd,arg)
   elif switch[i] in {':', '='}: arg = substr(switch, i + 1)
   # cmd[sub]:rest => (cmd,[sub]:rest)
@@ -174,17 +180,17 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
   var i = 0
   var n = hintMin
   var isBracket = false
-  if i < len(arg) and arg[i] == '[':
+  if i < arg.len and arg[i] == '[':
     isBracket = true
     inc(i)
-  while i < len(arg) and (arg[i] notin {':', '=', ']'}):
-    add(id, arg[i])
+  while i < arg.len and (arg[i] notin {':', '=', ']'}):
+    id.add(arg[i])
     inc(i)
   if isBracket:
-    if i < len(arg) and arg[i] == ']': inc(i)
+    if i < arg.len and arg[i] == ']': inc(i)
     else: invalidCmdLineOption(conf, pass, orig, info)
 
-  if i < len(arg) and (arg[i] in {':', '='}): inc(i)
+  if i < arg.len and (arg[i] in {':', '='}): inc(i)
   else: invalidCmdLineOption(conf, pass, orig, info)
   if state == wHint:
     let x = findStr(lineinfos.HintsToStr, id)
@@ -194,17 +200,21 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
     let x = findStr(lineinfos.WarningsToStr, id)
     if x >= 0: n = TNoteKind(x + ord(warnMin))
     else: localError(conf, info, "unknown warning: " & id)
-  case substr(arg, i).normalize
-  of "on":
-    incl(conf.notes, n)
-    incl(conf.mainPackageNotes, n)
-    incl(conf.enableNotes, n)
-  of "off":
-    excl(conf.notes, n)
-    excl(conf.mainPackageNotes, n)
-    incl(conf.disableNotes, n)
-    excl(conf.foreignPackageNotes, n)
-  else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
+
+  let val = substr(arg, i).normalize
+  if val notin ["on", "off"]:
+    localError(conf, info, errOnOrOffExpectedButXFound % arg)
+  elif n notin conf.cmdlineNotes or pass == passCmd1:
+    if pass == passCmd1: incl(conf.cmdlineNotes, n)
+    incl(conf.modifiedyNotes, n)
+    case val
+    of "on":
+      incl(conf.notes, n)
+      incl(conf.mainPackageNotes, n)
+    of "off":
+      excl(conf.notes, n)
+      excl(conf.mainPackageNotes, n)
+      excl(conf.foreignPackageNotes, n)
 
 proc processCompile(conf: ConfigRef; filename: string) =
   var found = findFile(conf, filename)
@@ -215,19 +225,22 @@ const
   errNoneBoehmRefcExpectedButXFound = "'none', 'boehm' or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found"
+  errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found"
 
 proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo): bool =
   case switch.normalize
   of "gc":
     case arg.normalize
-    of "boehm":        result = conf.selectedGC == gcBoehm
-    of "refc":         result = conf.selectedGC == gcRefc
-    of "v2":           result = false
+    of "boehm": result = conf.selectedGC == gcBoehm
+    of "refc": result = conf.selectedGC == gcRefc
+    of "v2": result = false
     of "markandsweep": result = conf.selectedGC == gcMarkAndSweep
     of "generational": result = false
-    of "destructors":  result = conf.selectedGC == gcDestructors
-    of "go":           result = conf.selectedGC == gcGo
-    of "none":         result = conf.selectedGC == gcNone
+    of "destructors", "arc": result = conf.selectedGC == gcArc
+    of "orc": result = conf.selectedGC == gcOrc
+    of "hooks": result = conf.selectedGC == gcHooks
+    of "go": result = conf.selectedGC == gcGo
+    of "none": result = conf.selectedGC == gcNone
     of "stack", "regions": result = conf.selectedGC == gcRegions
     else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "opt":
@@ -239,15 +252,22 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
   of "verbosity": result = $conf.verbosity == arg
   of "app":
     case arg.normalize
-    of "gui":       result = contains(conf.globalOptions, optGenGuiApp)
-    of "console":   result = not contains(conf.globalOptions, optGenGuiApp)
-    of "lib":       result = contains(conf.globalOptions, optGenDynLib) and
+    of "gui": result = contains(conf.globalOptions, optGenGuiApp)
+    of "console": result = not contains(conf.globalOptions, optGenGuiApp)
+    of "lib": result = contains(conf.globalOptions, optGenDynLib) and
                       not contains(conf.globalOptions, optGenGuiApp)
     of "staticlib": result = contains(conf.globalOptions, optGenStaticLib) and
                       not contains(conf.globalOptions, optGenGuiApp)
     else: localError(conf, info, errGuiConsoleOrLibExpectedButXFound % arg)
   of "dynliboverride":
     result = isDynlibOverride(conf, arg)
+  of "exceptions":
+    case arg.normalize
+    of "cpp": result = conf.exc == excCpp
+    of "setjmp": result = conf.exc == excSetjmp
+    of "quirky": result = conf.exc == excQuirky
+    of "goto": result = conf.exc == excGoto
+    else: localError(conf, info, errInvalidExceptionSystem % arg)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool =
@@ -261,8 +281,9 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "hints": result = contains(conf.options, optHints)
   of "threadanalysis": result = contains(conf.globalOptions, optThreadAnalysis)
   of "stacktrace": result = contains(conf.options, optStackTrace)
+  of "stacktracemsgs": result = contains(conf.options, optStackTraceMsgs)
   of "linetrace": result = contains(conf.options, optLineTrace)
-  of "debugger": result = contains(conf.options, optEndb)
+  of "debugger": result = contains(conf.globalOptions, optCDebug)
   of "profiler": result = contains(conf.options, optProfiler)
   of "memtracker": result = contains(conf.options, optMemTracker)
   of "checks", "x": result = conf.options * ChecksOptions == ChecksOptions
@@ -275,8 +296,10 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "fieldchecks": result = contains(conf.options, optFieldCheck)
   of "rangechecks": result = contains(conf.options, optRangeCheck)
   of "boundchecks": result = contains(conf.options, optBoundsCheck)
+  of "refchecks": result = contains(conf.options, optRefCheck)
   of "overflowchecks": result = contains(conf.options, optOverflowCheck)
-  of "movechecks": result = contains(conf.options, optMoveCheck)
+  of "staticboundchecks": result = contains(conf.options, optStaticBoundsCheck)
+  of "stylechecks": result = contains(conf.options, optStyleCheck)
   of "linedir": result = contains(conf.options, optLineDir)
   of "assertions", "a": result = contains(conf.options, optAssert)
   of "run", "r": result = contains(conf.globalOptions, optRun)
@@ -354,6 +377,14 @@ proc dynlibOverride(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, in
     expectArg(conf, switch, arg, pass, info)
     options.inclDynlibOverride(conf, arg)
 
+proc handleStdinInput*(conf: ConfigRef) =
+  conf.projectName = "stdinfile"
+  conf.projectFull = conf.projectName.AbsoluteFile
+  conf.projectPath = AbsoluteDir getCurrentDir()
+  conf.projectIsStdin = true
+  if conf.outDir.isEmpty:
+    conf.outDir = getNimcacheDir(conf)
+
 proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
                     conf: ConfigRef) =
   var
@@ -361,8 +392,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   case switch.normalize
   of "path", "p":
     expectArg(conf, switch, arg, pass, info)
-    addPath(conf, if pass == passPP: processCfgPath(conf, arg, info)
-                  else: processPath(conf, arg, info), info)
+    for path in nimbleSubs(conf, arg):
+      addPath(conf, if pass == passPP: processCfgPath(conf, path, info)
+                    else: processPath(conf, path, info), info)
   of "nimblepath", "babelpath":
     # keep the old name for compat
     if pass in {passCmd2, passPP} and optNoNimblePath notin conf.globalOptions:
@@ -375,6 +407,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "nonimblepath", "nobabelpath":
     expectNoArg(conf, switch, arg, pass, info)
     disableNimblePath(conf)
+  of "clearnimblepath":
+    expectNoArg(conf, switch, arg, pass, info)
+    clearNimblePath(conf)
   of "excludepath":
     expectArg(conf, switch, arg, pass, info)
     let path = processPath(conf, arg, info)
@@ -382,26 +417,32 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     conf.lazyPaths.keepItIf(it != path)
   of "nimcache":
     expectArg(conf, switch, arg, pass, info)
-    conf.nimcacheDir = processPath(conf, arg, info, true)
+    conf.nimcacheDir = processPath(conf, arg, info, notRelativeToProj=true)
   of "out", "o":
     expectArg(conf, switch, arg, pass, info)
-    let f = splitFile(arg.expandTilde)
+    let f = splitFile(processPath(conf, arg, info, notRelativeToProj=true).string)
     conf.outFile = RelativeFile f.name & f.ext
     conf.outDir = toAbsoluteDir f.dir
   of "outdir":
     expectArg(conf, switch, arg, pass, info)
-    conf.outDir = toAbsoluteDir arg.expandTilde
+    conf.outDir = processPath(conf, arg, info, notRelativeToProj=true)
   of "docseesrcurl":
     expectArg(conf, switch, arg, pass, info)
     conf.docSeeSrcUrl = arg
+  of "docroot":
+    conf.docRoot = if arg.len == 0: "@default" else: arg
   of "mainmodule", "m":
     discard "allow for backwards compatibility, but don't do anything"
   of "define", "d":
     expectArg(conf, switch, arg, pass, info)
     if {':', '='} in arg:
       splitSwitch(conf, arg, key, val, pass, info)
+      if cmpIgnoreStyle(key, "nimQuirky") == 0:
+        conf.exc = excQuirky
       defineSymbol(conf.symbols, key, val)
     else:
+      if cmpIgnoreStyle(arg, "nimQuirky") == 0:
+        conf.exc = excQuirky
       defineSymbol(conf.symbols, arg)
   of "undef", "u":
     expectArg(conf, switch, arg, pass, info)
@@ -432,30 +473,58 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
   of "gc":
     expectArg(conf, switch, arg, pass, info)
-    case arg.normalize
-    of "boehm":
-      conf.selectedGC = gcBoehm
-      defineSymbol(conf.symbols, "boehmgc")
-    of "refc":
-      conf.selectedGC = gcRefc
-    of "v2":
-      message(conf, info, warnDeprecated, "--gc:v2 is deprecated; using default gc")
-    of "markandsweep":
-      conf.selectedGC = gcMarkAndSweep
-      defineSymbol(conf.symbols, "gcmarkandsweep")
-    of "destructors":
-      conf.selectedGC = gcDestructors
-      defineSymbol(conf.symbols, "gcdestructors")
-    of "go":
-      conf.selectedGC = gcGo
-      defineSymbol(conf.symbols, "gogc")
-    of "none":
-      conf.selectedGC = gcNone
-      defineSymbol(conf.symbols, "nogc")
-    of "stack", "regions":
-      conf.selectedGC= gcRegions
-      defineSymbol(conf.symbols, "gcregions")
-    else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
+    if pass in {passCmd2, passPP}:
+      case arg.normalize
+      of "boehm":
+        conf.selectedGC = gcBoehm
+        defineSymbol(conf.symbols, "boehmgc")
+        incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS
+      of "refc":
+        conf.selectedGC = gcRefc
+      of "v2":
+        message(conf, info, warnDeprecated, "--gc:v2 is deprecated; using default gc")
+      of "markandsweep":
+        conf.selectedGC = gcMarkAndSweep
+        defineSymbol(conf.symbols, "gcmarkandsweep")
+      of "destructors", "arc":
+        conf.selectedGC = gcArc
+        if conf.cmd != cmdCompileToCpp:
+          conf.exc = excGoto
+        defineSymbol(conf.symbols, "gcdestructors")
+        defineSymbol(conf.symbols, "gcarc")
+        incl conf.globalOptions, optSeqDestructors
+        incl conf.globalOptions, optTinyRtti
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+          defineSymbol(conf.symbols, "nimV2")
+      of "orc":
+        conf.selectedGC = gcOrc
+        if conf.cmd != cmdCompileToCpp:
+          conf.exc = excGoto
+        defineSymbol(conf.symbols, "gcdestructors")
+        defineSymbol(conf.symbols, "gcorc")
+        incl conf.globalOptions, optSeqDestructors
+        incl conf.globalOptions, optTinyRtti
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+          defineSymbol(conf.symbols, "nimV2")
+      of "hooks":
+        conf.selectedGC = gcHooks
+        defineSymbol(conf.symbols, "gchooks")
+        incl conf.globalOptions, optSeqDestructors
+        processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+      of "go":
+        conf.selectedGC = gcGo
+        defineSymbol(conf.symbols, "gogc")
+      of "none":
+        conf.selectedGC = gcNone
+        defineSymbol(conf.symbols, "nogc")
+      of "stack", "regions":
+        conf.selectedGC = gcRegions
+        defineSymbol(conf.symbols, "gcregions")
+      else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "warnings", "w":
     if processOnOffSwitchOrList(conf, {optWarns}, arg, pass, info): listWarnings(conf)
   of "warning": processSpecificNote(arg, wWarning, pass, info, switch, conf)
@@ -464,28 +533,23 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if processOnOffSwitchOrList(conf, {optHints}, arg, pass, info): listHints(conf)
   of "threadanalysis": processOnOffSwitchG(conf, {optThreadAnalysis}, arg, pass, info)
   of "stacktrace": processOnOffSwitch(conf, {optStackTrace}, arg, pass, info)
+  of "stacktracemsgs": processOnOffSwitch(conf, {optStackTraceMsgs}, arg, pass, info)
   of "excessivestacktrace": processOnOffSwitchG(conf, {optExcessiveStackTrace}, arg, pass, info)
   of "linetrace": processOnOffSwitch(conf, {optLineTrace}, arg, pass, info)
   of "debugger":
     case arg.normalize
-    of "on", "endb":
-      conf.options.incl optEndb
-      defineSymbol(conf.symbols, "endb")
-    of "off":
-      conf.options.excl optEndb
-      undefSymbol(conf.symbols, "endb")
-    of "native", "gdb":
-      incl(conf.globalOptions, optCDebug)
-      conf.options = conf.options + {optLineDir} - {optEndb}
+    of "on", "native", "gdb":
+      conf.globalOptions.incl optCDebug
+      conf.options.incl optLineDir
       #defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
-      undefSymbol(conf.symbols, "endb")
+    of "off":
+      conf.globalOptions.excl optCDebug
     else:
-      localError(conf, info, "expected endb|gdb but found " & arg)
+      localError(conf, info, "expected native|gdb|on|off but found " & arg)
   of "g": # alias for --debugger:native
-    incl(conf.globalOptions, optCDebug)
-    conf.options = conf.options + {optLineDir} - {optEndb}
+    conf.globalOptions.incl optCDebug
+    conf.options.incl optLineDir
     #defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
-    undefSymbol(conf.symbols, "endb")
   of "profiler":
     processOnOffSwitch(conf, {optProfiler}, arg, pass, info)
     if optProfiler in conf.options: defineSymbol(conf.symbols, "profiler")
@@ -508,7 +572,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       undefSymbol(conf.symbols, "useNimRtl")
   of "oldnewlines":
     case arg.normalize
-    of "","on":
+    of "", "on":
       conf.oldNewlines = true
       defineSymbol(conf.symbols, "nimOldNewlines")
     of "off":
@@ -529,14 +593,16 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "fieldchecks": processOnOffSwitch(conf, {optFieldCheck}, arg, pass, info)
   of "rangechecks": processOnOffSwitch(conf, {optRangeCheck}, arg, pass, info)
   of "boundchecks": processOnOffSwitch(conf, {optBoundsCheck}, arg, pass, info)
+  of "refchecks": processOnOffSwitch(conf, {optRefCheck}, arg, pass, info)
   of "overflowchecks": processOnOffSwitch(conf, {optOverflowCheck}, arg, pass, info)
-  of "movechecks": processOnOffSwitch(conf, {optMoveCheck}, arg, pass, info)
+  of "staticboundchecks": processOnOffSwitch(conf, {optStaticBoundsCheck}, arg, pass, info)
+  of "stylechecks": processOnOffSwitch(conf, {optStyleCheck}, arg, pass, info)
   of "linedir": processOnOffSwitch(conf, {optLineDir}, arg, pass, info)
   of "assertions", "a": processOnOffSwitch(conf, {optAssert}, arg, pass, info)
   of "deadcodeelim": discard # deprecated, dead code elim always on
   of "threads":
     processOnOffSwitchG(conf, {optThreads}, arg, pass, info)
-    #if optThreads in conf.globalOptions: incl(conf.notes, warnGcUnsafe)
+    #if optThreads in conf.globalOptions: conf.setNote(warnGcUnsafe)
   of "tlsemulation": processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info)
   of "taintmode": processOnOffSwitchG(conf, {optTaintMode}, arg, pass, info)
   of "implicitstatic":
@@ -609,28 +675,31 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       conf.implicitIncludes.add findModule(conf, arg, toFullPath(conf, info)).string
   of "listcmd":
     processOnOffSwitchG(conf, {optListCmd}, arg, pass, info)
+  of "asm":
+    processOnOffSwitchG(conf, {optProduceAsm}, arg, pass, info)
   of "genmapping":
     processOnOffSwitchG(conf, {optGenMapping}, arg, pass, info)
   of "os":
     expectArg(conf, switch, arg, pass, info)
-    if pass in {passCmd1, passPP}:
-      let theOS = platform.nameToOS(arg)
-      if theOS == osNone:
-        let osList = platform.listOSnames().join(", ")
-        localError(conf, info, "unknown OS: '$1'. Available options are: $2" % [arg, $osList])
-      elif theOS != conf.target.hostOS:
-        setTarget(conf.target, theOS, conf.target.targetCPU)
+    let theOS = platform.nameToOS(arg)
+    if theOS == osNone:
+      let osList = platform.listOSnames().join(", ")
+      localError(conf, info, "unknown OS: '$1'. Available options are: $2" % [arg, $osList])
+    else:
+      setTarget(conf.target, theOS, conf.target.targetCPU)
   of "cpu":
     expectArg(conf, switch, arg, pass, info)
-    if pass in {passCmd1, passPP}:
-      let cpu = platform.nameToCPU(arg)
-      if cpu == cpuNone:
-        let cpuList = platform.listCPUnames().join(", ")
-        localError(conf, info, "unknown CPU: '$1'. Available options are: $2" % [ arg, cpuList])
-      elif cpu != conf.target.hostCPU:
-        setTarget(conf.target, conf.target.targetOS, cpu)
+    let cpu = platform.nameToCPU(arg)
+    if cpu == cpuNone:
+      let cpuList = platform.listCPUnames().join(", ")
+      localError(conf, info, "unknown CPU: '$1'. Available options are: $2" % [ arg, cpuList])
+    else:
+      setTarget(conf.target, conf.target.targetOS, cpu)
   of "run", "r":
     processOnOffSwitchG(conf, {optRun}, arg, pass, info)
+  of "maxloopiterationsvm":
+    expectArg(conf, switch, arg, pass, info)
+    conf.maxLoopIterationsVM = parseInt(arg)
   of "errormax":
     expectArg(conf, switch, arg, pass, info)
     # Note: `nim check` (etc) can overwrite this.
@@ -645,9 +714,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if verbosity notin {0..3}:
       localError(conf, info, "invalid verbosity level: '$1'" % arg)
     conf.verbosity = verbosity
-    conf.notes = NotesVerbosity[conf.verbosity]
-    incl(conf.notes, conf.enableNotes)
-    excl(conf.notes, conf.disableNotes)
+    var verb = NotesVerbosity[conf.verbosity]
+    ## We override the default `verb` by explicitly modified (set/unset) notes.
+    conf.notes = (conf.modifiedyNotes * conf.notes + verb) -
+      (conf.modifiedyNotes * verb - conf.notes)
     conf.mainPackageNotes = conf.notes
   of "parallelbuild":
     expectArg(conf, switch, arg, pass, info)
@@ -666,7 +736,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     helpOnError(conf, pass)
   of "symbolfiles": discard "ignore for backwards compat"
   of "incremental":
-    when not defined(nimIncremental):
+    when not nimIncremental:
       localError(conf, info, "the compiler was not built with " &
         "incremental compilation features; bootstrap with " &
         "-d:nimIncremental to enable")
@@ -739,10 +809,22 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         conf.features.incl parseEnum[Feature](arg)
       except ValueError:
         localError(conf, info, "unknown experimental feature")
+  of "legacy":
+    try:
+      conf.legacyFeatures.incl parseEnum[LegacyFeature](arg)
+    except ValueError:
+      localError(conf, info, "unknown obsolete feature")
   of "nocppexceptions":
     expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optNoCppExceptions)
+    conf.exc = low(ExceptionSystem)
     defineSymbol(conf.symbols, "noCppExceptions")
+  of "exceptions":
+    case arg.normalize
+    of "cpp": conf.exc = excCpp
+    of "setjmp": conf.exc = excSetjmp
+    of "quirky": conf.exc = excQuirky
+    of "goto": conf.exc = excGoto
+    else: localError(conf, info, errInvalidExceptionSystem % arg)
   of "cppdefine":
     expectArg(conf, switch, arg, pass, info)
     if conf != nil:
@@ -752,14 +834,22 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if pass in {passCmd2, passPP}:
       doAssert(conf != nil)
       incl(conf.features, destructor)
-      incl(conf.globalOptions, optNimV2)
+      incl(conf.globalOptions, optTinyRtti)
+      incl(conf.globalOptions, optOwnedRefs)
+      incl(conf.globalOptions, optSeqDestructors)
       defineSymbol(conf.symbols, "nimV2")
-      conf.selectedGC = gcDestructors
-      defineSymbol(conf.symbols, "gcdestructors")
+      conf.selectedGC = gcHooks
+      defineSymbol(conf.symbols, "gchooks")
+      defineSymbol(conf.symbols, "nimSeqsV2")
+      defineSymbol(conf.symbols, "nimOwnedEnabled")
+  of "seqsv2":
+    processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
+    if pass in {passCmd2, passPP}:
+      defineSymbol(conf.symbols, "nimSeqsV2")
   of "stylecheck":
     case arg.normalize
     of "off": conf.globalOptions = conf.globalOptions - {optStyleHint, optStyleError}
-    of "hint": conf.globalOptions = conf.globalOptions + {optStyleHint}
+    of "hint": conf.globalOptions = conf.globalOptions + {optStyleHint} - {optStyleError}
     of "error": conf.globalOptions = conf.globalOptions + {optStyleError}
     else: localError(conf, info, errOffHintsError % arg)
   of "showallmismatches":
@@ -777,13 +867,41 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "expandmacro":
     expectArg(conf, switch, arg, pass, info)
     conf.macrosToExpand[arg] = "T"
-  of "":
-    conf.projectName = "-"
+  of "oldgensym":
+    processOnOffSwitchG(conf, {optNimV019}, arg, pass, info)
+  of "useversion":
+    expectArg(conf, switch, arg, pass, info)
+    case arg
+    of "1.0":
+      defineSymbol(conf.symbols, "NimMajor", "1")
+      defineSymbol(conf.symbols, "NimMinor", "0")
+      # always be compatible with 1.0.100:
+      defineSymbol(conf.symbols, "NimPatch", "100")
+      # old behaviors go here:
+      defineSymbol(conf.symbols, "nimOldRelativePathBehavior")
+      ast.eqTypeFlags.excl {tfGcSafe, tfNoSideEffect}
+      conf.globalOptions.incl optNimV1Emulation
+    else:
+      localError(conf, info, "unknown Nim version; currently supported values are: {1.0}")
+  of "benchmarkvm":
+    processOnOffSwitchG(conf, {optBenchmarkVM}, arg, pass, info)
+  of "sinkinference":
+    processOnOffSwitch(conf, {optSinkInference}, arg, pass, info)
+  of "panics":
+    processOnOffSwitchG(conf, {optPanics}, arg, pass, info)
+    if optPanics in conf.globalOptions:
+      defineSymbol(conf.symbols, "nimPanics")
+  of "sourcemap":
+    conf.globalOptions.incl optSourcemap
+    conf.options.incl optLineDir
+    # processOnOffSwitchG(conf, {optSourcemap, opt}, arg, pass, info)
+  of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -)
+    handleStdinInput(conf)
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)
 
-template gCmdLineInfo*(): untyped = newLineInfo(config, AbsoluteFile"command line", 1, 1)
+template gCmdLineInfo*(): untyped = newLineInfo(commandLineIdx, 1, 1)
 
 proc processCommand*(switch: string, pass: TCmdLinePass; config: ConfigRef) =
   var cmd, arg: string
@@ -806,7 +924,7 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser;
                       argsCount: var int; config: ConfigRef): bool =
   if argsCount == 0:
     # nim filename.nims  is the same as "nim e filename.nims":
-    if p.key.endswith(".nims"):
+    if p.key.endsWith(".nims"):
       config.command = "e"
       incl(config.globalOptions, optWasNimscript)
       config.projectName = unixToNativePath(p.key)

compiler/condsyms.nim

@@ -10,7 +10,7 @@
 # This module handles the conditional symbols.
 
 import
-  strtabs, platform, strutils, idents
+  strtabs
 
 from options import Feature
 from lineinfos import HintsToStr, WarningsToStr
@@ -48,13 +48,11 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimalias")
   defineSymbol("nimlocks")
   defineSymbol("nimnode")
-  defineSymbol("nimnomagic64")
   defineSymbol("nimvarargstyped")
   defineSymbol("nimtypedescfixed")
   defineSymbol("nimKnowsNimvm")
   defineSymbol("nimArrIdx")
   defineSymbol("nimHasalignOf")
-  defineSymbol("nimNewShiftOps")
   defineSymbol("nimDistros")
   defineSymbol("nimHasCppDefine")
   defineSymbol("nimGenericInOutFlags")
@@ -86,6 +84,8 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasSignatureHashInMacro")
   defineSymbol("nimHasDefault")
   defineSymbol("nimMacrosSizealignof")
+  defineSymbol("nimNoZeroExtendMagic")
+  defineSymbol("nimMacrosGetNodeId")
   for f in low(Feature)..high(Feature):
     defineSymbol("nimHas" & $f)
 
@@ -93,3 +93,27 @@ proc initDefines*(symbols: StringTableRef) =
     defineSymbol("nimHasWarning" & s)
   for s in HintsToStr:
     defineSymbol("nimHasHint" & s)
+
+  defineSymbol("nimFixedOwned")
+  defineSymbol("nimHasStyleChecks")
+  defineSymbol("nimToOpenArrayCString")
+  defineSymbol("nimHasUsed")
+  defineSymbol("nimFixedForwardGeneric")
+  defineSymbol("nimnomagic64")
+  defineSymbol("nimNewShiftOps")
+  defineSymbol("nimHasCursor")
+  defineSymbol("nimHasExceptionsQuery")
+  defineSymbol("nimHasIsNamedTuple")
+  defineSymbol("nimHashOrdinalFixed")
+
+  when defined(nimHasLibFFI):
+    # Renaming as we can't conflate input vs output define flags; e.g. this
+    # will report the right thing regardless of whether user adds
+    # `-d:nimHasLibFFI` in his user config.
+    defineSymbol("nimHasLibFFIEnabled")
+
+  defineSymbol("nimHasSinkInference")
+  defineSymbol("nimNewIntegerOps")
+  defineSymbol("nimHasInvariant")
+  defineSymbol("nimHasStacktraceMsgs")
+  defineSymbol("nimHasStacktracesModule")

compiler/configuration.nim

@@ -1,6 +0,0 @@
-## Use the module 'lineinfos' instead!
-
-{.deprecated.}
-
-import lineinfos
-export lineinfos

compiler/cursors.nim

@@ -1,72 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2019 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-import
-  intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
-  strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
-  lineinfos, parampatterns
-
-##[
-This module implements "cursor" detection. A cursor is a local variable
-that is used for navigation in a datastructure, it does not "own" the
-data it aliases but it might update the underlying datastructure.
-
-Two primary examples for cursors that I have in mind and that are critical
-for optimization:
-
-1. Local string variable introduced by ``for x in a``::
-
-  var i = 0
-  while i < a.len:
-    let cursor = a[i]
-    use cursor
-    inc i
-
-2. Local ``ref`` variable for navigation::
-
-  var cursor = listHead
-  while cursor != nil:
-    use cursor
-    cursor = cursor.next
-
-Cursors are very interesting for the optimizer because they can be copyMem'ed
-and don't need a destructor.
-
-More formally, a cursor is a variable that is set on all paths to
-a *location* or a proc call that produced a ``lent/var`` type. All statements
-that come after these assignments MUST not mutate what the cursor aliases.
-
-Mutations *through* the cursor are allowed if the cursor has ref semantics.
-
-Look at this complex real world example taken from the compiler itself:
-
-.. code-block:: Nim
-
-  proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope =
-    var t = typ
-    while true:
-      if t.sym != nil and {sfImportc, sfExportc} * t.sym.flags != {}:
-        return t.sym.loc.r
-
-      if t.kind in irrelevantForBackend:
-        t = t.lastSon
-      else:
-        break
-    let typ = if typ.kind in {tyAlias, tySink, tyOwned}: typ.lastSon else: typ
-    if typ.loc.r == nil:
-      typ.loc.r = typ.typeName & $sig
-    result = typ.loc.r
-    if result == nil: internalError(m.config, "getTypeName: " & $typ.kind)
-
-Here `t` is a cursor but without a control flow based analysis we are unlikely
-to detect it.
-
-]##
-
-# Araq: I owe you an implementation. For now use the .cursor pragma. :-/

compiler/depends.nim

@@ -10,8 +10,7 @@
 # This module implements a dependency file generator.
 
 import
-  os, options, ast, astalgo, msgs, ropes, idents, passes, modulepaths,
-  pathutils
+  options, ast, ropes, idents, passes, modulepaths, pathutils
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -26,7 +25,7 @@ type
     dotGraph: Rope
 
 proc addDependencyAux(b: Backend; importing, imported: string) =
-  addf(b.dotGraph, "$1 -> \"$2\";$n", [rope(importing), rope(imported)])
+  b.dotGraph.addf("$1 -> \"$2\";$n", [rope(importing), rope(imported)])
   # s1 -> s2_4[label="[0-9]"];
 
 proc addDotDependency(c: PPassContext, n: PNode): PNode =
@@ -35,14 +34,14 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode =
   let b = Backend(g.graph.backend)
   case n.kind
   of nkImportStmt:
-    for i in 0 ..< sonsLen(n):
-      var imported = getModuleName(g.config, n.sons[i])
+    for i in 0..<n.len:
+      var imported = getModuleName(g.config, n[i])
       addDependencyAux(b, g.module.name.s, imported)
   of nkFromStmt, nkImportExceptStmt:
-    var imported = getModuleName(g.config, n.sons[0])
+    var imported = getModuleName(g.config, n[0])
     addDependencyAux(b, g.module.name.s, imported)
   of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr:
-    for i in 0 ..< sonsLen(n): discard addDotDependency(c, n.sons[i])
+    for i in 0..<n.len: discard addDotDependency(c, n[i])
   else:
     discard
 

compiler/dfa.nim

@@ -29,35 +29,36 @@
 ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
 ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
 
-import ast, astalgo, types, intsets, tables, msgs, options, lineinfos, renderer
+import ast, types, intsets, lineinfos, renderer
 
 from patterns import sameTrees
 
 type
   InstrKind* = enum
-    goto, fork, join, def, use
+    goto, fork, def, use
   Instr* = object
-    n*: PNode
+    n*: PNode # contains the def/use location.
     case kind*: InstrKind
-    of def, use: sym*: PSym # 'sym' can also be 'nil' and
-                            # then 'n' contains the def/use location.
-                            # This is used so that we can track object
-                            # and tuple field accesses precisely.
-    of goto, fork, join: dest*: int
+    of goto, fork: dest*: int
+    else: discard
 
   ControlFlowGraph* = seq[Instr]
 
   TPosition = distinct int
+
   TBlock = object
-    label: PSym
-    fixups: seq[TPosition]
+    case isTryBlock: bool
+    of false:
+      label: PSym
+      breakFixups: seq[(TPosition, seq[PNode])] #Contains the gotos for the breaks along with their pending finales
+    of true:
+      finale: PNode
+      raiseFixups: seq[TPosition] #Contains the gotos for the raises
 
   Con = object
     code: ControlFlowGraph
     inCall, inTryStmt: int
     blocks: seq[TBlock]
-    tryStmtFixups: seq[TPosition]
-    forks: seq[TPosition]
     owner: PSym
 
 proc debugInfo(info: TLineInfo): string =
@@ -69,7 +70,7 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
   var jumpTargets = initIntSet()
   let last = if last < 0: c.len-1 else: min(last, c.len-1)
   for i in start..last:
-    if c[i].kind in {goto, fork, join}:
+    if c[i].kind in {goto, fork}:
       jumpTargets.incl(i+c[i].dest)
   var i = start
   while i <= last:
@@ -80,7 +81,7 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
     case c[i].kind
     of def, use:
       result.add renderTree(c[i].n)
-    of goto, fork, join:
+    of goto, fork:
       result.add "L"
       result.addInt c[i].dest+i
     result.add("\t#")
@@ -94,13 +95,11 @@ proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
   ## echos the ControlFlowGraph for debugging purposes.
   var buf = ""
   codeListing(c, buf, start, last)
-  when declared(echo):
-    echo buf
+  echo buf
 
 proc forkI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: fork, dest: 0)
-  c.forks.add result
 
 proc gotoI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
@@ -108,6 +107,24 @@ proc gotoI(c: var Con; n: PNode): TPosition =
 
 #[
 
+Join is no more
+===============
+Instead of generating join instructions we adapt our traversal of the CFG.
+
+When encountering a fork we split into two paths, we follow the path
+starting at "pc + 1" until it encounters the joinpoint: "pc + forkInstr.dest".
+If we encounter gotos that would jump further than the current joinpoint,
+as can happen with gotos generated by unstructured controlflow such as break, raise or return,
+we simply suspend following the current path, and follow the other path until the new joinpoint
+which is simply the instruction pointer returned to us by the now suspended path.
+If the path we are following now, also encounters a goto that exceeds the joinpoint
+we repeat the process; suspending the current path and evaluating the other one with a new joinpoint.
+If we eventually reach a common joinpoint we join the two paths.
+This new "ping-pong" approach has the obvious advantage of not requiring join instructions, as such
+cutting down on the CFG size but is also mandatory for correctly handling complicated cases
+of unstructured controlflow.
+
+
 Design of join
 ==============
 
@@ -119,10 +136,10 @@ use(x)
 
 Generates:
 
-L0: fork L1
+L0: fork lab1
   join L0  # patched.
   goto Louter
-L1:
+lab1:
   def x
   join L0
 Louter:
@@ -256,33 +273,38 @@ duplicate the 'join' instructions on breaks and return exits!
 
 ]#
 
-proc joinI(c: var Con; fromFork: TPosition; n: PNode) =
-  let dist = fromFork.int - c.code.len
-  c.code.add Instr(n: n, kind: join, dest: dist)
-
 proc genLabel(c: Con): TPosition =
   result = TPosition(c.code.len)
 
 proc jmpBack(c: var Con, n: PNode, p = TPosition(0)) =
   let dist = p.int - c.code.len
-  doAssert(-0x7fff < dist and dist < 0x7fff)
+  doAssert(low(int) div 2 + 1 < dist and dist < high(int) div 2)
   c.code.add Instr(n: n, kind: goto, dest: dist)
 
 proc patch(c: var Con, p: TPosition) =
   # patch with current index
   let p = p.int
   let diff = c.code.len - p
-  doAssert(-0x7fff < diff and diff < 0x7fff)
+  doAssert(low(int) div 2 + 1 < diff and diff < high(int) div 2)
   c.code[p].dest = diff
 
+proc gen(c: var Con; n: PNode) # {.noSideEffect.}
+
 proc popBlock(c: var Con; oldLen: int) =
-  for f in c.blocks[oldLen].fixups:
-    c.patch(f)
+  var exits: seq[TPosition]
+  exits.add c.gotoI(newNode(nkEmpty))
+  for f in c.blocks[oldLen].breakFixups:
+    c.patch(f[0])
+    for finale in f[1]:
+      c.gen(finale)
+    exits.add c.gotoI(newNode(nkEmpty))
+  for e in exits:
+    c.patch e
   c.blocks.setLen(oldLen)
 
 template withBlock(labl: PSym; body: untyped) {.dirty.} =
   var oldLen {.gensym.} = c.blocks.len
-  c.blocks.add TBlock(label: labl, fixups: @[])
+  c.blocks.add TBlock(isTryBlock: false, label: labl)
   body
   popBlock(c, oldLen)
 
@@ -290,8 +312,6 @@ proc isTrue(n: PNode): bool =
   n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or
     n.kind == nkIntLit and n.intVal != 0
 
-proc gen(c: var Con; n: PNode) # {.noSideEffect.}
-
 when true:
   proc genWhile(c: var Con; n: PNode) =
     # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
@@ -303,12 +323,13 @@ when true:
 
     Becomes:
 
-    if cond:
-      body
+    block:
       if cond:
         body
         if cond:
           body
+          if cond:
+            body
 
     We still need to ensure 'break' resolves properly, so an AST to AST
     translation is impossible.
@@ -331,76 +352,48 @@ when true:
     L4:
       join F1
     ]#
-    if isTrue(n.sons[0]):
+    if isTrue(n[0]):
       # 'while true' is an idiom in Nim and so we produce
       # better code for it:
-      for i in 0..2:
-        withBlock(nil):
-          c.gen(n.sons[1])
+      withBlock(nil):
+        for i in 0..2:
+          c.gen(n[1])
     else:
-      let oldForksLen = c.forks.len
-      var endings: array[3, TPosition]
-      for i in 0..2:
-        withBlock(nil):
-          c.gen(n.sons[0])
+      withBlock(nil):
+        var endings: array[3, TPosition]
+        for i in 0..2:
+          c.gen(n[0])
           endings[i] = c.forkI(n)
-          c.gen(n.sons[1])
-      for i in countdown(endings.high, 0):
-        let endPos = endings[i]
-        c.patch(endPos)
-        c.joinI(c.forks.pop(), n)
-      doAssert(c.forks.len == oldForksLen)
+          c.gen(n[1])
+        for i in countdown(endings.high, 0):
+          let endPos = endings[i]
+          c.patch(endPos)
 
 else:
 
   proc genWhile(c: var Con; n: PNode) =
-    # L1:
+    # lab1:
     #   cond, tmp
-    #   fork tmp, L2
+    #   fork tmp, lab2
     #   body
-    #   jmp L1
-    # L2:
-    let oldForksLen = c.forks.len
-    let L1 = c.genLabel
+    #   jmp lab1
+    # lab2:
+    let lab1 = c.genLabel
     withBlock(nil):
-      if isTrue(n.sons[0]):
-        c.gen(n.sons[1])
-        c.jmpBack(n, L1)
+      if isTrue(n[0]):
+        c.gen(n[1])
+        c.jmpBack(n, lab1)
       else:
-        c.gen(n.sons[0])
-        let L2 = c.forkI(n)
-        c.gen(n.sons[1])
-        c.jmpBack(n, L1)
-        c.patch(L2)
-    setLen(c.forks, oldForksLen)
-
-proc genBlock(c: var Con; n: PNode) =
-  withBlock(n.sons[0].sym):
-    c.gen(n.sons[1])
-
-proc genJoins(c: var Con; n: PNode) =
-  for i in countdown(c.forks.high, 0): joinI(c, c.forks[i], n)
-
-proc genBreak(c: var Con; n: PNode) =
-  genJoins(c, n)
-  let L1 = c.gotoI(n)
-  if n.sons[0].kind == nkSym:
-    #echo cast[int](n.sons[0].sym)
-    for i in countdown(c.blocks.len-1, 0):
-      if c.blocks[i].label == n.sons[0].sym:
-        c.blocks[i].fixups.add L1
-        return
-    #globalError(n.info, "VM problem: cannot find 'break' target")
-  else:
-    c.blocks[c.blocks.high].fixups.add L1
+        c.gen(n[0])
+        let lab2 = c.forkI(n)
+        c.gen(n[1])
+        c.jmpBack(n, lab1)
+        c.patch(lab2)
 
 template forkT(n, body) =
-  let oldLen = c.forks.len
-  let L1 = c.forkI(n)
+  let lab1 = c.forkI(n)
   body
-  c.patch(L1)
-  c.joinI(L1, n)
-  setLen(c.forks, oldLen)
+  c.patch(lab1)
 
 proc genIf(c: var Con, n: PNode) =
   #[
@@ -415,15 +408,15 @@ proc genIf(c: var Con, n: PNode) =
     D
 
   cond
-  fork L1
+  fork lab1
   A
   goto Lend
-  L1:
+  lab1:
     condB
-    fork L2
+    fork lab2
     B
     goto Lend2
-  L2:
+  lab2:
     condC
     fork L3
     C
@@ -439,53 +432,49 @@ proc genIf(c: var Con, n: PNode) =
     join F1
 
   ]#
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
-  for i in 0 ..< len(n):
-    var it = n.sons[i]
-    c.gen(it.sons[0])
+  for i in 0..<n.len:
+    var it = n[i]
+    c.gen(it[0])
     if it.len == 2:
       let elsePos = forkI(c, it[1])
-      c.gen(it.sons[1])
-      endings.add(c.gotoI(it.sons[1]))
+      c.gen(it[1])
+      endings.add(c.gotoI(it[1]))
       c.patch(elsePos)
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
 
 proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, a
-  #   fork L1
+  #   fork lab1
   #   asgn dest, b
-  # L1:
+  # lab1:
   #   join F1
-  c.gen(n.sons[1])
+  c.gen(n[1])
   forkT(n):
-    c.gen(n.sons[2])
+    c.gen(n[2])
 
 proc genCase(c: var Con; n: PNode) =
-  #  if (!expr1) goto L1;
+  #  if (!expr1) goto lab1;
   #    thenPart
   #    goto LEnd
-  #  L1:
-  #  if (!expr2) goto L2;
+  #  lab1:
+  #  if (!expr2) goto lab2;
   #    thenPart2
   #    goto LEnd
-  #  L2:
+  #  lab2:
   #    elsePart
   #  Lend:
-  let isExhaustive = skipTypes(n.sons[0].typ,
+  let isExhaustive = skipTypes(n[0].typ,
     abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
 
   var endings: seq[TPosition] = @[]
-  let oldLen = c.forks.len
-  c.gen(n.sons[0])
-  for i in 1 ..< n.len:
-    let it = n.sons[i]
+  c.gen(n[0])
+  for i in 1..<n.len:
+    let it = n[i]
     if it.len == 1:
-      c.gen(it.sons[0])
+      c.gen(it[0])
     elif i == n.len-1 and isExhaustive:
       # treat the last branch as 'else' if this is an exhaustive case statement.
       c.gen(it.lastSon)
@@ -497,33 +486,58 @@ proc genCase(c: var Con; n: PNode) =
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
+
+proc genBlock(c: var Con; n: PNode) =
+  withBlock(n[0].sym):
+    c.gen(n[1])
+
+proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) =
+  let lab1 = c.gotoI(n)
+  if c.blocks[i].isTryBlock:
+    c.blocks[i].raiseFixups.add lab1
+  else:
+    var trailingFinales: seq[PNode]
+    if c.inTryStmt > 0: #Ok, we are in a try, lets see which (if any) try's we break out from:
+      for b in countdown(c.blocks.high, i):
+        if c.blocks[b].isTryBlock:
+          trailingFinales.add c.blocks[b].finale
+
+    c.blocks[i].breakFixups.add (lab1, trailingFinales)
+
+proc genBreak(c: var Con; n: PNode) =
+  if n[0].kind == nkSym:
+    #echo cast[int](n[0].sym)
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock and c.blocks[i].label == n[0].sym:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    #globalError(n.info, "VM problem: cannot find 'break' target")
+  else:
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
 
 proc genTry(c: var Con; n: PNode) =
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
+
+  let oldLen = c.blocks.len
+  c.blocks.add TBlock(isTryBlock: true, finale: if n[^1].kind == nkFinally: n[^1] else: newNode(nkEmpty))
+
   inc c.inTryStmt
-  let oldFixups = c.tryStmtFixups.len
-
   #let elsePos = c.forkI(n)
-  c.gen(n.sons[0])
+  c.gen(n[0])
   dec c.inTryStmt
-  for i in oldFixups..c.tryStmtFixups.high:
-    let f = c.tryStmtFixups[i]
-    c.patch(f)
-    # we also need to produce join instructions
-    # for the 'fork' that might preceed the goto instruction
-    if f.int-1 >= 0 and c.code[f.int-1].kind == fork:
-      c.joinI(TPosition(f.int-1), n)
 
-  setLen(c.tryStmtFixups, oldFixups)
+  for f in c.blocks[oldLen].raiseFixups:
+    c.patch(f)
+
+  c.blocks.setLen oldLen
 
   #c.patch(elsePos)
-  for i in 1 ..< n.len:
-    let it = n.sons[i]
+  for i in 1..<n.len:
+    let it = n[i]
     if it.kind != nkFinally:
-      var blen = len(it)
       let endExcept = c.forkI(it)
       c.gen(it.lastSon)
       endings.add(c.gotoI(it))
@@ -531,39 +545,39 @@ proc genTry(c: var Con; n: PNode) =
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
 
   # join the 'elsePos' forkI instruction:
-  #c.joinI(c.forks.pop(), n)
+  #c.joinI(c.blocks[^1].forks.pop(), n)
 
   let fin = lastSon(n)
   if fin.kind == nkFinally:
-    c.gen(fin.sons[0])
-  doAssert(c.forks.len == oldLen)
+    c.gen(fin[0])
 
 template genNoReturn(c: var Con; n: PNode) =
   # leave the graph
   c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 proc genRaise(c: var Con; n: PNode) =
-  genJoins(c, n)
-  gen(c, n.sons[0])
+  gen(c, n[0])
   if c.inTryStmt > 0:
-    c.tryStmtFixups.add c.gotoI(n)
+    for i in countdown(c.blocks.high, 0):
+      if c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    assert false #Unreachable
   else:
     genNoReturn(c, n)
 
 proc genImplicitReturn(c: var Con) =
   if c.owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter} and resultPos < c.owner.ast.len:
-    gen(c, c.owner.ast.sons[resultPos])
+    gen(c, c.owner.ast[resultPos])
 
 proc genReturn(c: var Con; n: PNode) =
-  genJoins(c, n)
-  if n.sons[0].kind != nkEmpty:
-    gen(c, n.sons[0])
+  if n[0].kind != nkEmpty:
+    gen(c, n[0])
   else:
     genImplicitReturn(c)
-  genNoReturn(c, n)
+  genBreakOrRaiseAux(c, 0, n)
 
 const
   InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp}
@@ -573,16 +587,6 @@ const
                 nkObjDownConv, nkObjUpConv}
   PathKinds1 = {nkHiddenStdConv, nkHiddenSubConv}
 
-proc getRoot(n: PNode): PNode =
-  result = n
-  while true:
-    case result.kind
-    of PathKinds0:
-      result = result[0]
-    of PathKinds1:
-      result = result[1]
-    else: break
-
 proc skipConvDfa*(n: PNode): PNode =
   result = n
   while true:
@@ -594,64 +598,72 @@ proc skipConvDfa*(n: PNode): PNode =
     else: break
 
 proc genUse(c: var Con; orig: PNode) =
-  let n = dfa.getRoot(orig)
+  var n = orig
+  while true:
+    case n.kind
+    of PathKinds0 - {nkBracketExpr}:
+      n = n[0]
+    of nkBracketExpr:
+      gen(c, n[1])
+      n = n[0]
+    of PathKinds1:
+      n = n[1]
+    else: break
+    if n.kind in nkCallKinds:
+      gen(c, n)
   if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: orig, kind: use, sym: if orig != n: nil else: n.sym)
+    c.code.add Instr(n: orig, kind: use)
 
-proc aliases(obj, field: PNode): bool =
+proc aliases*(obj, field: PNode): bool =
   var n = field
   var obj = obj
-  while obj.kind in {nkHiddenSubConv, nkHiddenStdConv, nkObjDownConv, nkObjUpConv}:
+  while obj.kind in {nkHiddenSubConv, nkHiddenStdConv, nkObjDownConv, nkObjUpConv,
+                     nkAddr, nkHiddenAddr, nkDerefExpr, nkHiddenDeref}:
     obj = obj[0]
   while true:
     if sameTrees(obj, n): return true
     case n.kind
-    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-       nkObjDownConv, nkObjUpConv, nkHiddenDeref, nkDerefExpr:
+    of PathKinds0, PathKinds1:
       n = n[0]
-    of nkBracketExpr:
-      let x = n[0]
-      if x.typ != nil and x.typ.skipTypes(abstractInst).kind == tyTuple:
-        n = x
-      else:
-        break
     else:
       break
-  return false
 
-proc instrTargets*(ins: Instr; loc: PNode): bool =
-  assert ins.kind in {def, use}
-  if ins.sym != nil and loc.kind == nkSym:
-    result = ins.sym == loc.sym
-  else:
-    result = ins.n == loc or sameTrees(ins.n, loc)
-  if not result:
-    # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
-    # def x.f; question: does it affect the full 'x'? No.
-    # def x; question: does it affect the 'x.f'? Yes.
-    # use x.f;  question: does it affect the full 'x'? No.
-    # use x; question does it affect 'x.f'? Yes.
-    result = aliases(ins.n, loc) or aliases(loc, ins.n)
+proc useInstrTargets*(ins: Instr; loc: PNode): bool =
+  assert ins.kind == use
+  result = sameTrees(ins.n, loc) or
+    ins.n.aliases(loc) or loc.aliases(ins.n)
+  # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
+  # use x.f;  question: does it affect the full 'x'? No.
+  # use x; question does it affect 'x.f'? Yes.
+
+proc defInstrTargets*(ins: Instr; loc: PNode): bool =
+  assert ins.kind == def
+  result = sameTrees(ins.n, loc) or ins.n.aliases(loc)
+  # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
+  # def x.f; question: does it affect the full 'x'? No.
+  # def x; question: does it affect the 'x.f'? Yes.
 
 proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
   var n = orig
   while true:
     case n.kind
     of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-       nkObjDownConv, nkObjUpConv:
+       nkObjDownConv, nkObjUpConv, nkHiddenAddr, nkAddr:
       n = n[0]
+    of nkBracketExpr:
+      # in a[i] the 'i' must be known
+      if n.len > 1 and n[1].kind in {nkCharLit..nkUInt64Lit}:
+        n = n[0]
+      else:
+        return false
     of nkHiddenDeref, nkDerefExpr:
       # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
       # pointer indirection.
+      # bug #14159, we cannot reason about sinkParam[].location as it can
+      # still be shared for tyRef.
       n = n[0]
-      return n.kind == nkSym and n.sym.owner == owner and (isSinkParam(n.sym) or
-          n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned, tyVar})
-    of nkBracketExpr:
-      let x = n[0]
-      if x.typ != nil and x.typ.skipTypes(abstractInst).kind == tyTuple:
-        n = x
-      else:
-        break
+      return n.kind == nkSym and n.sym.owner == owner and (
+          n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
     else:
       break
   # XXX Allow closure deref operations here if we know
@@ -671,18 +683,29 @@ proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
   # lower level C++ optimizer to specialize this code.
 
 proc genDef(c: var Con; n: PNode) =
-  if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: n, kind: def, sym: n.sym)
-  elif isAnalysableFieldAccess(n, c.owner):
-    c.code.add Instr(n: n, kind: def, sym: nil)
+  var m = n
+  # XXX do something about this duplicated logic here.
+  while true:
+    case m.kind
+    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
+        nkObjDownConv, nkObjUpConv, nkHiddenAddr, nkAddr:
+      m = m[0]
+    of nkBracketExpr:
+      gen(c, m[1])
+      m = m[0]
+    of nkHiddenDeref, nkDerefExpr:
+      m = m[0]
+    else:
+      break
 
-proc canRaise(fn: PNode): bool =
-  const magicsThatCanRaise = {
-    mNone, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst}
-  if fn.kind == nkSym and fn.sym.magic notin magicsThatCanRaise:
-    result = false
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    c.code.add Instr(n: n, kind: def)
+  elif isAnalysableFieldAccess(n, c.owner):
+    c.code.add Instr(n: n, kind: def)
   else:
-    result = true
+    # bug #13314: An assignment to t5.w = -5 is a usage of 't5'
+    # we still need to gather the use information:
+    gen(c, n)
 
 proc genCall(c: var Con; n: PNode) =
   gen(c, n[0])
@@ -692,22 +715,24 @@ proc genCall(c: var Con; n: PNode) =
   for i in 1..<n.len:
     gen(c, n[i])
     when false:
-      if t != nil and i < t.len and t.sons[i].kind == tyVar:
+      if t != nil and i < t.len and t[i].kind == tyVar:
         # This is wrong! Pass by var is a 'might def', not a 'must def'
         # like the other defs we emit. This is not good enough for a move
         # optimizer.
         genDef(c, n[i])
   # every call can potentially raise:
-  if c.inTryStmt > 0 and canRaise(n[0]):
+  if c.inTryStmt > 0 and canRaiseConservative(n[0]):
     # we generate the instruction sequence:
-    # fork L1
+    # fork lab1
     # goto exceptionHandler (except or finally)
-    # L1:
+    # lab1:
     # join F1
     let endGoto = c.forkI(n)
-    c.tryStmtFixups.add c.gotoI(n)
+    for i in countdown(c.blocks.high, 0):
+      if c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        break
     c.patch(endGoto)
-    c.joinI(c.forks.pop(), n)
   dec c.inCall
 
 proc genMagic(c: var Con; n: PNode; m: TMagic) =
@@ -725,23 +750,23 @@ proc genVarSection(c: var Con; n: PNode) =
       discard
     elif a.kind == nkVarTuple:
       gen(c, a.lastSon)
-      for i in 0 .. a.len-3: genDef(c, a[i])
+      for i in 0..<a.len-2: genDef(c, a[i])
     else:
       gen(c, a.lastSon)
       if a.lastSon.kind != nkEmpty:
-        genDef(c, a.sons[0])
+        genDef(c, a[0])
 
 proc gen(c: var Con; n: PNode) =
   case n.kind
   of nkSym: genUse(c, n)
   of nkCallKinds:
-    if n.sons[0].kind == nkSym:
-      let s = n.sons[0].sym
+    if n[0].kind == nkSym:
+      let s = n[0].sym
       if s.magic != mNone:
         genMagic(c, n, s.magic)
       else:
         genCall(c, n)
-      if sfNoReturn in n.sons[0].sym.flags:
+      if sfNoReturn in n[0].sym.flags:
         genNoReturn(c, n)
     else:
       genCall(c, n)
@@ -757,7 +782,7 @@ proc gen(c: var Con; n: PNode) =
   of nkIfStmt, nkIfExpr: genIf(c, n)
   of nkWhenStmt:
     # This is "when nimvm" node. Chose the first branch.
-    gen(c, n.sons[0].sons[1])
+    gen(c, n[0][1])
   of nkCaseStmt: genCase(c, n)
   of nkWhileStmt: genWhile(c, n)
   of nkBlockExpr, nkBlockStmt: genBlock(c, n)
@@ -766,13 +791,13 @@ proc gen(c: var Con; n: PNode) =
   of nkBreakStmt: genBreak(c, n)
   of nkTryStmt, nkHiddenTryStmt: genTry(c, n)
   of nkStmtList, nkStmtListExpr, nkChckRangeF, nkChckRange64, nkChckRange,
-     nkBracket, nkCurly, nkPar, nkTupleConstr, nkClosure, nkObjConstr:
+     nkBracket, nkCurly, nkPar, nkTupleConstr, nkClosure, nkObjConstr, nkYieldStmt:
     for x in n: gen(c, x)
   of nkPragmaBlock: gen(c, n.lastSon)
-  of nkDiscardStmt, nkObjDownConv, nkObjUpConv: gen(c, n.sons[0])
+  of nkDiscardStmt, nkObjDownConv, nkObjUpConv: gen(c, n[0])
   of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast, nkHiddenSubConv, nkHiddenStdConv:
-    gen(c, n.sons[1])
-  of nkStringToCString, nkCStringToString: gen(c, n.sons[0])
+    gen(c, n[1])
+  of nkStringToCString, nkCStringToString: gen(c, n[0])
   of nkVarSection, nkLetSection: genVarSection(c, n)
   of nkDefer:
     doAssert false, "dfa construction pass requires the elimination of 'defer'"
@@ -781,6 +806,7 @@ proc gen(c: var Con; n: PNode) =
 proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
   ## constructs a control flow graph for ``body``.
   var c = Con(code: @[], blocks: @[], owner: s)
-  gen(c, body)
-  genImplicitReturn(c)
+  withBlock(s):
+    gen(c, body)
+    genImplicitReturn(c)
   shallowCopy(result, c.code)

compiler/docgen.nim

@@ -15,17 +15,19 @@ import
   ast, strutils, strtabs, options, msgs, os, ropes, idents,
   wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast,
   packages/docutils/rst, packages/docutils/rstgen,
-  packages/docutils/highlite, json, xmltree, cgi, trees, types,
-  typesrenderer, astalgo, modulepaths, lineinfos, sequtils, intsets,
+  json, xmltree, cgi, trees, types,
+  typesrenderer, astalgo, lineinfos, intsets,
   pathutils, trees
 
 const
   exportSection = skField
+  htmldocsDir = RelativeDir"htmldocs"
 
 type
   TSections = array[TSymKind, Rope]
   TDocumentor = object of rstgen.RstGenerator
     modDesc: Rope       # module description
+    module: PSym
     modDeprecationMsg: Rope
     toc, section: TSections
     indexValFilename: string
@@ -42,20 +44,68 @@ type
     destFile*: AbsoluteFile
     thisDir*: AbsoluteDir
     examples: string
+    wroteCss*: bool
 
   PDoc* = ref TDocumentor ## Alias to type less.
 
+proc nativeToUnix(path: string): string =
+  doAssert not path.isAbsolute # absolute files need more care for the drive
+  when DirSep == '\\':
+    result = replace(path, '\\', '/')
+  else: result = path
+
+proc docOutDir(conf: ConfigRef, subdir: RelativeDir = RelativeDir""): AbsoluteDir =
+  if not conf.outDir.isEmpty: conf.outDir else: conf.projectPath / subdir
+
+proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): RelativeFile =
+  ## returns a relative file that will be appended to outDir
+  let file2 = $file
+  template bail() =
+    result = relativeTo(file, conf.projectPath)
+  proc nimbleDir(): AbsoluteDir =
+    getNimbleFile(conf, file2).parentDir.AbsoluteDir
+  case conf.docRoot:
+  of "@default": # using `@` instead of `$` to avoid shell quoting complications
+    result = getRelativePathFromConfigPath(conf, file)
+    let dir = nimbleDir()
+    if not dir.isEmpty:
+      let result2 = relativeTo(file, dir)
+      if not result2.isEmpty and (result.isEmpty or result2.string.len < result.string.len):
+        result = result2
+    if result.isEmpty: bail()
+  of "@pkg":
+    let dir = nimbleDir()
+    if dir.isEmpty: bail()
+    else: result = relativeTo(file, dir)
+  of "@path":
+    result = getRelativePathFromConfigPath(conf, file)
+    if result.isEmpty: bail()
+  elif conf.docRoot.len > 0:
+    doAssert conf.docRoot.isAbsolute, conf.docRoot # or globalError
+    doAssert conf.docRoot.existsDir, conf.docRoot
+    result = relativeTo(file, conf.docRoot.AbsoluteDir)
+  else:
+    bail()
+  if isAbsolute(result.string):
+    result = file.string.splitPath()[1].RelativeFile
+  if isTitle:
+    result = result.string.nativeToUnix.RelativeFile
+  else:
+    result = result.string.replace("..", "@@").RelativeFile ## refs #13223
+  doAssert not result.isEmpty
+  doAssert not isAbsolute(result.string)
+
 proc whichType(d: PDoc; n: PNode): PSym =
   if n.kind == nkSym:
     if d.types.strTableContains(n.sym):
       result = n.sym
   else:
-    for i in 0..<safeLen(n):
+    for i in 0..<n.safeLen:
       let x = whichType(d, n[i])
       if x != nil: return x
 
 proc attachToType(d: PDoc; p: PSym): PSym =
-  let params = p.ast.sons[paramsPos]
+  let params = p.ast[paramsPos]
   template check(i) =
     result = whichType(d, params[i])
     if result != nil: return result
@@ -68,7 +118,7 @@ proc attachToType(d: PDoc; p: PSym): PSym =
 
 template declareClosures =
   proc compilerMsgHandler(filename: string, line, col: int,
-                          msgKind: rst.MsgKind, arg: string) {.procvar, gcsafe.} =
+                          msgKind: rst.MsgKind, arg: string) {.gcsafe.} =
     # translate msg kind:
     var k: TMsgKind
     case msgKind
@@ -85,7 +135,7 @@ template declareClosures =
     {.gcsafe.}:
       globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg)
 
-  proc docgenFindFile(s: string): string {.procvar, gcsafe.} =
+  proc docgenFindFile(s: string): string {.gcsafe.} =
     result = options.findFile(conf, s).string
     if result.len == 0:
       result = getCurrentDir() / s
@@ -102,12 +152,11 @@ proc parseRst(text, filename: string,
 proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
                  ext: string, dir: RelativeDir; guessTarget: bool): AbsoluteFile =
   if optWholeProject in conf.globalOptions:
-    let d = if conf.outDir.isEmpty: conf.projectPath / dir else: conf.outDir
+    let d = conf.docOutDir(dir)
     createDir(d)
     result = d / changeFileExt(filename, ext)
   elif guessTarget:
-    let d = if not conf.outDir.isEmpty: conf.outDir
-            else: conf.projectPath
+    let d = conf.docOutDir
     createDir(d)
     result = d / changeFileExt(filename, ext)
   elif not conf.outFile.isEmpty:
@@ -115,9 +164,10 @@ proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
   else:
     result = getOutFile(conf, filename, ext)
 
-proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, outExt: string = HtmlExt): PDoc =
+proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, outExt: string = HtmlExt, module: PSym = nil): PDoc =
   declareClosures()
   new(result)
+  result.module = module
   result.conf = conf
   result.cache = cache
   initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex),
@@ -146,10 +196,10 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   initStrTable result.types
   result.onTestSnippet =
     proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) =
+      inc(gen.id)
       var d = TDocumentor(gen)
       var outp: AbsoluteFile
       if filename.len == 0:
-        inc(d.id)
         let nameOnly = splitFile(d.filename).name
         outp = getNimcacheDir(conf) / RelativeDir(nameOnly) /
                RelativeFile(nameOnly & "_snippet_" & $d.id & ".nim")
@@ -171,16 +221,16 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
       if execShellCmd(c2) != status:
         rawMessage(conf, errGenerated, "executing of external program failed: " & c2)
   result.emitted = initIntSet()
-  result.destFile = getOutFile2(conf, relativeTo(filename, conf.projectPath),
-                                outExt, RelativeDir"htmldocs", false)
+  result.destFile = getOutFile2(conf, presentationPath(conf, filename),
+                                outExt, htmldocsDir, false)
   result.thisDir = result.destFile.splitFile.dir
 
 template dispA(conf: ConfigRef; dest: var Rope, xml, tex: string, args: openArray[Rope]) =
-  if conf.cmd != cmdRst2tex: addf(dest, xml, args)
-  else: addf(dest, tex, args)
+  if conf.cmd != cmdRst2tex: dest.addf(xml, args)
+  else: dest.addf(tex, args)
 
 proc getVarIdx(varnames: openArray[string], id: string): int =
-  for i in 0 .. high(varnames):
+  for i in 0..high(varnames):
     if cmpIgnoreStyle(varnames[i], id) == 0:
       return i
   result = -1
@@ -189,44 +239,43 @@ proc ropeFormatNamedVars(conf: ConfigRef; frmt: FormatStr,
                          varnames: openArray[string],
                          varvalues: openArray[Rope]): Rope =
   var i = 0
-  var L = len(frmt)
   result = nil
   var num = 0
-  while i < L:
+  while i < frmt.len:
     if frmt[i] == '$':
       inc(i)                  # skip '$'
       case frmt[i]
       of '#':
-        add(result, varvalues[num])
+        result.add(varvalues[num])
         inc(num)
         inc(i)
       of '$':
-        add(result, "$")
+        result.add("$")
         inc(i)
       of '0'..'9':
         var j = 0
         while true:
           j = (j * 10) + ord(frmt[i]) - ord('0')
           inc(i)
-          if (i > L + 0 - 1) or not (frmt[i] in {'0'..'9'}): break
+          if (i > frmt.len + 0 - 1) or not (frmt[i] in {'0'..'9'}): break
         if j > high(varvalues) + 1:
           rawMessage(conf, errGenerated, "Invalid format string; too many $s: " & frmt)
         num = j
-        add(result, varvalues[j - 1])
+        result.add(varvalues[j - 1])
       of 'A'..'Z', 'a'..'z', '\x80'..'\xFF':
         var id = ""
         while true:
-          add(id, frmt[i])
+          id.add(frmt[i])
           inc(i)
           if not (frmt[i] in {'A'..'Z', '_', 'a'..'z', '\x80'..'\xFF'}): break
         var idx = getVarIdx(varnames, id)
-        if idx >= 0: add(result, varvalues[idx])
+        if idx >= 0: result.add(varvalues[idx])
         else: rawMessage(conf, errGenerated, "unknown substition variable: " & id)
       of '{':
         var id = ""
         inc(i)
         while i < frmt.len and frmt[i] != '}':
-          add(id, frmt[i])
+          id.add(frmt[i])
           inc(i)
         if i >= frmt.len:
           rawMessage(conf, errGenerated, "expected closing '}'")
@@ -234,15 +283,15 @@ proc ropeFormatNamedVars(conf: ConfigRef; frmt: FormatStr,
           inc(i)                # skip }
         # search for the variable:
         let idx = getVarIdx(varnames, id)
-        if idx >= 0: add(result, varvalues[idx])
+        if idx >= 0: result.add(varvalues[idx])
         else: rawMessage(conf, errGenerated, "unknown substition variable: " & id)
       else:
-        add(result, "$")
+        result.add("$")
     var start = i
-    while i < L:
+    while i < frmt.len:
       if frmt[i] != '$': inc(i)
       else: break
-    if i - 1 >= start: add(result, substr(frmt, start, i - 1))
+    if i - 1 >= start: result.add(substr(frmt, start, i - 1))
 
 proc genComment(d: PDoc, n: PNode): string =
   result = ""
@@ -259,8 +308,8 @@ proc genRecCommentAux(d: PDoc, n: PNode): Rope =
     if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef,
                   nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkHiddenStdConv}:
       # notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}:
-      for i in 0 ..< len(n):
-        result = genRecCommentAux(d, n.sons[i])
+      for i in 0..<n.len:
+        result = genRecCommentAux(d, n[i])
         if result != nil: return
   else:
     when defined(nimNoNilSeqs): n.comment = ""
@@ -287,19 +336,18 @@ proc getPlainDocstring(n: PNode): string =
   if startsWith(n.comment, "##"):
     result = n.comment
   if result.len < 1:
-    for i in 0 ..< safeLen(n):
-      result = getPlainDocstring(n.sons[i])
+    for i in 0..<n.safeLen:
+      result = getPlainDocstring(n[i])
       if result.len > 0: return
 
 proc belongsToPackage(conf: ConfigRef; module: PSym): bool =
-  result = module.kind == skModule and module.owner != nil and
-      module.owner.id == conf.mainPackageId
+  result = module.kind == skModule and module.getnimblePkgId == conf.mainPackageId
 
 proc externalDep(d: PDoc; module: PSym): string =
-  if optWholeProject in d.conf.globalOptions:
+  if optWholeProject in d.conf.globalOptions or d.conf.docRoot.len > 0:
     let full = AbsoluteFile toFullPath(d.conf, FileIndex module.position)
-    let tmp = getOutFile2(d.conf, full.relativeTo(d.conf.projectPath), HtmlExt,
-        RelativeDir"htmldocs", sfMainModule notin module.flags)
+    let tmp = getOutFile2(d.conf, presentationPath(d.conf, full), HtmlExt,
+                          htmldocsDir, sfMainModule notin module.flags)
     result = relativeTo(tmp, d.thisDir, '/').string
   else:
     result = extractFilename toFullPath(d.conf, FileIndex module.position)
@@ -358,7 +406,7 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
         dispA(d.conf, result, "<span class=\"Identifier\">$1</span>",
               "\\spanIdentifier{$1}", [rope(esc(d.target, literal))])
     of tkSpaces, tkInvalid:
-      add(result, literal)
+      result.add(literal)
     of tkCurlyDotLe:
       dispA(d.conf, result, "<span>" & # This span is required for the JS to work properly
         """<span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span>
@@ -406,6 +454,7 @@ proc runAllExamples(d: PDoc) =
                 elif isDefined(d.conf, "objc"): "objc"
                 else: "c"
   if os.execShellCmd(os.getAppFilename() & " " & backend &
+                    " --warning[UnusedImport]:off" &
                     " --path:" & quoteShell(d.conf.projectPath) &
                     " --nimcache:" & quoteShell(outputDir) &
                     " -r " & quoteShell(outp)) != 0:
@@ -415,12 +464,6 @@ proc runAllExamples(d: PDoc) =
     rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
     removeFile(outp.changeFileExt(ExeExt))
 
-proc extractImports(n: PNode; result: PNode) =
-  if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
-    result.add copyTree(n)
-    n.kind = nkEmpty
-    return
-  for i in 0..<n.safeLen: extractImports(n[i], result)
 
 proc prepareExamples(d: PDoc; n: PNode) =
   var docComment = newTree(nkCommentStmt)
@@ -430,12 +473,20 @@ proc prepareExamples(d: PDoc; n: PNode) =
       docComment,
       newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
   runnableExamples.info = n.info
-  let imports = newTree(nkStmtList)
-  var savedLastSon = copyTree n.lastSon
-  extractImports(savedLastSon, imports)
-  for imp in imports: runnableExamples.add imp
-  runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon)
+  for a in n.lastSon: runnableExamples.add a
   testExample(d, runnableExamples)
+  when false:
+    proc extractImports(n: PNode; result: PNode) =
+      if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
+        result.add copyTree(n)
+        n.kind = nkEmpty
+        return
+      for i in 0..<n.safeLen: extractImports(n[i], result)
+    let imports = newTree(nkStmtList)
+    var savedLastSon = copyTree n.lastSon
+    extractImports(savedLastSon, imports)
+    for imp in imports: runnableExamples.add imp
+    runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon)
 
 proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope) =
   if n.info.fileIndex != orig.info.fileIndex: return
@@ -462,7 +513,7 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope) =
         nodeToHighlightedHtml(d, b, dest, {}, nil)
       dest.add(d.config.getOrDefault"doc.listing_end" % id)
   else: discard
-  for i in 0 ..< n.safeLen:
+  for i in 0..<n.safeLen:
     getAllRunnableExamplesRec(d, n[i], orig, dest)
 
 proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
@@ -471,8 +522,8 @@ proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
 proc isVisible(d: PDoc; n: PNode): bool =
   result = false
   if n.kind == nkPostfix:
-    if n.len == 2 and n.sons[0].kind == nkIdent:
-      var v = n.sons[0].ident
+    if n.len == 2 and n[0].kind == nkIdent:
+      var v = n[0].ident
       result = v.id == ord(wStar) or v.id == ord(wMinus)
   elif n.kind == nkSym:
     # we cannot generate code for forwarded symbols here as we have no
@@ -485,12 +536,12 @@ proc isVisible(d: PDoc; n: PNode): bool =
     if result and containsOrIncl(d.emitted, n.sym.id):
       result = false
   elif n.kind == nkPragmaExpr:
-    result = isVisible(d, n.sons[0])
+    result = isVisible(d, n[0])
 
 proc getName(d: PDoc, n: PNode, splitAfter = -1): string =
   case n.kind
-  of nkPostfix: result = getName(d, n.sons[1], splitAfter)
-  of nkPragmaExpr: result = getName(d, n.sons[0], splitAfter)
+  of nkPostfix: result = getName(d, n[1], splitAfter)
+  of nkPragmaExpr: result = getName(d, n[0], splitAfter)
   of nkSym: result = esc(d.target, n.sym.renderDefinitionName, splitAfter)
   of nkIdent: result = esc(d.target, n.ident.s, splitAfter)
   of nkAccQuoted:
@@ -504,8 +555,8 @@ proc getName(d: PDoc, n: PNode, splitAfter = -1): string =
 
 proc getNameIdent(cache: IdentCache; n: PNode): PIdent =
   case n.kind
-  of nkPostfix: result = getNameIdent(cache, n.sons[1])
-  of nkPragmaExpr: result = getNameIdent(cache, n.sons[0])
+  of nkPostfix: result = getNameIdent(cache, n[1])
+  of nkPragmaExpr: result = getNameIdent(cache, n[0])
   of nkSym: result = n.sym.name
   of nkIdent: result = n.ident
   of nkAccQuoted:
@@ -519,13 +570,13 @@ proc getNameIdent(cache: IdentCache; n: PNode): PIdent =
 
 proc getRstName(n: PNode): PRstNode =
   case n.kind
-  of nkPostfix: result = getRstName(n.sons[1])
-  of nkPragmaExpr: result = getRstName(n.sons[0])
+  of nkPostfix: result = getRstName(n[1])
+  of nkPragmaExpr: result = getRstName(n[0])
   of nkSym: result = newRstNode(rnLeaf, n.sym.renderDefinitionName)
   of nkIdent: result = newRstNode(rnLeaf, n.ident.s)
   of nkAccQuoted:
-    result = getRstName(n.sons[0])
-    for i in 1 ..< n.len: result.text.add(getRstName(n[i]).text)
+    result = getRstName(n[0])
+    for i in 1..<n.len: result.text.add(getRstName(n[i]).text)
   of nkOpenSymChoice, nkClosedSymChoice:
     result = getRstName(n[0])
   else:
@@ -570,7 +621,7 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string =
   of skTemplate: result.add(".t")
   of skConverter: result.add(".c")
   else: discard
-  if len(n) > paramsPos and n[paramsPos].kind == nkFormalParams:
+  if n.len > paramsPos and n[paramsPos].kind == nkFormalParams:
     let params = renderParamTypes(n[paramsPos])
     if params.len > 0:
       result.add(defaultParamSeparator)
@@ -589,7 +640,7 @@ proc docstringSummary(rstText: string): string =
   ##
   ## Most docstrings will contain a one liner summary, so stripping at the
   ## first newline is usually fine. If after that the content is still too big,
-  ## it is stripped at the first comma, colon or dot, usual english sentence
+  ## it is stripped at the first comma, colon or dot, usual English sentence
   ## separators.
   ##
   ## No guarantees are made on the size of the output, but it should be small.
@@ -628,8 +679,12 @@ proc genDeprecationMsg(d: PDoc, n: PNode): Rope =
   else:
     doAssert false
 
-proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
-  if not isVisible(d, nameNode): return
+type DocFlags = enum
+  kDefault
+  kForceExport
+
+proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
+  if (docFlags != kForceExport) and not isVisible(d, nameNode): return
   let
     name = getName(d, nameNode)
     nameRope = name.rope
@@ -650,8 +705,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
     plainName.add(literal)
 
   var pragmaNode: PNode = nil
-  if n.isCallable and n.sons[pragmasPos].kind != nkEmpty:
-    pragmaNode = findPragma(n.sons[pragmasPos], wDeprecated)
+  if n.isCallable and n[pragmasPos].kind != nkEmpty:
+    pragmaNode = findPragma(n[pragmasPos], wDeprecated)
 
   inc(d.id)
   let
@@ -677,17 +732,19 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
       let cwd = canonicalizePath(d.conf, getCurrentDir())
       var path = toFullPath(d.conf, n.info)
       if path.startsWith(cwd):
-        path = path[cwd.len+1 .. ^1].replace('\\', '/')
+        path = path[cwd.len+1..^1].replace('\\', '/')
     let gitUrl = getConfigVar(d.conf, "git.url")
     if gitUrl.len > 0:
-      let defaultBranch = if NimPatch mod 2 == 1: "devel" else: "master"
+      let defaultBranch =
+        if NimPatch mod 2 == 1: "devel"
+        else: "version-$1-$2" % [$NimMajor, $NimMinor]
       let commit = getConfigVar(d.conf, "git.commit", defaultBranch)
       let develBranch = getConfigVar(d.conf, "git.devel", "devel")
       dispA(d.conf, seeSrcRope, "$1", "", [ropeFormatNamedVars(d.conf, docItemSeeSrc,
           ["path", "line", "url", "commit", "devel"], [rope path.string,
           rope($n.info.line), rope gitUrl, rope commit, rope develBranch])])
 
-  add(d.section[k], ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.item"),
+  d.section[k].add(ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.item"),
     ["name", "header", "desc", "itemID", "header_plain", "itemSym",
       "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "seeSrc", "deprecationMsg"],
     [nameRope, result, comm, itemIDRope, plainNameRope, plainSymbolRope,
@@ -710,7 +767,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
       setIndexTerm(d[], external, symbolOrId, plain, nameNode.sym.name.s & '.' & plain,
         xmltree.escape(getPlainDocstring(e).docstringSummary))
 
-  add(d.toc[k], ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.item.toc"),
+  d.toc[k].add(ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.item.toc"),
     ["name", "header", "desc", "itemID", "header_plain", "itemSym",
       "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "attype"],
     [rope(getName(d, nameNode, d.splitAfter)), result, comm,
@@ -742,6 +799,35 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode =
     result["description"] = %comm
   if r.buf.len > 0:
     result["code"] = %r.buf
+  if k in routineKinds:
+    result["signature"] = newJObject()
+    if n[paramsPos][0].kind != nkEmpty:
+      result["signature"]["return"] = %($n[paramsPos][0])
+    if n[paramsPos].len > 1:
+      result["signature"]["arguments"] = newJArray()
+    for paramIdx in 1 ..< n[paramsPos].len:
+      for identIdx in 0 ..< n[paramsPos][paramIdx].len - 2:
+        let
+          paramName = $n[paramsPos][paramIdx][identIdx]
+          paramType = $n[paramsPos][paramIdx][^2]
+        if n[paramsPos][paramIdx][^1].kind != nkEmpty:
+          let paramDefault = $n[paramsPos][paramIdx][^1]
+          result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault}
+        else:
+          result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType}
+    if n[pragmasPos].kind != nkEmpty:
+      result["signature"]["pragmas"] = newJArray()
+      for pragma in n[pragmasPos]:
+        result["signature"]["pragmas"].add %($pragma)
+    if n[genericParamsPos].kind != nkEmpty:
+      result["signature"]["genericParams"] = newJArray()
+      for genericParam in n[genericParamsPos]:
+        var param = %{"name": %($genericParam)}
+        if genericParam.sym.typ.sons.len > 0:
+          param["types"] = newJArray()
+        for kind in genericParam.sym.typ.sons:
+          param["types"].add %($kind)
+        result["signature"]["genericParams"].add param
 
 proc checkForFalse(n: PNode): bool =
   result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0
@@ -756,11 +842,11 @@ proc traceDeps(d: PDoc, it: PNode) =
     a.add dir
     a.add sep # dummy entry, replaced in the loop
     for x in it[2]:
-      a.sons[2] = x
+      a[2] = x
       traceDeps(d, a)
   elif it.kind == nkSym and belongsToPackage(d.conf, it.sym):
     let external = externalDep(d, it.sym)
-    if d.section[k] != nil: add(d.section[k], ", ")
+    if d.section[k] != nil: d.section[k].add(", ")
     dispA(d.conf, d.section[k],
           "<a class=\"reference external\" href=\"$2\">$1</a>",
           "$1", [rope esc(d.target, changeFileExt(external, "")),
@@ -770,7 +856,7 @@ proc exportSym(d: PDoc; s: PSym) =
   const k = exportSection
   if s.kind == skModule and belongsToPackage(d.conf, s):
     let external = externalDep(d, s)
-    if d.section[k] != nil: add(d.section[k], ", ")
+    if d.section[k] != nil: d.section[k].add(", ")
     dispA(d.conf, d.section[k],
           "<a class=\"reference external\" href=\"$2\">$1</a>",
           "$1", [rope esc(d.target, changeFileExt(external, "")),
@@ -779,7 +865,7 @@ proc exportSym(d: PDoc; s: PSym) =
     let module = originatingModule(s)
     if belongsToPackage(d.conf, module):
       let external = externalDep(d, module)
-      if d.section[k] != nil: add(d.section[k], ", ")
+      if d.section[k] != nil: d.section[k].add(", ")
       # XXX proper anchor generation here
       dispA(d.conf, d.section[k],
             "<a href=\"$2#$1\"><span class=\"Identifier\">$1</span></a>",
@@ -787,37 +873,37 @@ proc exportSym(d: PDoc; s: PSym) =
             rope changeFileExt(external, "html")])
 
 proc documentNewEffect(cache: IdentCache; n: PNode): PNode =
-  let s = n.sons[namePos].sym
+  let s = n[namePos].sym
   if tfReturnsNew in s.typ.flags:
     result = newIdentNode(getIdent(cache, "new"), n.info)
 
 proc documentEffect(cache: IdentCache; n, x: PNode, effectType: TSpecialWord, idx: int): PNode =
   let spec = effectSpec(x, effectType)
   if isNil(spec):
-    let s = n.sons[namePos].sym
+    let s = n[namePos].sym
 
-    let actual = s.typ.n.sons[0]
+    let actual = s.typ.n[0]
     if actual.len != effectListLen: return
-    let real = actual.sons[idx]
+    let real = actual[idx]
 
     # warning: hack ahead:
     var effects = newNodeI(nkBracket, n.info, real.len)
-    for i in 0 ..< real.len:
+    for i in 0..<real.len:
       var t = typeToString(real[i].typ)
       if t.startsWith("ref "): t = substr(t, 4)
-      effects.sons[i] = newIdentNode(getIdent(cache, t), n.info)
+      effects[i] = newIdentNode(getIdent(cache, t), n.info)
       # set the type so that the following analysis doesn't screw up:
-      effects.sons[i].typ = real[i].typ
+      effects[i].typ = real[i].typ
 
     result = newNode(nkExprColonExpr, n.info, @[
       newIdentNode(getIdent(cache, specialWords[effectType]), n.info), effects])
 
 proc documentWriteEffect(cache: IdentCache; n: PNode; flag: TSymFlag; pragmaName: string): PNode =
-  let s = n.sons[namePos].sym
+  let s = n[namePos].sym
   let params = s.typ.n
 
   var effects = newNodeI(nkBracket, n.info)
-  for i in 1 ..< params.len:
+  for i in 1..<params.len:
     if params[i].kind == nkSym and flag in params[i].sym.flags:
       effects.add params[i]
 
@@ -826,8 +912,8 @@ proc documentWriteEffect(cache: IdentCache; n: PNode; flag: TSymFlag; pragmaName
       newIdentNode(getIdent(cache, pragmaName), n.info), effects])
 
 proc documentRaises*(cache: IdentCache; n: PNode) =
-  if n.sons[namePos].kind != nkSym: return
-  let pragmas = n.sons[pragmasPos]
+  if n[namePos].kind != nkSym: return
+  let pragmas = n[pragmasPos]
   let p1 = documentEffect(cache, n, pragmas, wRaises, exceptionEffects)
   let p2 = documentEffect(cache, n, pragmas, wTags, tagEffects)
   let p3 = documentWriteEffect(cache, n, sfWrittenTo, "writes")
@@ -836,59 +922,65 @@ proc documentRaises*(cache: IdentCache; n: PNode) =
 
   if p1 != nil or p2 != nil or p3 != nil or p4 != nil or p5 != nil:
     if pragmas.kind == nkEmpty:
-      n.sons[pragmasPos] = newNodeI(nkPragma, n.info)
-    if p1 != nil: n.sons[pragmasPos].add p1
-    if p2 != nil: n.sons[pragmasPos].add p2
-    if p3 != nil: n.sons[pragmasPos].add p3
-    if p4 != nil: n.sons[pragmasPos].add p4
-    if p5 != nil: n.sons[pragmasPos].add p5
+      n[pragmasPos] = newNodeI(nkPragma, n.info)
+    if p1 != nil: n[pragmasPos].add p1
+    if p2 != nil: n[pragmasPos].add p2
+    if p3 != nil: n[pragmasPos].add p3
+    if p4 != nil: n[pragmasPos].add p4
+    if p5 != nil: n[pragmasPos].add p5
 
-proc generateDoc*(d: PDoc, n, orig: PNode) =
+proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
+  template genItemAux(skind) =
+    genItem(d, n, n[namePos], skind, docFlags)
   case n.kind
   of nkPragma:
     let pragmaNode = findPragma(n, wDeprecated)
-    add(d.modDeprecationMsg, genDeprecationMsg(d, pragmaNode))
-  of nkCommentStmt: add(d.modDesc, genComment(d, n))
+    d.modDeprecationMsg.add(genDeprecationMsg(d, pragmaNode))
+  of nkCommentStmt: d.modDesc.add(genComment(d, n))
   of nkProcDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    genItem(d, n, n.sons[namePos], skProc)
+    genItemAux(skProc)
   of nkFuncDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    genItem(d, n, n.sons[namePos], skFunc)
+    genItemAux(skFunc)
   of nkMethodDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    genItem(d, n, n.sons[namePos], skMethod)
+    genItemAux(skMethod)
   of nkIteratorDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    genItem(d, n, n.sons[namePos], skIterator)
-  of nkMacroDef: genItem(d, n, n.sons[namePos], skMacro)
-  of nkTemplateDef: genItem(d, n, n.sons[namePos], skTemplate)
+    genItemAux(skIterator)
+  of nkMacroDef: genItemAux(skMacro)
+  of nkTemplateDef: genItemAux(skTemplate)
   of nkConverterDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    genItem(d, n, n.sons[namePos], skConverter)
+    genItemAux(skConverter)
   of nkTypeSection, nkVarSection, nkLetSection, nkConstSection:
-    for i in 0 ..< sonsLen(n):
-      if n.sons[i].kind != nkCommentStmt:
+    for i in 0..<n.len:
+      if n[i].kind != nkCommentStmt:
         # order is always 'type var let const':
-        genItem(d, n.sons[i], n.sons[i].sons[0],
-                succ(skType, ord(n.kind)-ord(nkTypeSection)))
+        genItem(d, n[i], n[i][0],
+                succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags)
   of nkStmtList:
-    for i in 0 ..< sonsLen(n): generateDoc(d, n.sons[i], orig)
+    for i in 0..<n.len: generateDoc(d, n[i], orig)
   of nkWhenStmt:
     # generate documentation for the first branch only:
-    if not checkForFalse(n.sons[0].sons[0]):
-      generateDoc(d, lastSon(n.sons[0]), orig)
+    if not checkForFalse(n[0][0]):
+      generateDoc(d, lastSon(n[0]), orig)
   of nkImportStmt:
     for it in n: traceDeps(d, it)
   of nkExportStmt:
     for it in n:
-      if it.kind == nkSym: exportSym(d, it.sym)
+      if it.kind == nkSym:
+        if d.module != nil and d.module == it.sym.owner:
+          generateDoc(d, it.sym.ast, orig, kForceExport)
+        else:
+          exportSym(d, it.sym)
   of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept"
-  of nkFromStmt, nkImportExceptStmt: traceDeps(d, n.sons[0])
+  of nkFromStmt, nkImportExceptStmt: traceDeps(d, n[0])
   of nkCallKinds:
     var comm: Rope = nil
     getAllRunnableExamples(d, n, comm)
-    if comm != nil: add(d.modDesc, comm)
+    if comm != nil: d.modDesc.add(comm)
   else: discard
 
 proc add(d: PDoc; j: JsonNode) =
@@ -900,39 +992,39 @@ proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
     if includeComments:
       d.add %*{"comment": genComment(d, n)}
     else:
-      add(d.modDesc, genComment(d, n))
+      d.modDesc.add(genComment(d, n))
   of nkProcDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    d.add genJsonItem(d, n, n.sons[namePos], skProc)
+    d.add genJsonItem(d, n, n[namePos], skProc)
   of nkFuncDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    d.add genJsonItem(d, n, n.sons[namePos], skFunc)
+    d.add genJsonItem(d, n, n[namePos], skFunc)
   of nkMethodDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    d.add genJsonItem(d, n, n.sons[namePos], skMethod)
+    d.add genJsonItem(d, n, n[namePos], skMethod)
   of nkIteratorDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    d.add genJsonItem(d, n, n.sons[namePos], skIterator)
+    d.add genJsonItem(d, n, n[namePos], skIterator)
   of nkMacroDef:
-    d.add genJsonItem(d, n, n.sons[namePos], skMacro)
+    d.add genJsonItem(d, n, n[namePos], skMacro)
   of nkTemplateDef:
-    d.add genJsonItem(d, n, n.sons[namePos], skTemplate)
+    d.add genJsonItem(d, n, n[namePos], skTemplate)
   of nkConverterDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    d.add genJsonItem(d, n, n.sons[namePos], skConverter)
+    d.add genJsonItem(d, n, n[namePos], skConverter)
   of nkTypeSection, nkVarSection, nkLetSection, nkConstSection:
-    for i in 0 ..< sonsLen(n):
-      if n.sons[i].kind != nkCommentStmt:
+    for i in 0..<n.len:
+      if n[i].kind != nkCommentStmt:
         # order is always 'type var let const':
-        d.add genJsonItem(d, n.sons[i], n.sons[i].sons[0],
+        d.add genJsonItem(d, n[i], n[i][0],
                 succ(skType, ord(n.kind)-ord(nkTypeSection)))
   of nkStmtList:
-    for i in 0 ..< sonsLen(n):
-      generateJson(d, n.sons[i], includeComments)
+    for i in 0..<n.len:
+      generateJson(d, n[i], includeComments)
   of nkWhenStmt:
     # generate documentation for the first branch only:
-    if not checkForFalse(n.sons[0].sons[0]):
-      generateJson(d, lastSon(n.sons[0]), includeComments)
+    if not checkForFalse(n[0][0]):
+      generateJson(d, lastSon(n[0]), includeComments)
   else: discard
 
 proc genTagsItem(d: PDoc, n, nameNode: PNode, k: TSymKind): string =
@@ -946,36 +1038,36 @@ proc generateTags*(d: PDoc, n: PNode, r: var Rope) =
       r.add stripped
   of nkProcDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    r.add genTagsItem(d, n, n.sons[namePos], skProc)
+    r.add genTagsItem(d, n, n[namePos], skProc)
   of nkFuncDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    r.add genTagsItem(d, n, n.sons[namePos], skFunc)
+    r.add genTagsItem(d, n, n[namePos], skFunc)
   of nkMethodDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    r.add genTagsItem(d, n, n.sons[namePos], skMethod)
+    r.add genTagsItem(d, n, n[namePos], skMethod)
   of nkIteratorDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    r.add genTagsItem(d, n, n.sons[namePos], skIterator)
+    r.add genTagsItem(d, n, n[namePos], skIterator)
   of nkMacroDef:
-    r.add genTagsItem(d, n, n.sons[namePos], skMacro)
+    r.add genTagsItem(d, n, n[namePos], skMacro)
   of nkTemplateDef:
-    r.add genTagsItem(d, n, n.sons[namePos], skTemplate)
+    r.add genTagsItem(d, n, n[namePos], skTemplate)
   of nkConverterDef:
     when useEffectSystem: documentRaises(d.cache, n)
-    r.add genTagsItem(d, n, n.sons[namePos], skConverter)
+    r.add genTagsItem(d, n, n[namePos], skConverter)
   of nkTypeSection, nkVarSection, nkLetSection, nkConstSection:
-    for i in 0 ..< sonsLen(n):
-      if n.sons[i].kind != nkCommentStmt:
+    for i in 0..<n.len:
+      if n[i].kind != nkCommentStmt:
         # order is always 'type var let const':
-        r.add genTagsItem(d, n.sons[i], n.sons[i].sons[0],
+        r.add genTagsItem(d, n[i], n[i][0],
                 succ(skType, ord(n.kind)-ord(nkTypeSection)))
   of nkStmtList:
-    for i in 0 ..< sonsLen(n):
-      generateTags(d, n.sons[i], r)
+    for i in 0..<n.len:
+      generateTags(d, n[i], r)
   of nkWhenStmt:
     # generate documentation for the first branch only:
-    if not checkForFalse(n.sons[0].sons[0]):
-      generateTags(d, lastSon(n.sons[0]), r)
+    if not checkForFalse(n[0][0]):
+      generateTags(d, lastSon(n[0]), r)
   else: discard
 
 proc genSection(d: PDoc, kind: TSymKind) =
@@ -992,6 +1084,12 @@ proc genSection(d: PDoc, kind: TSymKind) =
       "sectionid", "sectionTitle", "sectionTitleID", "content"], [
       ord(kind).rope, title, rope(ord(kind) + 50), d.toc[kind]])
 
+const nimdocOutCss = "nimdoc.out.css"
+  # `out` to make it easier to use with gitignore in user's repos
+
+proc cssHref(outDir: AbsoluteDir, destFile: AbsoluteFile): Rope =
+  rope($relativeTo(outDir / nimdocOutCss.RelativeFile, destFile.splitFile().dir, '/'))
+
 proc genOutFile(d: PDoc): Rope =
   var
     code, content: Rope
@@ -1000,21 +1098,22 @@ proc genOutFile(d: PDoc): Rope =
   var tmp = ""
   renderTocEntries(d[], j, 1, tmp)
   var toc = tmp.rope
-  for i in low(TSymKind) .. high(TSymKind):
+  for i in low(TSymKind)..high(TSymKind):
     genSection(d, i)
-    add(toc, d.toc[i])
+    toc.add(d.toc[i])
   if toc != nil:
     toc = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.toc"), ["content"], [toc])
-  for i in low(TSymKind) .. high(TSymKind): add(code, d.section[i])
+  for i in low(TSymKind)..high(TSymKind): code.add(d.section[i])
 
   # Extract the title. Non API modules generate an entry in the index table.
   if d.meta[metaTitle].len != 0:
     title = d.meta[metaTitle]
-    let external = AbsoluteFile(d.filename).relativeTo(d.conf.projectPath, '/').changeFileExt(HtmlExt).string
+    let external = presentationPath(d.conf, AbsoluteFile d.filename).changeFileExt(HtmlExt).string.nativeToUnix
     setIndexTerm(d[], external, "", title)
   else:
     # Modules get an automatic title for the HTML, but no entry in the index.
-    title = extractFilename(changeFileExt(d.filename, ""))
+    # better than `extractFilename(changeFileExt(d.filename, ""))` as it disambiguates dups
+    title = $presentationPath(d.conf, AbsoluteFile d.filename, isTitle = true).changeFileExt("")
 
   let bodyname = if d.hasToc and not d.isPureRst: "doc.body_toc_group"
                  elif d.hasToc: "doc.body_toc"
@@ -1025,25 +1124,30 @@ proc genOutFile(d: PDoc): Rope =
       rope(getClockStr()), code, d.modDeprecationMsg])
   if optCompileOnly notin d.conf.globalOptions:
     # XXX what is this hack doing here? 'optCompileOnly' means raw output!?
-    code = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.file"), ["title",
-        "tableofcontents", "moduledesc", "date", "time",
+    code = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.file"), [
+        "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
         "content", "author", "version", "analytics", "deprecationMsg"],
-        [title.rope, toc, d.modDesc, rope(getDateStr()),
-                     rope(getClockStr()), content, d.meta[metaAuthor].rope,
-                     d.meta[metaVersion].rope, d.analytics.rope, d.modDeprecationMsg])
+        [cssHref(d.conf.outDir, d.destFile), title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
+        content, d.meta[metaAuthor].rope, d.meta[metaVersion].rope, d.analytics.rope, d.modDeprecationMsg])
   else:
     code = content
   result = code
 
 proc generateIndex*(d: PDoc) =
   if optGenIndex in d.conf.globalOptions:
-    let dir = if not d.conf.outDir.isEmpty: d.conf.outDir
-              else: d.conf.projectPath / RelativeDir"htmldocs"
+    let dir = d.conf.docOutDir(htmldocsDir)
     createDir(dir)
-    let dest = dir / changeFileExt(relativeTo(AbsoluteFile d.filename,
-                                              d.conf.projectPath), IndexExt)
+    let dest = dir / changeFileExt(presentationPath(d.conf, AbsoluteFile d.filename), IndexExt)
     writeIndexFile(d[], dest.string)
 
+proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
+  if d.module == nil or sfMainModule in d.module.flags: # nil for eg for commandRst2Html
+    if d.conf.outDir.isEmpty: d.conf.outDir = d.conf.docOutDir
+    if d.conf.outFile.isEmpty:
+      d.conf.outFile = outfile.relativeTo(d.conf.outDir)
+      if isAbsolute(d.conf.outFile.string):
+        d.conf.outFile = splitPath(d.conf.outFile.string)[1].RelativeFile
+
 proc writeOutput*(d: PDoc, useWarning = false) =
   runAllExamples(d)
   var content = genOutFile(d)
@@ -1051,11 +1155,18 @@ proc writeOutput*(d: PDoc, useWarning = false) =
     writeRope(stdout, content)
   else:
     template outfile: untyped = d.destFile
-    #let outfile = getOutFile2(d.conf, shortenDir(d.conf, filename), outExt, "htmldocs")
-    createDir(outfile.splitFile.dir)
+    #let outfile = getOutFile2(d.conf, shortenDir(d.conf, filename), outExt, htmldocsDir)
+    let dir = outfile.splitFile.dir
+    createDir(dir)
+    updateOutfile(d, outfile)
     if not writeRope(content, outfile):
       rawMessage(d.conf, if useWarning: warnCannotOpenFile else: errCannotOpenFile,
         outfile.string)
+    elif not d.wroteCss:
+      let cssSource = $d.conf.getPrefixDir() / "doc" / "nimdoc.css"
+      let cssDest = $dir / nimdocOutCss
+      copyFile(cssSource, cssDest)
+      d.wroteCss = true
 
 proc writeOutputJson*(d: PDoc, useWarning = false) =
   runAllExamples(d)
@@ -1073,6 +1184,7 @@ proc writeOutputJson*(d: PDoc, useWarning = false) =
     if open(f, d.destFile.string, fmWrite):
       write(f, $content)
       close(f)
+      updateOutfile(d, d.destFile)
     else:
       localError(d.conf, newLineInfo(d.conf, AbsoluteFile d.filename, -1, -1),
                  warnUser, "unable to open file \"" & d.destFile.string &
@@ -1158,12 +1270,17 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) =
 proc commandBuildIndex*(cache: IdentCache, conf: ConfigRef) =
   var content = mergeIndexes(conf.projectFull.string).rope
 
-  let code = ropeFormatNamedVars(conf, getConfigVar(conf, "doc.file"), ["title",
-      "tableofcontents", "moduledesc", "date", "time",
+  var outFile = RelativeFile"theindex"
+  if conf.outFile != RelativeFile"":
+    outFile = conf.outFile
+  let filename = getOutFile(conf, outFile, HtmlExt)
+
+  let code = ropeFormatNamedVars(conf, getConfigVar(conf, "doc.file"), [
+      "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
       "content", "author", "version", "analytics"],
-      ["Index".rope, nil, nil, rope(getDateStr()),
-                   rope(getClockStr()), content, nil, nil, nil])
+      [cssHref(conf.outDir, filename), rope"Index", nil, nil, rope(getDateStr()),
+      rope(getClockStr()), content, nil, nil, nil])
   # no analytics because context is not available
-  let filename = getOutFile(conf, RelativeFile"theindex", HtmlExt)
+
   if not writeRope(code, filename):
     rawMessage(conf, errCannotOpenFile, filename.string)

compiler/docgen2.nim

@@ -11,8 +11,7 @@
 # semantic checking.
 
 import
-  os, options, ast, astalgo, msgs, ropes, idents, passes, docgen, lineinfos,
-  pathutils
+  options, ast, msgs, passes, docgen, lineinfos, pathutils
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -23,8 +22,8 @@ type
     config: ConfigRef
   PGen = ref TGen
 
-template shouldProcess(g): bool =
-  (g.module.owner.id == g.doc.conf.mainPackageId and optWholeProject in g.doc.conf.globalOptions) or
+proc shouldProcess(g: PGen): bool =
+  (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or
       sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
 
 template closeImpl(body: untyped) {.dirty.} =
@@ -63,7 +62,7 @@ template myOpenImpl(ext: untyped) {.dirty.} =
   g.module = module
   g.config = graph.config
   var d = newDocumentor(AbsoluteFile toFullPath(graph.config, FileIndex module.position),
-      graph.cache, graph.config, ext)
+      graph.cache, graph.config, ext, module)
   d.hasToc = true
   g.doc = d
   result = g

compiler/enumtostr.nim

@@ -13,7 +13,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
   result.typ = newType(tyProc, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
-  addSon(result.typ.n, newNodeI(nkEffectList, info))
+  result.typ.n.add newNodeI(nkEffectList, info)
 
   result.typ.addParam dest
 
@@ -22,7 +22,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
   caseStmt.add(newSymNode dest)
 
   # copy the branches over, but replace the fields with the for loop body:
-  for i in 0 ..< t.n.len:
+  for i in 0..<t.n.len:
     assert(t.n[i].kind == nkSym)
     var field = t.n[i].sym
     let val = if field.ast == nil: field.name.s else: field.ast.strVal
@@ -33,28 +33,35 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
   body.add(caseStmt)
 
   var n = newNodeI(nkProcDef, info, bodyPos+2)
-  for i in 0 ..< n.len: n.sons[i] = newNodeI(nkEmpty, info)
-  n.sons[namePos] = newSymNode(result)
-  n.sons[paramsPos] = result.typ.n
-  n.sons[bodyPos] = body
-  n.sons[resultPos] = newSymNode(res)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = body
+  n[resultPos] = newSymNode(res)
   result.ast = n
   incl result.flags, sfFromGeneric
+  incl result.flags, sfNeverRaises
 
-proc searchObjCase(obj: PNode; field: PSym): PNode =
+proc searchObjCaseImpl(obj: PNode; field: PSym): PNode =
   case obj.kind
   of nkSym:
     result = nil
   of nkElse, nkOfBranch:
-    result = searchObjCase(obj.lastSon, field)
+    result = searchObjCaseImpl(obj.lastSon, field)
   else:
     if obj.kind == nkRecCase and obj[0].kind == nkSym and obj[0].sym == field:
       result = obj
     else:
       for x in obj:
-        result = searchObjCase(x, field)
+        result = searchObjCaseImpl(x, field)
         if result != nil: break
 
+proc searchObjCase(t: PType; field: PSym): PNode =
+  result = searchObjCaseImpl(t.n, field)
+  if result == nil and t.len > 0:
+    result = searchObjCase(t[0].skipTypes({tyAlias, tyGenericInst, tyRef, tyPtr}), field)
+  doAssert result != nil
+
 proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph): PSym =
   result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), t.owner, info)
 
@@ -67,7 +74,7 @@ proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGra
   result.typ = newType(tyProc, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
-  addSon(result.typ.n, newNodeI(nkEffectList, info))
+  result.typ.n.add newNodeI(nkEffectList, info)
 
   result.typ.addParam dest
 
@@ -75,12 +82,11 @@ proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGra
   var caseStmt = newNodeI(nkCaseStmt, info)
   caseStmt.add(newSymNode dest)
 
-  let subObj = searchObjCase(t.n, field)
-  doAssert subObj != nil
-  for i in 1 ..< subObj.len:
+  let subObj = searchObjCase(t, field)
+  for i in 1..<subObj.len:
     let ofBranch = subObj[i]
     var newBranch = newNodeI(ofBranch.kind, ofBranch.info)
-    for j in 0..ofBranch.len-2:
+    for j in 0..<ofBranch.len-1:
       newBranch.add ofBranch[j]
 
     newBranch.add newTree(nkStmtList, newTree(nkFastAsgn, newSymNode(res), newIntNode(nkInt8Lit, i)))
@@ -89,10 +95,11 @@ proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGra
   body.add(caseStmt)
 
   var n = newNodeI(nkProcDef, info, bodyPos+2)
-  for i in 0 ..< n.len: n.sons[i] = newNodeI(nkEmpty, info)
-  n.sons[namePos] = newSymNode(result)
-  n.sons[paramsPos] = result.typ.n
-  n.sons[bodyPos] = body
-  n.sons[resultPos] = newSymNode(res)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = body
+  n[resultPos] = newSymNode(res)
   result.ast = n
   incl result.flags, sfFromGeneric
+  incl result.flags, sfNeverRaises

compiler/evalffi.nim

@@ -9,13 +9,16 @@
 
 ## This file implements the FFI part of the evaluator for Nim code.
 
-import ast, astalgo, ropes, types, options, tables, dynlib, msgs, os, lineinfos
+import ast, types, options, tables, dynlib, msgs, lineinfos
+from os import getAppFilename
 import pkg/libffi
 
 when defined(windows):
   const libcDll = "msvcrt.dll"
 elif defined(linux):
   const libcDll = "libc.so(.6|.5|)"
+elif defined(bsd):
+  const libcDll = "/lib/libc.so.7"
 elif defined(osx):
   const libcDll = "/usr/lib/libSystem.dylib"
 else:
@@ -27,7 +30,7 @@ var
   gDllCache = initTable[string, LibHandle]()
 
 when defined(windows):
-  var gExeHandle = loadLib(os.getAppFilename())
+  var gExeHandle = loadLib(getAppFilename())
 else:
   var gExeHandle = loadLib()
 
@@ -46,38 +49,35 @@ proc getDll(conf: ConfigRef, cache: var TDllCache; dll: string; info: TLineInfo)
 const
   nkPtrLit = nkIntLit # hopefully we can get rid of this hack soon
 
-var myerrno {.importc: "errno", header: "<errno.h>".}: cint ## error variable
-
 proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode =
-  let name = $sym.loc.r
+  let name = sym.cname # $sym.loc.r would point to internal name
   # the AST does not support untyped pointers directly, so we use an nkIntLit
   # that contains the address instead:
   result = newNodeIT(nkPtrLit, sym.info, sym.typ)
-  case name
-  of "stdin":  result.intVal = cast[ByteAddress](system.stdin)
-  of "stdout": result.intVal = cast[ByteAddress](system.stdout)
-  of "stderr": result.intVal = cast[ByteAddress](system.stderr)
-  of "vmErrnoWrapper": result.intVal = cast[ByteAddress](myerrno)
-  else:
+  when true:
+    var libPathMsg = ""
     let lib = sym.annex
     if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}:
       globalError(conf, sym.info, "dynlib needs to be a string lit")
     var theAddr: pointer
-    if (lib.isNil or lib.kind == libHeader) and not gExehandle.isNil:
+    if (lib.isNil or lib.kind == libHeader) and not gExeHandle.isNil:
+      libPathMsg = "current exe: " & getAppFilename() & " nor libc: " & libcDll
       # first try this exe itself:
-      theAddr = gExehandle.symAddr(name)
+      theAddr = gExeHandle.symAddr(name)
       # then try libc:
       if theAddr.isNil:
         let dllhandle = getDll(conf, gDllCache, libcDll, sym.info)
         theAddr = dllhandle.symAddr(name)
     elif not lib.isNil:
       let dll = if lib.kind == libHeader: libcDll else: lib.path.strVal
+      libPathMsg = dll
       let dllhandle = getDll(conf, gDllCache, dll, sym.info)
       theAddr = dllhandle.symAddr(name)
-    if theAddr.isNil: globalError(conf, sym.info, "cannot import: " & sym.name.s)
+    if theAddr.isNil: globalError(conf, sym.info,
+      "cannot import symbol: " & name & " from " & libPathMsg)
     result.intVal = cast[ByteAddress](theAddr)
 
-proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType =
+proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type =
   if t == nil: return addr libffi.type_void
 
   case t.kind
@@ -94,7 +94,7 @@ proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType =
      tyTyped, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
     result = addr libffi.type_pointer
   of tyDistinct, tyAlias, tySink:
-    result = mapType(conf, t.sons[0])
+    result = mapType(conf, t[0])
   else:
     result = nil
   # too risky:
@@ -120,16 +120,16 @@ proc packSize(conf: ConfigRef, v: PNode, typ: PType): int =
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     else:
-      result = sizeof(pointer) + packSize(conf, v.sons[0], typ.lastSon)
+      result = sizeof(pointer) + packSize(conf, v[0], typ.lastSon)
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    result = packSize(conf, v, typ.sons[0])
+    result = packSize(conf, v, typ[0])
   of tyArray:
     # consider: ptr array[0..1000_000, int] which is common for interfacing;
     # we use the real length here instead
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     elif v.len != 0:
-      result = v.len * packSize(conf, v.sons[0], typ.sons[1])
+      result = v.len * packSize(conf, v[0], typ[1])
   else:
     result = getSize(conf, typ).int
 
@@ -138,16 +138,16 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer)
 proc getField(conf: ConfigRef, n: PNode; position: int): PSym =
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      result = getField(conf, n.sons[i], position)
+    for i in 0..<n.len:
+      result = getField(conf, n[i], position)
       if result != nil: return
   of nkRecCase:
-    result = getField(conf, n.sons[0], position)
+    result = getField(conf, n[0], position)
     if result != nil: return
-    for i in 1 ..< sonsLen(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = getField(conf, lastSon(n.sons[i]), position)
+        result = getField(conf, lastSon(n[i]), position)
         if result != nil: return
       else: internalError(conf, n.info, "getField(record case branch)")
   of nkSym:
@@ -158,12 +158,12 @@ proc packObject(conf: ConfigRef, x: PNode, typ: PType, res: pointer) =
   internalAssert conf, x.kind in {nkObjConstr, nkPar, nkTupleConstr}
   # compute the field's offsets:
   discard getSize(conf, typ)
-  for i in ord(x.kind == nkObjConstr) ..< sonsLen(x):
-    var it = x.sons[i]
+  for i in ord(x.kind == nkObjConstr)..<x.len:
+    var it = x[i]
     if it.kind == nkExprColonExpr:
-      internalAssert conf, it.sons[0].kind == nkSym
-      let field = it.sons[0].sym
-      pack(conf, it.sons[1], field.typ, res +! field.offset)
+      internalAssert conf, it[0].kind == nkSym
+      let field = it[0].sym
+      pack(conf, it[1], field.typ, res +! field.offset)
     elif typ.n != nil:
       let field = getField(conf, typ.n, i)
       pack(conf, it, field.typ, res +! field.offset)
@@ -224,19 +224,19 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
         packRecCheck = 0
         globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ))
       inc packRecCheck
-      pack(conf, v.sons[0], typ.lastSon, res +! sizeof(pointer))
+      pack(conf, v[0], typ.lastSon, res +! sizeof(pointer))
       dec packRecCheck
       awr(pointer, res +! sizeof(pointer))
   of tyArray:
-    let baseSize = getSize(conf, typ.sons[1])
-    for i in 0 ..< v.len:
-      pack(conf, v.sons[i], typ.sons[1], res +! i * baseSize)
+    let baseSize = getSize(conf, typ[1])
+    for i in 0..<v.len:
+      pack(conf, v[i], typ[1], res +! i * baseSize)
   of tyObject, tyTuple:
     packObject(conf, v, typ, res)
   of tyNil:
     discard
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    pack(conf, v, typ.sons[0], res)
+    pack(conf, v, typ[0], res)
   else:
     globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ))
 
@@ -245,14 +245,14 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode
 proc unpackObjectAdd(conf: ConfigRef, x: pointer, n, result: PNode) =
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      unpackObjectAdd(conf, x, n.sons[i], result)
+    for i in 0..<n.len:
+      unpackObjectAdd(conf, x, n[i], result)
   of nkRecCase:
     globalError(conf, result.info, "case objects cannot be unpacked")
   of nkSym:
     var pair = newNodeI(nkExprColonExpr, result.info, 2)
-    pair.sons[0] = n
-    pair.sons[1] = unpack(conf, x +! n.sym.offset, n.sym.typ, nil)
+    pair[0] = n
+    pair[1] = unpack(conf, x +! n.sym.offset, n.sym.typ, nil)
     #echo "offset: ", n.sym.name.s, " ", n.sym.offset
     result.add pair
   else: discard
@@ -275,28 +275,28 @@ proc unpackObject(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
       globalError(conf, n.info, "cannot map value from FFI")
     if typ.n.isNil:
       globalError(conf, n.info, "cannot unpack unnamed tuple")
-    for i in ord(n.kind == nkObjConstr) ..< sonsLen(n):
-      var it = n.sons[i]
+    for i in ord(n.kind == nkObjConstr)..<n.len:
+      var it = n[i]
       if it.kind == nkExprColonExpr:
-        internalAssert conf, it.sons[0].kind == nkSym
-        let field = it.sons[0].sym
-        it.sons[1] = unpack(conf, x +! field.offset, field.typ, it.sons[1])
+        internalAssert conf, it[0].kind == nkSym
+        let field = it[0].sym
+        it[1] = unpack(conf, x +! field.offset, field.typ, it[1])
       else:
         let field = getField(conf, typ.n, i)
-        n.sons[i] = unpack(conf, x +! field.offset, field.typ, it)
+        n[i] = unpack(conf, x +! field.offset, field.typ, it)
 
 proc unpackArray(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   if n.isNil:
     result = newNode(nkBracket)
     result.typ = typ
-    newSeq(result.sons, lengthOrd(conf, typ).int)
+    newSeq(result.sons, lengthOrd(conf, typ).toInt)
   else:
     result = n
     if result.kind != nkBracket:
       globalError(conf, n.info, "cannot map value from FFI")
-  let baseSize = getSize(conf, typ.sons[1])
-  for i in 0 ..< result.len:
-    result.sons[i] = unpack(conf, x +! i * baseSize, typ.sons[1], result.sons[i])
+  let baseSize = getSize(conf, typ[1])
+  for i in 0..<result.len:
+    result[i] = unpack(conf, x +! i * baseSize, typ[1], result[i])
 
 proc canonNodeKind(k: TNodeKind): TNodeKind =
   case k
@@ -327,7 +327,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
     else:
       reset n[]
       result = n
-      result.kind = nkNilLit
+      result[] = TNode(kind: nkNilLit)
       result.typ = typ
 
   template awi(kind, v: untyped): untyped = aw(kind, v, intVal)
@@ -376,7 +376,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
       awi(nkPtrLit, cast[ByteAddress](p))
     elif n != nil and n.len == 1:
       internalAssert(conf, n.kind == nkRefTy)
-      n.sons[0] = unpack(conf, p, typ.lastSon, n.sons[0])
+      n[0] = unpack(conf, p, typ.lastSon, n[0])
       result = n
     else:
       globalError(conf, n.info, "cannot map value from FFI " & typeToString(typ))
@@ -419,41 +419,41 @@ proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode =
     dealloc a
 
 proc callForeignFunction*(conf: ConfigRef, call: PNode): PNode =
-  internalAssert conf, call.sons[0].kind == nkPtrLit
+  internalAssert conf, call[0].kind == nkPtrLit
 
   var cif: TCif
-  var sig: TParamList
+  var sig: ParamList
   # use the arguments' types for varargs support:
-  for i in 1..call.len-1:
-    sig[i-1] = mapType(conf, call.sons[i].typ)
+  for i in 1..<call.len:
+    sig[i-1] = mapType(conf, call[i].typ)
     if sig[i-1].isNil:
       globalError(conf, call.info, "cannot map FFI type")
 
-  let typ = call.sons[0].typ
+  let typ = call[0].typ
   if prep_cif(cif, mapCallConv(conf, typ.callConv, call.info), cuint(call.len-1),
-              mapType(conf, typ.sons[0]), sig) != OK:
+              mapType(conf, typ[0]), sig) != OK:
     globalError(conf, call.info, "error in FFI call")
 
-  var args: TArgList
-  let fn = cast[pointer](call.sons[0].intVal)
-  for i in 1 .. call.len-1:
-    var t = call.sons[i].typ
-    args[i-1] = alloc0(packSize(conf, call.sons[i], t))
-    pack(conf, call.sons[i], t, args[i-1])
-  let retVal = if isEmptyType(typ.sons[0]): pointer(nil)
-               else: alloc(getSize(conf, typ.sons[0]).int)
+  var args: ArgList
+  let fn = cast[pointer](call[0].intVal)
+  for i in 1..<call.len:
+    var t = call[i].typ
+    args[i-1] = alloc0(packSize(conf, call[i], t))
+    pack(conf, call[i], t, args[i-1])
+  let retVal = if isEmptyType(typ[0]): pointer(nil)
+               else: alloc(getSize(conf, typ[0]).int)
 
   libffi.call(cif, fn, retVal, args)
 
   if retVal.isNil:
     result = newNode(nkEmpty)
   else:
-    result = unpack(conf, retVal, typ.sons[0], nil)
+    result = unpack(conf, retVal, typ[0], nil)
     result.info = call.info
 
   if retVal != nil: dealloc retVal
-  for i in 1 .. call.len-1:
-    call.sons[i] = unpack(conf, args[i-1], typ.sons[i], call[i])
+  for i in 1..<call.len:
+    call[i] = unpack(conf, args[i-1], typ[i], call[i])
     dealloc args[i-1]
 
 proc callForeignFunction*(conf: ConfigRef, fn: PNode, fntyp: PType,
@@ -462,39 +462,39 @@ proc callForeignFunction*(conf: ConfigRef, fn: PNode, fntyp: PType,
   internalAssert conf, fn.kind == nkPtrLit
 
   var cif: TCif
-  var sig: TParamList
+  var sig: ParamList
   for i in 0..len-1:
     var aTyp = args[i+start].typ
     if aTyp.isNil:
       internalAssert conf, i+1 < fntyp.len
-      aTyp = fntyp.sons[i+1]
+      aTyp = fntyp[i+1]
       args[i+start].typ = aTyp
     sig[i] = mapType(conf, aTyp)
     if sig[i].isNil: globalError(conf, info, "cannot map FFI type")
 
   if prep_cif(cif, mapCallConv(conf, fntyp.callConv, info), cuint(len),
-              mapType(conf, fntyp.sons[0]), sig) != OK:
+              mapType(conf, fntyp[0]), sig) != OK:
     globalError(conf, info, "error in FFI call")
 
-  var cargs: TArgList
+  var cargs: ArgList
   let fn = cast[pointer](fn.intVal)
-  for i in 0 .. len-1:
+  for i in 0..len-1:
     let t = args[i+start].typ
     cargs[i] = alloc0(packSize(conf, args[i+start], t))
     pack(conf, args[i+start], t, cargs[i])
-  let retVal = if isEmptyType(fntyp.sons[0]): pointer(nil)
-               else: alloc(getSize(conf, fntyp.sons[0]).int)
+  let retVal = if isEmptyType(fntyp[0]): pointer(nil)
+               else: alloc(getSize(conf, fntyp[0]).int)
 
   libffi.call(cif, fn, retVal, cargs)
 
   if retVal.isNil:
     result = newNode(nkEmpty)
   else:
-    result = unpack(conf, retVal, fntyp.sons[0], nil)
+    result = unpack(conf, retVal, fntyp[0], nil)
     result.info = info
 
   if retVal != nil: dealloc retVal
-  for i in 0 .. len-1:
+  for i in 0..len-1:
     let t = args[i+start].typ
     args[i+start] = unpack(conf, cargs[i], t, args[i+start])
     dealloc cargs[i]

compiler/evaltempl.nim

@@ -10,8 +10,7 @@
 ## Template evaluation engine. Now hygienic.
 
 import
-  strutils, options, ast, astalgo, msgs, os, idents, wordrecg, renderer,
-  lineinfos
+  strutils, options, ast, astalgo, msgs, renderer, lineinfos, idents
 
 type
   TemplCtx = object
@@ -21,6 +20,7 @@ type
     mapping: TIdTable # every gensym'ed symbol needs to be mapped to some
                       # new symbol
     config: ConfigRef
+    ic: IdentCache
 
 proc copyNode(ctx: TemplCtx, a, b: PNode): PNode =
   result = copyNode(a)
@@ -38,11 +38,11 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
   of nkSym:
     var s = templ.sym
     if (s.owner == nil and s.kind == skParam) or s.owner == c.owner:
-      if s.kind == skParam and sfGenSym notin s.flags:
-        handleParam actual.sons[s.position]
+      if s.kind == skParam and {sfGenSym, sfTemplateParam} * s.flags == {sfTemplateParam}:
+        handleParam actual[s.position]
       elif (s.owner != nil) and (s.kind == skGenericParam or
            s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam):
-        handleParam actual.sons[s.owner.typ.len + s.position - 1]
+        handleParam actual[s.owner.typ.len + s.position - 1]
       else:
         internalAssert c.config, sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
@@ -53,7 +53,11 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
           #if x.kind == skParam and x.owner.kind == skModule:
           #  internalAssert c.config, false
           idTablePut(c.mapping, s, x)
-        result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
+        if sfGenSym in s.flags and optNimV019 notin c.config.globalOptions:
+          result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $x.id),
+            if c.instLines: actual.info else: templ.info)
+        else:
+          result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
     else:
       result.add copyNode(c, templ, actual)
   of nkNone..nkIdent, nkType..nkNilLit: # atom
@@ -64,8 +68,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
     # "declarative" context (bug #9235).
     if c.isDeclarative:
       var res = copyNode(c, templ, actual)
-      for i in 0 ..< sonsLen(templ):
-        evalTemplateAux(templ.sons[i], actual, c, res)
+      for i in 0..<templ.len:
+        evalTemplateAux(templ[i], actual, c, res)
       result.add res
     else:
       result.add newNodeI(nkEmpty, templ.info)
@@ -78,8 +82,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
       c.isDeclarative = true
       isDeclarative = true
     var res = copyNode(c, templ, actual)
-    for i in 0 ..< sonsLen(templ):
-      evalTemplateAux(templ.sons[i], actual, c, res)
+    for i in 0..<templ.len:
+      evalTemplateAux(templ[i], actual, c, res)
     result.add res
     if isDeclarative: c.isDeclarative = false
 
@@ -118,22 +122,22 @@ proc evalTemplateArgs(n: PNode, s: PSym; conf: ConfigRef; fromHlo: bool): PNode
                 n.renderTree)
 
   result = newNodeI(nkArgList, n.info)
-  for i in 1 .. givenRegularParams:
-    result.addSon n[i]
+  for i in 1..givenRegularParams:
+    result.add n[i]
 
   # handle parameters with default values, which were
   # not supplied by the user
-  for i in givenRegularParams+1 .. expectedRegularParams:
-    let default = s.typ.n.sons[i].sym.ast
+  for i in givenRegularParams+1..expectedRegularParams:
+    let default = s.typ.n[i].sym.ast
     if default.isNil or default.kind == nkEmpty:
       localError(conf, n.info, errWrongNumberOfArguments)
-      addSon(result, newNodeI(nkEmpty, n.info))
+      result.add newNodeI(nkEmpty, n.info)
     else:
-      addSon(result, default.copyTree)
+      result.add default.copyTree
 
-  # add any generic paramaters
-  for i in 1 .. genericParams:
-    result.addSon n.sons[givenRegularParams + i]
+  # add any generic parameters
+  for i in 1..genericParams:
+    result.add n[givenRegularParams + i]
 
 # to prevent endless recursion in template instantiation
 const evalTemplateLimit* = 1000
@@ -151,7 +155,7 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
       if x.kind in nkCallKinds:
         for i in 1..<x.len:
           if x[i].kind in nkCallKinds:
-            x.sons[i].info = info
+            x[i].info = info
   else:
     result = newNodeI(nkStmtListExpr, info)
     var d = newNodeI(nkComesFrom, info)
@@ -161,7 +165,9 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
     result.typ = res.typ
 
 proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
-                   conf: ConfigRef; fromHlo=false): PNode =
+                   conf: ConfigRef;
+                   ic: IdentCache;
+                   fromHlo=false): PNode =
   inc(conf.evalTemplateCounter)
   if conf.evalTemplateCounter > evalTemplateLimit:
     globalError(conf, n.info, errTemplateInstantiationTooNested)
@@ -173,6 +179,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   ctx.owner = tmpl
   ctx.genSymOwner = genSymOwner
   ctx.config = conf
+  ctx.ic = ic
   initIdTable(ctx.mapping)
 
   let body = tmpl.getBody
@@ -180,7 +187,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   if isAtom(body):
     result = newNodeI(nkPar, body.info)
     evalTemplateAux(body, args, ctx, result)
-    if result.len == 1: result = result.sons[0]
+    if result.len == 1: result = result[0]
     else:
       localError(conf, result.info, "illformed AST: " &
                   renderTree(result, {renderNoComments}))
@@ -189,8 +196,8 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
     ctx.instLines = sfCallsite in tmpl.flags
     if ctx.instLines:
       result.info = n.info
-    for i in 0 ..< safeLen(body):
-      evalTemplateAux(body.sons[i], args, ctx, result)
+    for i in 0..<body.safeLen:
+      evalTemplateAux(body[i], args, ctx, result)
   result.flags.incl nfFromTemplate
   result = wrapInComesFrom(n.info, tmpl, result)
   #if ctx.debugActive:

compiler/extccomp.nim

@@ -14,7 +14,7 @@
 
 import
   ropes, os, strutils, osproc, platform, condsyms, options, msgs,
-  lineinfos, std / sha1, streams, pathutils, sequtils, times
+  lineinfos, std / sha1, streams, pathutils, sequtils, times, strtabs
 
 type
   TInfoCCProp* = enum         # properties of the C compiler:
@@ -48,6 +48,8 @@ type
                          # used on some platforms
     asmStmtFrmt: string, # format of ASM statement
     structStmtFmt: string, # Format for struct statement
+    produceAsm: string,  # Format how to produce assembler listings
+    cppXsupport: string, # what to do to enable C++X support
     props: TInfoCCProps] # properties of the C compiler
 
 
@@ -58,13 +60,16 @@ type
 template compiler(name, settings: untyped): untyped =
   proc name: TInfoCC {.compileTime.} = settings
 
+const
+  gnuAsmListing = "-Wa,-acdl=$asmfile -g -fverbose-asm -masm=intel"
+
 # GNU C and C++ Compiler
 compiler gcc:
   result = (
     name: "gcc",
     objExt: "o",
-    optSpeed: " -O3 ",
-    optSize: " -Os ",
+    optSpeed: " -O3 -fno-ident",
+    optSize: " -Os -fno-ident",
     compilerExe: "gcc",
     cppCompiler: "g++",
     compileTmpl: "-c $options $include -o $objfile $file",
@@ -80,6 +85,8 @@ compiler gcc:
     pic: "-fPIC",
     asmStmtFrmt: "asm($1);$n",
     structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name
+    produceAsm: gnuAsmListing,
+    cppXsupport: "-std=gnu++14 -funsigned-char",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm,
             hasAttribute})
 
@@ -105,6 +112,8 @@ compiler nintendoSwitchGCC:
     pic: "-fPIE",
     asmStmtFrmt: "asm($1);$n",
     structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name
+    produceAsm: gnuAsmListing,
+    cppXsupport: "-std=gnu++14 -funsigned-char",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm,
             hasAttribute})
 
@@ -138,12 +147,12 @@ compiler vcc:
     optSize: " /O1 /G7 ",
     compilerExe: "cl",
     cppCompiler: "cl",
-    compileTmpl: "/c$vccplatform$options $include /Fo$objfile $file",
+    compileTmpl: "/c$vccplatform $options $include /Fo$objfile $file",
     buildGui: " /link /SUBSYSTEM:WINDOWS ",
     buildDll: " /LD",
     buildLib: "lib /OUT:$libfile $objfiles",
     linkerExe: "cl",
-    linkTmpl: "$options $builddll$vccplatform /Fe$exefile $objfiles $buildgui",
+    linkTmpl: "$builddll$vccplatform /Fe$exefile $objfiles $buildgui $options",
     includeCmd: " /I",
     linkDirCmd: " /LIBPATH:",
     linkLibCmd: " $1.lib",
@@ -151,6 +160,8 @@ compiler vcc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$3$n$1 $2",
+    produceAsm: "/Fa$asmfile",
+    cppXsupport: "",
     props: {hasCpp, hasAssume, hasDeclspec})
 
 compiler clangcl:
@@ -196,6 +207,8 @@ compiler lcc:
     pic: "",
     asmStmtFrmt: "_asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
+    produceAsm: "",
+    cppXsupport: "",
     props: {})
 
 # Borland C Compiler
@@ -220,6 +233,8 @@ compiler bcc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
+    produceAsm: "",
+    cppXsupport: "",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard,
             hasAttribute})
 
@@ -245,6 +260,8 @@ compiler dmc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$3$n$1 $2",
+    produceAsm: "",
+    cppXsupport: "",
     props: {hasCpp})
 
 # Watcom C Compiler
@@ -269,6 +286,8 @@ compiler wcc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
+    produceAsm: "",
+    cppXsupport: "",
     props: {hasCpp})
 
 # Tiny C Compiler
@@ -291,9 +310,11 @@ compiler tcc:
     linkLibCmd: "", # XXX: not supported yet
     debug: " -g ",
     pic: "",
-    asmStmtFrmt: "__asm{$n$1$n}$n",
+    asmStmtFrmt: "asm($1);$n",
     structStmtFmt: "$1 $2",
-    props: {hasSwitchRange, hasComputedGoto})
+    produceAsm: gnuAsmListing,
+    cppXsupport: "",
+    props: {hasSwitchRange, hasComputedGoto, hasGnuAsm})
 
 # Pelles C Compiler
 compiler pcc:
@@ -318,6 +339,8 @@ compiler pcc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
+    produceAsm: "",
+    cppXsupport: "",
     props: {})
 
 # Your C Compiler
@@ -342,6 +365,8 @@ compiler ucc:
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$1 $2",
+    produceAsm: "",
+    cppXsupport: "",
     props: {})
 
 const
@@ -370,13 +395,15 @@ proc libNameTmpl(conf: ConfigRef): string {.inline.} =
 proc nameToCC*(name: string): TSystemCC =
   ## Returns the kind of compiler referred to by `name`, or ccNone
   ## if the name doesn't refer to any known compiler.
-  for i in succ(ccNone) .. high(TSystemCC):
+  for i in succ(ccNone)..high(TSystemCC):
     if cmpIgnoreStyle(name, CC[i].name) == 0:
       return i
   result = ccNone
 
-proc listCCnames(): seq[string] =
-  for i in succ(ccNone) .. high(TSystemCC):
+proc listCCnames(): string =
+  result = ""
+  for i in succ(ccNone)..high(TSystemCC):
+    if i > succ(ccNone): result.add ", "
     result.add CC[i].name
 
 proc isVSCompatible*(conf: ConfigRef): bool =
@@ -404,7 +431,7 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
                      CC[c].name & fullSuffix
     result = getConfigVar(conf, fullCCname)
     if result.len == 0:
-      # not overriden for this cross compilation setting?
+      # not overridden for this cross compilation setting?
       result = getConfigVar(conf, CC[c].name & fullSuffix)
   else:
     result = getConfigVar(conf, CC[c].name & fullSuffix)
@@ -412,17 +439,16 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
 proc setCC*(conf: ConfigRef; ccname: string; info: TLineInfo) =
   conf.cCompiler = nameToCC(ccname)
   if conf.cCompiler == ccNone:
-    let ccList = listCCnames().join(", ")
-    localError(conf, info, "unknown C compiler: '$1'. Available options are: $2" % [ccname, ccList])
+    localError(conf, info, "unknown C compiler: '$1'. Available options are: $2" % [ccname, listCCnames()])
   conf.compileOptions = getConfigVar(conf, conf.cCompiler, ".options.always")
   conf.linkOptions = ""
-  conf.ccompilerpath = getConfigVar(conf, conf.cCompiler, ".path")
-  for i in low(CC) .. high(CC): undefSymbol(conf.symbols, CC[i].name)
+  conf.cCompilerPath = getConfigVar(conf, conf.cCompiler, ".path")
+  for i in low(CC)..high(CC): undefSymbol(conf.symbols, CC[i].name)
   defineSymbol(conf.symbols, CC[conf.cCompiler].name)
 
 proc addOpt(dest: var string, src: string) =
-  if len(dest) == 0 or dest[len(dest)-1] != ' ': add(dest, " ")
-  add(dest, src)
+  if dest.len == 0 or dest[^1] != ' ': dest.add(" ")
+  dest.add(src)
 
 proc addLinkOption*(conf: ConfigRef; option: string) =
   addOpt(conf.linkOptions, option)
@@ -439,14 +465,14 @@ proc addCompileOptionCmd*(conf: ConfigRef; option: string) =
 
 proc initVars*(conf: ConfigRef) =
   # we need to define the symbol here, because ``CC`` may have never been set!
-  for i in low(CC) .. high(CC): undefSymbol(conf.symbols, CC[i].name)
+  for i in low(CC)..high(CC): undefSymbol(conf.symbols, CC[i].name)
   defineSymbol(conf.symbols, CC[conf.cCompiler].name)
   addCompileOption(conf, getConfigVar(conf, conf.cCompiler, ".options.always"))
   #addLinkOption(getConfigVar(cCompiler, ".options.linker"))
-  if len(conf.ccompilerpath) == 0:
-    conf.ccompilerpath = getConfigVar(conf, conf.cCompiler, ".path")
+  if conf.cCompilerPath.len == 0:
+    conf.cCompilerPath = getConfigVar(conf, conf.cCompiler, ".path")
 
-proc completeCFilePath*(conf: ConfigRef; cfile: AbsoluteFile,
+proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile,
                         createSubDir: bool = true): AbsoluteFile =
   result = completeGeneratedFilePath(conf, cfile, createSubDir)
 
@@ -457,6 +483,13 @@ proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile =
 proc addFileToCompile*(conf: ConfigRef; cf: Cfile) =
   conf.toCompile.add(cf)
 
+proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) =
+  let key = completeCfilePath(conf, withPackageName(conf, nimfile)).string
+  var value = conf.cfileSpecificOptions.getOrDefault(key)
+  if strutils.find(value, option, 0) < 0:
+    addOpt(value, option)
+    conf.cfileSpecificOptions[key] = value
+
 proc resetCompilationLists*(conf: ConfigRef) =
   conf.toCompile.setLen 0
   ## XXX: we must associate these with their originating module
@@ -469,7 +502,7 @@ proc addExternalFileToLink*(conf: ConfigRef; filename: AbsoluteFile) =
   conf.externalToLink.insert(filename.string, 0)
 
 proc execWithEcho(conf: ConfigRef; cmd: string, msg = hintExecuting): int =
-  rawMessage(conf, msg, cmd)
+  rawMessage(conf, msg, if msg == hintLinking and not(optListCmd in conf.globalOptions or conf.verbosity > 1): "" else: cmd)
   result = execCmd(cmd)
 
 proc execExternalProgram*(conf: ConfigRef; cmd: string, msg = hintExecuting) =
@@ -481,10 +514,7 @@ proc generateScript(conf: ConfigRef; script: Rope) =
   let (_, name, _) = splitFile(conf.outFile.string)
   let filename = getNimcacheDir(conf) / RelativeFile(addFileExt("compile_" & name,
                                      platform.OS[conf.target.targetOS].scriptExt))
-  if writeRope(script, filename):
-    copyFile(conf.libpath / RelativeFile"nimbase.h",
-             getNimcacheDir(conf) / RelativeFile"nimbase.h")
-  else:
+  if not writeRope(script, filename):
     rawMessage(conf, errGenerated, "could not write to file: " & filename.string)
 
 proc getOptSpeed(conf: ConfigRef; c: TSystemCC): string =
@@ -509,8 +539,10 @@ proc noAbsolutePaths(conf: ConfigRef): bool {.inline.} =
   # `optGenMapping` is included here for niminst.
   result = conf.globalOptions * {optGenScript, optGenMapping} != {}
 
-proc cFileSpecificOptions(conf: ConfigRef; nimname: string): string =
+proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string =
   result = conf.compileOptions
+  addOpt(result, conf.cfileSpecificOptions.getOrDefault(fullNimFile))
+
   for option in conf.compileOptionsCmd:
     if strutils.find(result, option, 0) < 0:
       addOpt(result, option)
@@ -531,20 +563,19 @@ proc cFileSpecificOptions(conf: ConfigRef; nimname: string): string =
   if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key))
 
 proc getCompileOptions(conf: ConfigRef): string =
-  result = cFileSpecificOptions(conf, "__dummy__")
+  result = cFileSpecificOptions(conf, "__dummy__", "__dummy__")
 
 proc vccplatform(conf: ConfigRef): string =
   # VCC specific but preferable over the config hacks people
   # had to do before, see #11306
-  case conf.target.targetCPU
-  of cpuI386:
-    result = " --platform:x86"
-  of cpuArm:
-    result = " --platform:arm"
-  of cpuAmd64:
-    result = " --platform:amd64"
-  else:
-    result = ""
+  if conf.cCompiler == ccVcc:
+    let exe = getConfigVar(conf, conf.cCompiler, ".exe")
+    if "vccexe.exe" == extractFilename(exe):
+      result = case conf.target.targetCPU
+        of cpuI386: " --platform:x86"
+        of cpuArm: " --platform:arm"
+        of cpuAmd64: " --platform:amd64"
+        else: ""
 
 proc getLinkOptions(conf: ConfigRef): string =
   result = conf.linkOptions & " " & conf.linkOptionsCmd & " "
@@ -557,8 +588,11 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} =
   result = (optGenScript in conf.globalOptions and conf.target.targetOS == osWindows) or
            (conf.target.hostOS == osWindows)
 
+proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool =
+  conf.cmd == cmdCompileToCpp and not cfile.string.endsWith(".c")
+
 proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string =
-  result = if conf.cmd == cmdCompileToCpp and not cfile.string.endsWith(".c"):
+  result = if useCpp(conf, cfile):
              CC[compiler].cppCompiler
            else:
              CC[compiler].compilerExe
@@ -567,33 +601,65 @@ proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile):
       "Compiler '$1' doesn't support the requested target" %
       CC[compiler].name)
 
+proc ccHasSaneOverflow*(conf: ConfigRef): bool =
+  if conf.cCompiler == ccGcc:
+    result = false # assume an old or crappy GCC
+    var exe = getConfigVar(conf, conf.cCompiler, ".exe")
+    if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
+    let (s, exitCode) = try: execCmdEx(exe & " --version") except: ("", 1)
+    if exitCode == 0:
+      var i = 0
+      var j = 0
+      # the version is the last part of the first line:
+      while i < s.len and s[i] != '\n':
+        if s[i] in {' ', '\t'}: j = i+1
+        inc i
+      if j > 0:
+        var major = 0
+        while j < s.len and s[j] in {'0'..'9'}:
+          major = major * 10 + (ord(s[j]) - ord('0'))
+          inc j
+        if i < s.len and s[j] == '.': inc j
+        while j < s.len and s[j] in {'0'..'9'}:
+          inc j
+        if j+1 < s.len and s[j] == '.' and s[j+1] in {'0'..'9'}:
+          # we found a third version number, chances are high
+          # we really parsed the version:
+          result = major >= 5
+  else:
+    result = conf.cCompiler == ccCLang
+
 proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
            elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler
            else: getCompilerExe(conf, compiler, AbsoluteFile"")
 
-proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): string =
-  var c = conf.cCompiler
-  var options = cFileSpecificOptions(conf, cfile.nimname)
+proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
+                         isMainFile = false; produceOutput = false): string =
+  let c = conf.cCompiler
+  # We produce files like module.nim.cpp, so the absolute Nim filename is not
+  # cfile.name but `cfile.cname.changeFileExt("")`:
+  var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string)
   var exe = getConfigVar(conf, c, ".exe")
   if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname)
 
   if needsExeExt(conf): exe = addFileExt(exe, "exe")
   if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and
       ospNeedsPIC in platform.OS[conf.target.targetOS].props:
-    add(options, ' ' & CC[c].pic)
+    options.add(' ' & CC[c].pic)
 
-  var includeCmd, compilePattern: string
+  if useCpp(conf, cfile.cname):
+    options.add(' ' & CC[c].cppXsupport)
+
+  var compilePattern: string
+  # compute include paths:
+  var includeCmd = CC[c].includeCmd & quoteShell(conf.libpath)
   if not noAbsolutePaths(conf):
-    # compute include paths:
-    includeCmd = CC[c].includeCmd & quoteShell(conf.libpath)
-
     for includeDir in items(conf.cIncludes):
       includeCmd.add(join([CC[c].includeCmd, includeDir.quoteShell]))
 
-    compilePattern = joinPath(conf.ccompilerpath, exe)
+    compilePattern = joinPath(conf.cCompilerPath, exe)
   else:
-    includeCmd = ""
     compilePattern = getCompilerExe(conf, c, cfile.cname)
 
   includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)]))
@@ -603,10 +669,10 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): str
 
   var objfile =
     if cfile.obj.isEmpty:
-      if not cfile.flags.contains(CfileFlag.External) or noAbsolutePaths(conf):
+      if CfileFlag.External notin cfile.flags or noAbsolutePaths(conf):
         toObjFile(conf, cf).string
       else:
-        completeCFilePath(conf, toObjFile(conf, cf)).string
+        completeCfilePath(conf, toObjFile(conf, cf)).string
     elif noAbsolutePaths(conf):
       extractFilename(cfile.obj.string)
     else:
@@ -614,19 +680,30 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): str
 
   # D files are required by nintendo switch libs for
   # compilation. They are basically a list of all includes.
-  let dfile = objfile.changeFileExt(".d").quoteShell()
+  let dfile = objfile.changeFileExt(".d").quoteShell
 
-  objfile = quoteShell(objfile)
   let cfsh = quoteShell(cf)
   result = quoteShell(compilePattern % [
     "dfile", dfile,
-    "file", cfsh, "objfile", objfile, "options", options,
+    "file", cfsh, "objfile", quoteShell(objfile), "options", options,
     "include", includeCmd, "nim", getPrefixDir(conf).string,
     "lib", conf.libpath.string])
-  add(result, ' ')
-  addf(result, CC[c].compileTmpl, [
+
+  if optProduceAsm in conf.globalOptions:
+    if CC[conf.cCompiler].produceAsm.len > 0:
+      let asmfile = objfile.changeFileExt(".asm").quoteShell
+      addOpt(result, CC[conf.cCompiler].produceAsm % ["asmfile", asmfile])
+      if produceOutput:
+        rawMessage(conf, hintUserRaw, "Produced assembler here: " & asmfile)
+    else:
+      if produceOutput:
+        rawMessage(conf, hintUserRaw, "Couldn't produce assembler listing " &
+          "for the selected C compiler: " & CC[conf.cCompiler].name)
+
+  result.add(' ')
+  result.addf(CC[c].compileTmpl, [
     "dfile", dfile,
-    "file", cfsh, "objfile", objfile,
+    "file", cfsh, "objfile", quoteShell(objfile),
     "options", options, "include", includeCmd,
     "nim", quoteShell(getPrefixDir(conf)),
     "lib", quoteShell(conf.libpath),
@@ -641,7 +718,7 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
     getCompileCFileCmd(conf, cfile))
 
 proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
-  if conf.cmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM}:
+  if conf.cmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM, cmdNone}:
     return false
 
   var hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1")
@@ -662,30 +739,18 @@ proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) =
   if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and
       not externalFileChanged(conf, c):
     c.flags.incl CfileFlag.Cached
+  else:
+    # make sure Nim keeps recompiling the external file on reruns
+    # if compilation is not successful
+    discard tryRemoveFile(c.obj.string)
   conf.toCompile.add(c)
 
 proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) =
   var c = Cfile(nimname: splitFile(filename).name, cname: filename,
-    obj: toObjFile(conf, completeCFilePath(conf, filename, false)),
+    obj: toObjFile(conf, completeCfilePath(conf, filename, false)),
     flags: {CfileFlag.External})
   addExternalFileToCompile(conf, c)
 
-proc compileCFiles(conf: ConfigRef; list: CFileList, script: var Rope, cmds: var TStringSeq,
-                  prettyCmds: var TStringSeq) =
-  var currIdx = 0
-  for it in list:
-    # call the C compiler for the .c file:
-    if it.flags.contains(CfileFlag.Cached): continue
-    var compileCmd = getCompileCFileCmd(conf, it, currIdx == list.len - 1)
-    inc currIdx
-    if optCompileOnly notin conf.globalOptions:
-      add(cmds, compileCmd)
-      let (_, name, _) = splitFile(it.cname)
-      add(prettyCmds, if hintCC in conf.notes: "CC: " & demanglePackageName(name) else: "")
-    if optGenScript in conf.globalOptions:
-      add(script, compileCmd)
-      add(script, "\n")
-
 proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
                 objfiles: string, isDllBuild: bool): string =
   if optGenStaticLib in conf.globalOptions:
@@ -700,11 +765,11 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
                                             "objfiles", objfiles]
   else:
     var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe")
-    if len(linkerExe) == 0: linkerExe = getLinkerExe(conf, conf.cCompiler)
+    if linkerExe.len == 0: linkerExe = getLinkerExe(conf, conf.cCompiler)
     # bug #6452: We must not use ``quoteShell`` here for ``linkerExe``
     if needsExeExt(conf): linkerExe = addFileExt(linkerExe, "exe")
     if noAbsolutePaths(conf): result = linkerExe
-    else: result = joinPath(conf.cCompilerpath, linkerExe)
+    else: result = joinPath(conf.cCompilerPath, linkerExe)
     let buildgui = if optGenGuiApp in conf.globalOptions and conf.target.targetOS == osWindows:
                      CC[conf.cCompiler].buildGui
                    else:
@@ -730,7 +795,7 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
         "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles,
         "exefile", exefile, "nim", getPrefixDir(conf).string, "lib", conf.libpath.string])
     result.add ' '
-    addf(result, linkTmpl, ["builddll", builddll,
+    result.addf(linkTmpl, ["builddll", builddll,
         "mapfile", mapfile,
         "buildgui", buildgui, "options", linkOptions,
         "objfiles", objfiles, "exefile", exefile,
@@ -757,7 +822,7 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
     # way of being able to debug and rebuild the program at the same time. This
     # is accomplished using the /PDB:<filename> flag (there also exists the
     # /PDBALTPATH:<filename> flag). The only downside is that the .pdb files are
-    # atleast 300kb big (when linking statically to the runtime - or else 5mb+)
+    # at least 300kb big (when linking statically to the runtime - or else 5mb+)
     # and will quickly accumulate. There is a hacky solution: we could try to
     # delete all .pdb files with a pattern and swallow exceptions.
     #
@@ -795,20 +860,27 @@ template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body
 
 proc execLinkCmd(conf: ConfigRef; linkCmd: string) =
   tryExceptOSErrorMessage(conf, "invocation of external linker program failed."):
-    execExternalProgram(conf, linkCmd,
-      if optListCmd in conf.globalOptions or conf.verbosity > 1: hintExecuting else: hintLinking)
+    execExternalProgram(conf, linkCmd, hintLinking)
+
+proc maybeRunDsymutil(conf: ConfigRef; exe: AbsoluteFile) =
+  when defined(osx):
+    if optCDebug in conf.globalOptions and optGenStaticLib notin conf.globalOptions:
+      # if needed, add an option to skip or override location
+      let cmd = "dsymutil " & $(exe).quoteShell
+      conf.extraCmds.add cmd
+      tryExceptOSErrorMessage(conf, "invocation of dsymutil failed."):
+        execExternalProgram(conf, cmd, hintExecuting)
 
 proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx: int)) =
   let runCb = proc (idx: int, p: Process) =
     let exitCode = p.peekExitCode
     if exitCode != 0:
       rawMessage(conf, errGenerated, "execution of an external compiler program '" &
-        cmds[idx] & "' failed with exit code: " & $exitCode & "\n\n" &
-        p.outputStream.readAll.strip)
+        cmds[idx] & "' failed with exit code: " & $exitCode & "\n\n")
   if conf.numberOfProcessors == 0: conf.numberOfProcessors = countProcessors()
   var res = 0
   if conf.numberOfProcessors <= 1:
-    for i in 0 .. high(cmds):
+    for i in 0..high(cmds):
       tryExceptOSErrorMessage(conf, "invocation of external compiler program failed."):
         res = execWithEcho(conf, cmds[i])
       if res != 0:
@@ -816,15 +888,8 @@ proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx:
           cmds[i])
   else:
     tryExceptOSErrorMessage(conf, "invocation of external compiler program failed."):
-      if optListCmd in conf.globalOptions or conf.verbosity > 1:
-        res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath},
-                            conf.numberOfProcessors, afterRunEvent=runCb)
-      elif conf.verbosity == 1:
-        res = execProcesses(cmds, {poStdErrToStdOut, poUsePath},
+      res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams},
                             conf.numberOfProcessors, prettyCb, afterRunEvent=runCb)
-      else:
-        res = execProcesses(cmds, {poStdErrToStdOut, poUsePath},
-                            conf.numberOfProcessors, afterRunEvent=runCb)
   if res != 0:
     if conf.numberOfProcessors <= 1:
       rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" %
@@ -856,7 +921,7 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) =
   finally:
     removeFile(linkerArgs)
 
-proc getObjFilePath(conf: ConfigRef, f: CFile): string =
+proc getObjFilePath(conf: ConfigRef, f: Cfile): string =
   if noAbsolutePaths(conf): f.obj.extractFilename
   else: f.obj.string
 
@@ -866,6 +931,13 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu
                    else: platform.OS[conf.target.targetOS].dllFrmt % basename
   result = conf.getNimcacheDir / RelativeFile(targetName)
 
+proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string =
+  if conf.hasHint(hintCC):
+    if optListCmd in conf.globalOptions or conf.verbosity > 1:
+      result = MsgKindToStr[hintCC] % (demanglePackageName(path.splitFile.name) & ": " & compileCmd)
+    else:
+      result = MsgKindToStr[hintCC] % demanglePackageName(path.splitFile.name)
+
 proc callCCompiler*(conf: ConfigRef) =
   var
     linkCmd: string
@@ -874,13 +946,22 @@ proc callCCompiler*(conf: ConfigRef) =
            # generated
   #var c = cCompiler
   var script: Rope = nil
-  var cmds: TStringSeq = @[]
-  var prettyCmds: TStringSeq = @[]
+  var cmds: TStringSeq
+  var prettyCmds: TStringSeq
   let prettyCb = proc (idx: int) =
-    when declared(echo):
-      let cmd = prettyCmds[idx]
-      if cmd != "": echo cmd
-  compileCFiles(conf, conf.toCompile, script, cmds, prettyCmds)
+    if prettyCmds[idx].len > 0: echo prettyCmds[idx]
+
+  for idx, it in conf.toCompile:
+    # call the C compiler for the .c file:
+    if CfileFlag.Cached in it.flags: continue
+    let compileCmd = getCompileCFileCmd(conf, it, idx == conf.toCompile.len - 1, produceOutput=true)
+    if optCompileOnly notin conf.globalOptions:
+      cmds.add(compileCmd)
+      prettyCmds.add displayProgressCC(conf, $it.cname, compileCmd)
+    if optGenScript in conf.globalOptions:
+      script.add(compileCmd)
+      script.add("\n")
+
   if optCompileOnly notin conf.globalOptions:
     execCmdsInParallel(conf, cmds, prettyCb)
   if optNoLinking notin conf.globalOptions:
@@ -888,8 +969,8 @@ proc callCCompiler*(conf: ConfigRef) =
     var objfiles = ""
     for it in conf.externalToLink:
       let objFile = if noAbsolutePaths(conf): it.extractFilename else: it
-      add(objfiles, ' ')
-      add(objfiles, quoteShell(
+      objfiles.add(' ')
+      objfiles.add(quoteShell(
           addFileExt(objFile, CC[conf.cCompiler].objExt)))
 
     if conf.hcrOn: # lets assume that optCompileOnly isn't on
@@ -899,14 +980,14 @@ proc callCCompiler*(conf: ConfigRef) =
         # don't relink each of the many binaries (one for each source file) if the nim code is
         # cached because that would take too much time for small changes - the only downside to
         # this is that if an external-to-link file changes the final target wouldn't be relinked
-        if x.flags.contains(CfileFlag.Cached): continue
+        if CfileFlag.Cached in x.flags: continue
         # we pass each object file as if it is the project file - a .dll will be created for each such
         # object file in the nimcache directory, and only in the case of the main project file will
         # there be probably an executable (if the project is such) which will be copied out of the nimcache
         let objFile = conf.getObjFilePath(x)
         let buildDll = idx != mainFileIdx
         let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll)
-        add(cmds, getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll))
+        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll))
         # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache
         # for more info check the comment inside of getLinkCmd() where the /PDB:<filename> MSVC flag is used
         if isVSCompatible(conf):
@@ -917,7 +998,7 @@ proc callCCompiler*(conf: ConfigRef) =
       prettyCmds = map(prettyCmds, proc (curr: string): string = return curr.replace("CC", "Link"))
       execCmdsInParallel(conf, cmds, prettyCb)
       # only if not cached - copy the resulting main file from the nimcache folder to its originally intended destination
-      if not conf.toCompile[mainFileIdx].flags.contains(CfileFlag.Cached):
+      if CfileFlag.Cached notin conf.toCompile[mainFileIdx].flags:
         let mainObjFile = getObjFilePath(conf, conf.toCompile[mainFileIdx])
         var src = conf.hcrLinkTargetName(mainObjFile, true)
         var dst = conf.prepareToWriteOutput
@@ -925,8 +1006,8 @@ proc callCCompiler*(conf: ConfigRef) =
     else:
       for x in conf.toCompile:
         let objFile = if noAbsolutePaths(conf): x.obj.extractFilename else: x.obj.string
-        add(objfiles, ' ')
-        add(objfiles, quoteShell(objFile))
+        objfiles.add(' ')
+        objfiles.add(quoteShell(objFile))
       let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput
                        else: AbsoluteFile(conf.projectName)
       linkCmd = getLinkCmd(conf, mainOutput, objfiles)
@@ -939,15 +1020,18 @@ proc callCCompiler*(conf: ConfigRef) =
           linkViaResponseFile(conf, linkCmd)
         else:
           execLinkCmd(conf, linkCmd)
+        maybeRunDsymutil(conf, mainOutput)
   else:
     linkCmd = ""
   if optGenScript in conf.globalOptions:
-    add(script, linkCmd)
-    add(script, "\n")
+    script.add(linkCmd)
+    script.add("\n")
     generateScript(conf, script)
 
 #from json import escapeJson
-import json
+import json, std / sha1
+
+template hashNimExe(): string = $secureHashFile(os.getAppFilename())
 
 proc writeJsonBuildInstructions*(conf: ConfigRef) =
   template lit(x: untyped) = f.write x
@@ -960,17 +1044,16 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
       f.write escapeJson(x)
 
   proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) =
-    var pastStart = false
-    for it in clist:
+    var comma = false
+    for i, it in clist:
       if CfileFlag.Cached in it.flags: continue
       let compileCmd = getCompileCFileCmd(conf, it)
-      if pastStart: lit "],\L"
+      if comma: lit ",\L" else: comma = true
       lit "["
       str it.cname.string
       lit ", "
       str compileCmd
-      pastStart = true
-    lit "]\L"
+      lit "]"
 
   proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList;
                  llist: seq[string]) =
@@ -979,21 +1062,35 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
       let objfile = if noAbsolutePaths(conf): it.extractFilename
                     else: it
       let objstr = addFileExt(objfile, CC[conf.cCompiler].objExt)
-      add(objfiles, ' ')
-      add(objfiles, objstr)
+      objfiles.add(' ')
+      objfiles.add(objstr)
       if pastStart: lit ",\L"
       str objstr
       pastStart = true
 
     for it in clist:
       let objstr = quoteShell(it.obj)
-      add(objfiles, ' ')
-      add(objfiles, objstr)
+      objfiles.add(' ')
+      objfiles.add(objstr)
       if pastStart: lit ",\L"
       str objstr
       pastStart = true
     lit "\L"
 
+  proc depfiles(conf: ConfigRef; f: File) =
+    var i = 0
+    for it in conf.m.fileInfos:
+      let path = it.fullPath.string
+      if isAbsolute(path): # TODO: else?
+        if i > 0: lit "],\L"
+        lit "["
+        str path
+        lit ", "
+        str $secureHashFile(path)
+        inc i
+    lit "]\L"
+
+
   var buf = newStringOfCap(50)
 
   let jsonFile = conf.getNimcacheDir / RelativeFile(conf.projectName & ".json")
@@ -1009,57 +1106,116 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
 
     lit "],\L\"linkcmd\": "
     str getLinkCmd(conf, conf.absOutFile, objfiles)
+
+    lit ",\L\"extraCmds\": "
+    lit $(%* conf.extraCmds)
+
+    lit ",\L\"stdinInput\": "
+    lit $(%* conf.projectIsStdin)
+
+    if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"):
+      lit ",\L\"cmdline\": "
+      str conf.commandLine
+      lit ",\L\"depfiles\":[\L"
+      depfiles(conf, f)
+      lit "],\L\"nimexe\": \L"
+      str hashNimExe()
+      lit "\L"
+
     lit "\L}\L"
     close(f)
 
+proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile): bool =
+  let jsonFile = toGeneratedFile(conf, projectfile, "json")
+  if not fileExists(jsonFile): return true
+  if not fileExists(conf.absOutFile): return true
+  result = false
+  try:
+    let data = json.parseFile(jsonFile.string)
+    if not data.hasKey("depfiles") or not data.hasKey("cmdline"):
+      return true
+    let oldCmdLine = data["cmdline"].getStr
+    if conf.commandLine != oldCmdLine:
+      return true
+    if hashNimExe() != data["nimexe"].getStr:
+      return true
+    if not data.hasKey("stdinInput"): return true
+    let stdinInput = data["stdinInput"].getBool
+    if conf.projectIsStdin or stdinInput:
+      # could optimize by returning false if stdin input was the same,
+      # but I'm not sure how to get full stding input
+      return true
+
+    let depfilesPairs = data["depfiles"]
+    doAssert depfilesPairs.kind == JArray
+    for p in depfilesPairs:
+      doAssert p.kind == JArray
+      # >= 2 for forwards compatibility with potential later .json files:
+      doAssert p.len >= 2
+      let depFilename = p[0].getStr
+      let oldHashValue = p[1].getStr
+      let newHashValue = $secureHashFile(depFilename)
+      if oldHashValue != newHashValue:
+        return true
+  except IOError, OSError, ValueError:
+    echo "Warning: JSON processing failed: ", getCurrentExceptionMsg()
+    result = true
+
 proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) =
   let jsonFile = toGeneratedFile(conf, projectfile, "json")
   try:
     let data = json.parseFile(jsonFile.string)
     let toCompile = data["compile"]
     doAssert toCompile.kind == JArray
-    var cmds: TStringSeq = @[]
-    var prettyCmds: TStringSeq = @[]
+    var cmds: TStringSeq
+    var prettyCmds: TStringSeq
+    let prettyCb = proc (idx: int) =
+      if prettyCmds[idx].len > 0: echo prettyCmds[idx]
+
     for c in toCompile:
       doAssert c.kind == JArray
       doAssert c.len >= 2
 
-      add(cmds, c[1].getStr)
-      let (_, name, _) = splitFile(c[0].getStr)
-      add(prettyCmds, "CC: " & name)
+      cmds.add(c[1].getStr)
+      prettyCmds.add displayProgressCC(conf, c[0].getStr, c[1].getStr)
 
-    let prettyCb = proc (idx: int) =
-      when declared(echo):
-        echo prettyCmds[idx]
     execCmdsInParallel(conf, cmds, prettyCb)
 
     let linkCmd = data["linkcmd"]
     doAssert linkCmd.kind == JString
     execLinkCmd(conf, linkCmd.getStr)
-  except:
-    when declared(echo):
-      echo getCurrentException().getStackTrace()
-    quit "error evaluating JSON file: " & jsonFile.string
+    if data.hasKey("extraCmds"):
+      let extraCmds = data["extraCmds"]
+      doAssert extraCmds.kind == JArray
+      for cmd in extraCmds:
+        doAssert cmd.kind == JString, $cmd.kind
+        let cmd2 = cmd.getStr
+        execExternalProgram(conf, cmd2, hintExecuting)
 
-proc genMappingFiles(conf: ConfigRef; list: CFileList): Rope =
+  except:
+    let e = getCurrentException()
+    quit "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() &
+         "error evaluating JSON file: " & jsonFile.string
+
+proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope =
   for it in list:
-    addf(result, "--file:r\"$1\"$N", [rope(it.cname.string)])
+    result.addf("--file:r\"$1\"$N", [rope(it.cname.string)])
 
 proc writeMapping*(conf: ConfigRef; symbolMapping: Rope) =
   if optGenMapping notin conf.globalOptions: return
   var code = rope("[C_Files]\n")
-  add(code, genMappingFiles(conf, conf.toCompile))
-  add(code, "\n[C_Compiler]\nFlags=")
-  add(code, strutils.escape(getCompileOptions(conf)))
+  code.add(genMappingFiles(conf, conf.toCompile))
+  code.add("\n[C_Compiler]\nFlags=")
+  code.add(strutils.escape(getCompileOptions(conf)))
 
-  add(code, "\n[Linker]\nFlags=")
-  add(code, strutils.escape(getLinkOptions(conf) & " " &
+  code.add("\n[Linker]\nFlags=")
+  code.add(strutils.escape(getLinkOptions(conf) & " " &
                             getConfigVar(conf, conf.cCompiler, ".options.linker")))
 
-  add(code, "\n[Environment]\nlibpath=")
-  add(code, strutils.escape(conf.libpath.string))
+  code.add("\n[Environment]\nlibpath=")
+  code.add(strutils.escape(conf.libpath.string))
 
-  addf(code, "\n[Symbols]$n$1", [symbolMapping])
+  code.addf("\n[Symbols]$n$1", [symbolMapping])
   let filename = conf.projectPath / RelativeFile"mapping.txt"
   if not writeRope(code, filename):
     rawMessage(conf, errGenerated, "could not write to file: " & filename.string)

compiler/filter_tmpl.nim

@@ -10,8 +10,8 @@
 # This module implements Nim's standard template filter.
 
 import
-  llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
-  renderer, filters, lineinfos, pathutils
+  llstream, strutils, ast, msgs, options,
+  filters, lineinfos, pathutils
 
 type
   TParseState = enum
@@ -22,7 +22,7 @@ type
     info: TLineInfo
     indent, emitPar: int
     x: string                # the current input line
-    outp: PLLStream          # the ouput will be parsed by pnimsyn
+    outp: PLLStream          # the output will be parsed by pnimsyn
     subsChar, nimDirective: char
     emit, conc, toStr: string
     curly, bracket, par: int
@@ -75,7 +75,7 @@ proc parseLine(p: var TTmplParser) =
     let d = j
     var keyw = ""
     while j < len and p.x[j] in PatternChars:
-      add(keyw, p.x[j])
+      keyw.add(p.x[j])
       inc(j)
 
     scanPar(p, j)

compiler/filters.nim

@@ -10,7 +10,7 @@
 # This module implements Nim's simple filters and helpers for filters.
 
 import
-  llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
+  llstream, idents, strutils, ast, msgs, options,
   renderer, pathutils
 
 proc invalidPragma(conf: ConfigRef; n: PNode) =
@@ -20,13 +20,13 @@ proc invalidPragma(conf: ConfigRef; n: PNode) =
 proc getArg(conf: ConfigRef; n: PNode, name: string, pos: int): PNode =
   result = nil
   if n.kind in {nkEmpty..nkNilLit}: return
-  for i in 1 ..< sonsLen(n):
-    if n.sons[i].kind == nkExprEqExpr:
-      if n.sons[i].sons[0].kind != nkIdent: invalidPragma(conf, n)
-      if cmpIgnoreStyle(n.sons[i].sons[0].ident.s, name) == 0:
-        return n.sons[i].sons[1]
+  for i in 1..<n.len:
+    if n[i].kind == nkExprEqExpr:
+      if n[i][0].kind != nkIdent: invalidPragma(conf, n)
+      if cmpIgnoreStyle(n[i][0].ident.s, name) == 0:
+        return n[i][1]
     elif i == pos:
-      return n.sons[i]
+      return n[i]
 
 proc charArg*(conf: ConfigRef; n: PNode, name: string, pos: int, default: char): char =
   var x = getArg(conf, n, name, pos)
@@ -55,7 +55,7 @@ proc filterStrip*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, cal
   var line = newStringOfCap(80)
   while llStreamReadLine(stdin, line):
     var stripped = strip(line, leading, trailing)
-    if len(pattern) == 0 or startsWith(stripped, pattern):
+    if pattern.len == 0 or startsWith(stripped, pattern):
       llStreamWriteln(result, stripped)
     else:
       llStreamWriteln(result, line)
@@ -63,7 +63,7 @@ proc filterStrip*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, cal
 
 proc filterReplace*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, call: PNode): PLLStream =
   var sub = strArg(conf, call, "sub", 1, "")
-  if len(sub) == 0: invalidPragma(conf, call)
+  if sub.len == 0: invalidPragma(conf, call)
   var by = strArg(conf, call, "by", 2, "")
   result = llStreamOpen("")
   var line = newStringOfCap(80)

compiler/forloops.nim

@@ -13,8 +13,8 @@ import ast, astalgo
 
 const
   someCmp = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
-    mEqUntracedRef, mLeI, mLeF64, mLeU, mLeU64, mLeEnum,
-    mLeCh, mLeB, mLePtr, mLtI, mLtF64, mLtU, mLtU64, mLtEnum,
+    mLeI, mLeF64, mLeU, mLeEnum,
+    mLeCh, mLeB, mLePtr, mLtI, mLtF64, mLtU, mLtEnum,
     mLtCh, mLtB, mLtPtr}
 
 proc isCounter(s: PSym): bool {.inline.} =
@@ -45,13 +45,13 @@ proc counterInTree(n, loop: PNode; counter: PSym): bool =
     for it in n:
       if counterInTree(it.lastSon): return true
   else:
-    for i in 0 ..< safeLen(n):
+    for i in 0..<n.safeLen:
       if counterInTree(n[i], loop, counter): return true
 
 proc copyExcept(n: PNode, x, dest: PNode) =
   if x == n: return
   if n.kind in {nkStmtList, nkStmtListExpr}:
-    for i in 0 ..< n.len: copyExcept(n[i], x, dest)
+    for i in 0..<n.len: copyExcept(n[i], x, dest)
   else:
     dest.add n
 

compiler/gorgeimpl.nim

@@ -9,7 +9,7 @@
 
 ## Module that implements ``gorge`` for the compiler.
 
-import msgs, std / sha1, os, osproc, streams, strutils, options,
+import msgs, std / sha1, os, osproc, streams, options,
   lineinfos, pathutils
 
 proc readOutput(p: Process): (string, int) =
@@ -35,11 +35,12 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
     var readSuccessful = false
     try:
       var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStderrToStdout})
+                           options={poEvalCommand, poStdErrToStdOut})
       if input.len != 0:
         p.inputStream.write(input)
         p.inputStream.close()
       result = p.readOutput
+      p.close()
       readSuccessful = true
       # only cache successful runs:
       if result[1] == 0:
@@ -49,10 +50,11 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
   else:
     try:
       var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStderrToStdout})
+                           options={poEvalCommand, poStdErrToStdOut})
       if input.len != 0:
         p.inputStream.write(input)
         p.inputStream.close()
       result = p.readOutput
+      p.close()
     except IOError, OSError:
       result = ("", -1)

compiler/guards.nim

@@ -10,22 +10,21 @@
 ## This module implements the 'implies' relation for guards.
 
 import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents,
-  saturate, modulegraphs, options, lineinfos
+  saturate, modulegraphs, options, lineinfos, int128
 
 const
   someEq = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
-    mEqUntracedRef, mEqStr, mEqSet, mEqCString}
+    mEqStr, mEqSet, mEqCString}
 
   # set excluded here as the semantics are vastly different:
-  someLe = {mLeI, mLeF64, mLeU, mLeU64, mLeEnum,
+  someLe = {mLeI, mLeF64, mLeU, mLeEnum,
             mLeCh, mLeB, mLePtr, mLeStr}
-  someLt = {mLtI, mLtF64, mLtU, mLtU64, mLtEnum,
+  someLt = {mLtI, mLtF64, mLtU, mLtEnum,
             mLtCh, mLtB, mLtPtr, mLtStr}
 
-  someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq,
-             mXLenStr, mXLenSeq}
+  someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq}
 
-  someIn = {mInRange, mInSet}
+  someIn = {mInSet}
 
   someHigh = {mHigh}
   # we don't list unsigned here because wrap around semantics suck for
@@ -35,8 +34,8 @@ const
   someMul = {mMulI, mMulF64}
   someDiv = {mDivI, mDivF64}
   someMod = {mModI}
-  someMax = {mMaxI, mMaxF64}
-  someMin = {mMinI, mMinF64}
+  someMax = {mMaxI}
+  someMin = {mMinI}
   someBinaryOp = someAdd+someSub+someMul+someMax+someMin
 
 proc isValue(n: PNode): bool = n.kind in {nkCharLit..nkNilLit}
@@ -65,16 +64,16 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
   while true:
     case n.kind
     of nkDotExpr, nkCheckedFieldExpr, nkObjUpConv, nkObjDownConv:
-      n = n.sons[0]
+      n = n[0]
     of nkDerefExpr, nkHiddenDeref:
-      n = n.sons[0]
+      n = n[0]
       inc derefs
     of nkBracketExpr:
-      if isConstExpr(n.sons[1]) or isLet(n.sons[1]) or isConstExpr(n.sons[1].skipConv):
-        n = n.sons[0]
+      if isConstExpr(n[1]) or isLet(n[1]) or isConstExpr(n[1].skipConv):
+        n = n[0]
       else: return
     of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-      n = n.sons[1]
+      n = n[1]
     else:
       break
   result = n.isLet and derefs <= ord(isApprox)
@@ -105,34 +104,34 @@ proc initOperators*(g: ModuleGraph): Operators =
 
 proc swapArgs(fact: PNode, newOp: PSym): PNode =
   result = newNodeI(nkCall, fact.info, 3)
-  result.sons[0] = newSymNode(newOp)
-  result.sons[1] = fact.sons[2]
-  result.sons[2] = fact.sons[1]
+  result[0] = newSymNode(newOp)
+  result[1] = fact[2]
+  result[2] = fact[1]
 
 proc neg(n: PNode; o: Operators): PNode =
   if n == nil: return nil
   case n.getMagic
   of mNot:
-    result = n.sons[1]
+    result = n[1]
   of someLt:
     # not (a < b)  ==  a >= b  ==  b <= a
     result = swapArgs(n, o.opLe)
   of someLe:
     result = swapArgs(n, o.opLt)
   of mInSet:
-    if n.sons[1].kind != nkCurly: return nil
-    let t = n.sons[2].typ.skipTypes(abstractInst)
+    if n[1].kind != nkCurly: return nil
+    let t = n[2].typ.skipTypes(abstractInst)
     result = newNodeI(nkCall, n.info, 3)
-    result.sons[0] = n.sons[0]
-    result.sons[2] = n.sons[2]
+    result[0] = n[0]
+    result[2] = n[2]
     if t.kind == tyEnum:
-      var s = newNodeIT(nkCurly, n.info, n.sons[1].typ)
+      var s = newNodeIT(nkCurly, n.info, n[1].typ)
       for e in t.n:
         let eAsNode = newIntNode(nkIntLit, e.sym.position)
-        if not inSet(n.sons[1], eAsNode): s.add eAsNode
-      result.sons[1] = s
+        if not inSet(n[1], eAsNode): s.add eAsNode
+      result[1] = s
     #elif t.kind notin {tyString, tySequence} and lengthOrd(t) < 1000:
-    #  result.sons[1] = complement(n.sons[1])
+    #  result[1] = complement(n[1])
     else:
       # not ({2, 3, 4}.contains(x))   x != 2 and x != 3 and x != 4
       # XXX todo
@@ -140,13 +139,13 @@ proc neg(n: PNode; o: Operators): PNode =
   of mOr:
     # not (a or b) --> not a and not b
     let
-      a = n.sons[1].neg(o)
-      b = n.sons[2].neg(o)
+      a = n[1].neg(o)
+      b = n[2].neg(o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
     elif a != nil:
       result = a
     elif b != nil:
@@ -154,19 +153,19 @@ proc neg(n: PNode; o: Operators): PNode =
   else:
     # leave  not (a == 4)  as it is
     result = newNodeI(nkCall, n.info, 2)
-    result.sons[0] = newSymNode(o.opNot)
-    result.sons[1] = n
+    result[0] = newSymNode(o.opNot)
+    result[1] = n
 
 proc buildCall(op: PSym; a: PNode): PNode =
   result = newNodeI(nkCall, a.info, 2)
-  result.sons[0] = newSymNode(op)
-  result.sons[1] = a
+  result[0] = newSymNode(op)
+  result[1] = a
 
 proc buildCall(op: PSym; a, b: PNode): PNode =
   result = newNodeI(nkInfix, a.info, 3)
-  result.sons[0] = newSymNode(op)
-  result.sons[1] = a
-  result.sons[2] = b
+  result[0] = newSymNode(op)
+  result[1] = a
+  result[2] = b
 
 proc `|+|`(a, b: PNode): PNode =
   result = copyNode(a)
@@ -250,35 +249,35 @@ proc pred(n: PNode): PNode =
   else:
     result = n
 
+proc buildLe*(o: Operators; a, b: PNode): PNode =
+  result = o.opLe.buildCall(a, b)
+
 proc canon*(n: PNode; o: Operators): PNode =
-  # XXX for now only the new code in 'semparallel' uses this
   if n.safeLen >= 1:
     result = shallowCopy(n)
-    for i in 0 ..< n.len:
-      result.sons[i] = canon(n.sons[i], o)
+    for i in 0..<n.len:
+      result[i] = canon(n[i], o)
   elif n.kind == nkSym and n.sym.kind == skLet and
       n.sym.astdef.getMagic in (someEq + someAdd + someMul + someMin +
-      someMax + someHigh + {mUnaryLt} + someSub + someLen + someDiv):
+      someMax + someHigh + someSub + someLen + someDiv):
     result = n.sym.astdef.copyTree
   else:
     result = n
   case result.getMagic
   of someEq, someAdd, someMul, someMin, someMax:
     # these are symmetric; put value as last:
-    if result.sons[1].isValue and not result.sons[2].isValue:
-      result = swapArgs(result, result.sons[0].sym)
+    if result[1].isValue and not result[2].isValue:
+      result = swapArgs(result, result[0].sym)
       # (4 + foo) + 2 --> (foo + 4) + 2
   of someHigh:
     # high == len+(-1)
     result = o.opAdd.buildCall(o.opLen.buildCall(result[1]), minusOne())
-  of mUnaryLt:
-    result = buildCall(o.opAdd, result[1], minusOne())
   of someSub:
     # x - 4  -->  x + (-4)
     result = negate(result[1], result[2], result, o)
   of someLen:
-    result.sons[0] = o.opLen.newSymNode
-  of someLt:
+    result[0] = o.opLen.newSymNode
+  of someLt - {mLtF64}:
     # x < y  same as x <= y-1:
     let y = n[2].canon(o)
     let p = pred(y)
@@ -318,16 +317,16 @@ proc canon*(n: PNode; o: Operators): PNode =
         if plus != nil and not isLetLocation(x, true):
           result = buildCall(result[0].sym, plus, y[1])
       else: discard
-    elif x.isValue and y.getMagic in someAdd and y[2].isValue:
+    elif x.isValue and y.getMagic in someAdd and y[2].kind == x.kind:
       # 0 <= a.len + 3
       # -3 <= a.len
-      result.sons[1] = x |-| y[2]
-      result.sons[2] = y[1]
-    elif x.isValue and y.getMagic in someSub and y[2].isValue:
+      result[1] = x |-| y[2]
+      result[2] = y[1]
+    elif x.isValue and y.getMagic in someSub and y[2].kind == x.kind:
       # 0 <= a.len - 3
       # 3 <= a.len
-      result.sons[1] = x |+| y[2]
-      result.sons[2] = y[1]
+      result[1] = x |+| y[2]
+      result[2] = y[1]
   else: discard
 
 proc buildAdd*(a: PNode; b: BiggestInt; o: Operators): PNode =
@@ -336,41 +335,43 @@ proc buildAdd*(a: PNode; b: BiggestInt; o: Operators): PNode =
 proc usefulFact(n: PNode; o: Operators): PNode =
   case n.getMagic
   of someEq:
-    if skipConv(n.sons[2]).kind == nkNilLit and (
-        isLetLocation(n.sons[1], false) or isVar(n.sons[1])):
-      result = o.opIsNil.buildCall(n.sons[1])
+    if skipConv(n[2]).kind == nkNilLit and (
+        isLetLocation(n[1], false) or isVar(n[1])):
+      result = o.opIsNil.buildCall(n[1])
     else:
-      if isLetLocation(n.sons[1], true) or isLetLocation(n.sons[2], true):
+      if isLetLocation(n[1], true) or isLetLocation(n[2], true):
         # XXX algebraic simplifications!  'i-1 < a.len' --> 'i < a.len+1'
         result = n
+      elif n[1].getMagic in someLen or n[2].getMagic in someLen:
+        result = n
   of someLe+someLt:
-    if isLetLocation(n.sons[1], true) or isLetLocation(n.sons[2], true):
+    if isLetLocation(n[1], true) or isLetLocation(n[2], true):
       # XXX algebraic simplifications!  'i-1 < a.len' --> 'i < a.len+1'
       result = n
     elif n[1].getMagic in someLen or n[2].getMagic in someLen:
       # XXX Rethink this whole idea of 'usefulFact' for semparallel
       result = n
   of mIsNil:
-    if isLetLocation(n.sons[1], false) or isVar(n.sons[1]):
+    if isLetLocation(n[1], false) or isVar(n[1]):
       result = n
   of someIn:
-    if isLetLocation(n.sons[1], true):
+    if isLetLocation(n[1], true):
       result = n
   of mAnd:
     let
-      a = usefulFact(n.sons[1], o)
-      b = usefulFact(n.sons[2], o)
+      a = usefulFact(n[1], o)
+      b = usefulFact(n[2], o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
     elif a != nil:
       result = a
     elif b != nil:
       result = b
   of mNot:
-    let a = usefulFact(n.sons[1], o)
+    let a = usefulFact(n[1], o)
     if a != nil:
       result = a.neg(o)
   of mOr:
@@ -381,13 +382,13 @@ proc usefulFact(n: PNode; o: Operators): PNode =
     #  (x == 3) or (y == 2)  ---> not ( not (x==3) and not (y == 2))
     #  not (x != 3 and y != 2)
     let
-      a = usefulFact(n.sons[1], o).neg(o)
-      b = usefulFact(n.sons[2], o).neg(o)
+      a = usefulFact(n[1], o).neg(o)
+      b = usefulFact(n[2], o).neg(o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
       result = result.neg(o)
   elif n.kind == nkSym and n.sym.kind == skLet:
     # consider:
@@ -404,10 +405,20 @@ type
   TModel* = object
     s*: seq[PNode] # the "knowledge base"
     o*: Operators
+    beSmart*: bool
 
 proc addFact*(m: var TModel, nn: PNode) =
   let n = usefulFact(nn, m.o)
-  if n != nil: m.s.add n
+  if n != nil:
+    if not m.beSmart:
+      m.s.add n
+    else:
+      let c = canon(n, m.o)
+      if c.getMagic == mAnd:
+        addFact(m, c[1])
+        addFact(m, c[2])
+      else:
+        m.s.add c
 
 proc addFactNeg*(m: var TModel, n: PNode) =
   let n = n.neg(m.o)
@@ -436,22 +447,22 @@ proc sameTree*(a, b: PNode): bool =
       if not result and a.sym.magic != mNone:
         result = a.sym.magic == b.sym.magic or sameOpr(a.sym, b.sym)
     of nkIdent: result = a.ident.id == b.ident.id
-    of nkCharLit..nkInt64Lit: result = a.intVal == b.intVal
+    of nkCharLit..nkUInt64Lit: result = a.intVal == b.intVal
     of nkFloatLit..nkFloat64Lit: result = a.floatVal == b.floatVal
     of nkStrLit..nkTripleStrLit: result = a.strVal == b.strVal
     of nkType: result = a.typ == b.typ
     of nkEmpty, nkNilLit: result = true
     else:
-      if sonsLen(a) == sonsLen(b):
-        for i in 0 ..< sonsLen(a):
-          if not sameTree(a.sons[i], b.sons[i]): return
+      if a.len == b.len:
+        for i in 0..<a.len:
+          if not sameTree(a[i], b[i]): return
         result = true
 
 proc hasSubTree(n, x: PNode): bool =
   if n.sameTree(x): result = true
   else:
-    for i in 0..safeLen(n)-1:
-      if hasSubTree(n.sons[i], x): return true
+    for i in 0..n.safeLen-1:
+      if hasSubTree(n[i], x): return true
 
 proc invalidateFacts*(m: var TModel, n: PNode) =
   # We are able to guard local vars (as opposed to 'let' variables)!
@@ -479,21 +490,21 @@ proc valuesUnequal(a, b: PNode): bool =
     result = not sameValue(a, b)
 
 proc impliesEq(fact, eq: PNode): TImplication =
-  let (loc, val) = if isLocation(eq.sons[1]): (1, 2) else: (2, 1)
+  let (loc, val) = if isLocation(eq[1]): (1, 2) else: (2, 1)
 
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], eq.sons[loc]):
+    if sameTree(fact[1], eq[loc]):
       # this is not correct; consider:  a == b;  a == 1 --> unknown!
-      if sameTree(fact.sons[2], eq.sons[val]): result = impYes
-      elif valuesUnequal(fact.sons[2], eq.sons[val]): result = impNo
-    elif sameTree(fact.sons[2], eq.sons[loc]):
-      if sameTree(fact.sons[1], eq.sons[val]): result = impYes
-      elif valuesUnequal(fact.sons[1], eq.sons[val]): result = impNo
+      if sameTree(fact[2], eq[val]): result = impYes
+      elif valuesUnequal(fact[2], eq[val]): result = impNo
+    elif sameTree(fact[2], eq[loc]):
+      if sameTree(fact[1], eq[val]): result = impYes
+      elif valuesUnequal(fact[1], eq[val]): result = impNo
   of mInSet:
     # remember: mInSet is 'contains' so the set comes first!
-    if sameTree(fact.sons[2], eq.sons[loc]) and isValue(eq.sons[val]):
-      if inSet(fact.sons[1], eq.sons[val]): result = impYes
+    if sameTree(fact[2], eq[loc]) and isValue(eq[val]):
+      if inSet(fact[1], eq[val]): result = impYes
       else: result = impNo
   of mNot, mOr, mAnd: assert(false, "impliesEq")
   else: discard
@@ -522,7 +533,7 @@ proc geImpliesIn(x, c, aSet: PNode): TImplication =
     var value = newIntNode(c.kind, c.intVal)
     let max = lastOrd(nil, x.typ)
     # don't iterate too often:
-    if max - value.intVal < 1000:
+    if max - getInt(value) < toInt128(1000):
       var i, pos, neg: int
       while value.intVal <= max:
         if inSet(aSet, value): inc pos
@@ -536,28 +547,28 @@ proc compareSets(a, b: PNode): TImplication =
   elif intersectSets(nil, a, b).len == 0: result = impNo
 
 proc impliesIn(fact, loc, aSet: PNode): TImplication =
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], loc):
-      if inSet(aSet, fact.sons[2]): result = impYes
+    if sameTree(fact[1], loc):
+      if inSet(aSet, fact[2]): result = impYes
       else: result = impNo
-    elif sameTree(fact.sons[2], loc):
-      if inSet(aSet, fact.sons[1]): result = impYes
+    elif sameTree(fact[2], loc):
+      if inSet(aSet, fact[1]): result = impYes
       else: result = impNo
   of mInSet:
-    if sameTree(fact.sons[2], loc):
-      result = compareSets(fact.sons[1], aSet)
+    if sameTree(fact[2], loc):
+      result = compareSets(fact[1], aSet)
   of someLe:
-    if sameTree(fact.sons[1], loc):
-      result = leImpliesIn(fact.sons[1], fact.sons[2], aSet)
-    elif sameTree(fact.sons[2], loc):
-      result = geImpliesIn(fact.sons[2], fact.sons[1], aSet)
+    if sameTree(fact[1], loc):
+      result = leImpliesIn(fact[1], fact[2], aSet)
+    elif sameTree(fact[2], loc):
+      result = geImpliesIn(fact[2], fact[1], aSet)
   of someLt:
-    if sameTree(fact.sons[1], loc):
-      result = leImpliesIn(fact.sons[1], fact.sons[2].pred, aSet)
-    elif sameTree(fact.sons[2], loc):
+    if sameTree(fact[1], loc):
+      result = leImpliesIn(fact[1], fact[2].pred, aSet)
+    elif sameTree(fact[2], loc):
       # 4 < x  -->  3 <= x
-      result = geImpliesIn(fact.sons[2], fact.sons[1].pred, aSet)
+      result = geImpliesIn(fact[2], fact[1].pred, aSet)
   of mNot, mOr, mAnd: assert(false, "impliesIn")
   else: discard
 
@@ -567,90 +578,93 @@ proc valueIsNil(n: PNode): TImplication =
   else: impUnknown
 
 proc impliesIsNil(fact, eq: PNode): TImplication =
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of mIsNil:
-    if sameTree(fact.sons[1], eq.sons[1]):
+    if sameTree(fact[1], eq[1]):
       result = impYes
   of someEq:
-    if sameTree(fact.sons[1], eq.sons[1]):
-      result = valueIsNil(fact.sons[2].skipConv)
-    elif sameTree(fact.sons[2], eq.sons[1]):
-      result = valueIsNil(fact.sons[1].skipConv)
+    if sameTree(fact[1], eq[1]):
+      result = valueIsNil(fact[2].skipConv)
+    elif sameTree(fact[2], eq[1]):
+      result = valueIsNil(fact[1].skipConv)
   of mNot, mOr, mAnd: assert(false, "impliesIsNil")
   else: discard
 
 proc impliesGe(fact, x, c: PNode): TImplication =
   assert isLocation(x)
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x = 4;  question x >= 56? --> true iff 4 >= 56
-        if leValue(c, fact.sons[2]): result = impYes
+        if leValue(c, fact[2]): result = impYes
         else: result = impNo
-    elif sameTree(fact.sons[2], x):
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impYes
+    elif sameTree(fact[2], x):
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impYes
         else: result = impNo
   of someLt:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x < 4;  question N <= x? --> false iff N <= 4
-        if leValue(fact.sons[2], c): result = impNo
+        if leValue(fact[2], c): result = impNo
         # fact:  x < 4;  question 2 <= x? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 < x; question: N-1 < x ?  --> true iff N-1 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c.pred, fact.sons[1]): result = impYes
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c.pred, fact[1]): result = impYes
   of someLe:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x <= 4;  question x >= 56? --> false iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impNo
+        if leValue(fact[2], c): result = impNo
         # fact:  x <= 4;  question x >= 2? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 <= x; question: x >= 2 ?  --> true iff 2 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impYes
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impYes
   of mNot, mOr, mAnd: assert(false, "impliesGe")
   else: discard
 
 proc impliesLe(fact, x, c: PNode): TImplication =
   if not isLocation(x):
+    if c.isValue:
+      if leValue(x, x): return impYes
+      else: return impNo
     return impliesGe(fact, c, x)
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x = 4;  question x <= 56? --> true iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impYes
+        if leValue(fact[2], c): result = impYes
         else: result = impNo
-    elif sameTree(fact.sons[2], x):
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(fact.sons[1], c): result = impYes
+    elif sameTree(fact[2], x):
+      if isValue(fact[1]) and isValue(c):
+        if leValue(fact[1], c): result = impYes
         else: result = impNo
   of someLt:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x < 4;  question x <= N? --> true iff N-1 <= 4
-        if leValue(fact.sons[2], c.pred): result = impYes
+        if leValue(fact[2], c.pred): result = impYes
         # fact:  x < 4;  question x <= 2? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 < x; question: x <= 1 ?  --> false iff 1 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impNo
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impNo
 
   of someLe:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x <= 4;  question x <= 56? --> true iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impYes
+        if leValue(fact[2], c): result = impYes
         # fact:  x <= 4;  question x <= 2? --> we don't know
 
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 <= x; question: x <= 2 ?  --> false iff 2 < 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1].pred): result = impNo
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1].pred): result = impNo
 
   of mNot, mOr, mAnd: assert(false, "impliesLe")
   else: discard
@@ -686,32 +700,32 @@ proc factImplies(fact, prop: PNode): TImplication =
 
     #  (not a) -> b  compute as  not (a -> b) ???
     #  == not a or not b == not (a and b)
-    let arg = fact.sons[1]
+    let arg = fact[1]
     case arg.getMagic
     of mIsNil, mEqRef:
       return ~factImplies(arg, prop)
     of mAnd:
       # not (a and b)  means  not a or not b:
       # a or b --> both need to imply 'prop'
-      let a = factImplies(arg.sons[1], prop)
-      let b = factImplies(arg.sons[2], prop)
+      let a = factImplies(arg[1], prop)
+      let b = factImplies(arg[2], prop)
       if a == b: return ~a
       return impUnknown
     else:
       return impUnknown
   of mAnd:
-    result = factImplies(fact.sons[1], prop)
+    result = factImplies(fact[1], prop)
     if result != impUnknown: return result
-    return factImplies(fact.sons[2], prop)
+    return factImplies(fact[2], prop)
   else: discard
 
-  case prop.sons[0].sym.magic
-  of mNot: result = ~fact.factImplies(prop.sons[1])
+  case prop[0].sym.magic
+  of mNot: result = ~fact.factImplies(prop[1])
   of mIsNil: result = impliesIsNil(fact, prop)
   of someEq: result = impliesEq(fact, prop)
-  of someLe: result = impliesLe(fact, prop.sons[1], prop.sons[2])
-  of someLt: result = impliesLt(fact, prop.sons[1], prop.sons[2])
-  of mInSet: result = impliesIn(fact, prop.sons[2], prop.sons[1])
+  of someLe: result = impliesLe(fact, prop[1], prop[2])
+  of someLt: result = impliesLt(fact, prop[1], prop[2])
+  of mInSet: result = impliesIn(fact, prop[2], prop[1])
   else: result = impUnknown
 
 proc doesImply*(facts: TModel, prop: PNode): TImplication =
@@ -802,7 +816,7 @@ proc ple(m: TModel; a, b: PNode): TImplication =
     if lastOrd(nil, a.typ) <= b.intVal: return impYes
   # 3 <= x   iff  low(x) <= 3
   if a.isValue and b.typ != nil and b.typ.isOrdinalType:
-    if firstOrd(nil, b.typ) <= a.intVal: return impYes
+    if a.intVal <= firstOrd(nil, b.typ): return impYes
 
   # x <= x
   if sameTree(a, b): return impYes
@@ -813,7 +827,11 @@ proc ple(m: TModel; a, b: PNode): TImplication =
 
   #   x <= y+c  if 0 <= c and x <= y
   #   x <= y+(-c)  if c <= 0  and y >= x
-  if b.getMagic in someAdd and zero() <=? b[2] and a <=? b[1]: return impYes
+  if b.getMagic in someAdd:
+    if zero() <=? b[2] and a <=? b[1]: return impYes
+    # x <= y-c  if x+c <= y
+    if b[2] <=? zero() and (canon(m.o.opSub.buildCall(a, b[2]), m.o) <=? b[1]):
+      return impYes
 
   #   x+c <= y  if c <= 0 and x <= y
   if a.getMagic in someAdd and a[2] <=? zero() and a[1] <=? b: return impYes
@@ -881,8 +899,8 @@ proc replaceSubTree(n, x, by: PNode): PNode =
     result = by
   elif hasSubTree(n, x):
     result = shallowCopy(n)
-    for i in 0 .. safeLen(n)-1:
-      result.sons[i] = replaceSubTree(n.sons[i], x, by)
+    for i in 0..n.safeLen-1:
+      result[i] = replaceSubTree(n[i], x, by)
   else:
     result = n
 
@@ -902,10 +920,13 @@ proc pleViaModelRec(m: var TModel; a, b: PNode): TImplication =
       # --> true  if  (len-100) <= (len-1)
       let x = fact[1]
       let y = fact[2]
-      if sameTree(x, a) and y.getMagic in someAdd and b.getMagic in someAdd and
-         sameTree(y[1], b[1]):
-        if ple(m, b[2], y[2]) == impYes:
-          return impYes
+      # x <= y.
+      # Question: x <= b? True iff y <= b.
+      if sameTree(x, a):
+        if ple(m, y, b) == impYes: return impYes
+        if y.getMagic in someAdd and b.getMagic in someAdd and sameTree(y[1], b[1]):
+          if ple(m, b[2], y[2]) == impYes:
+            return impYes
 
       # x <= y implies a <= b  if  a <= x and y <= b
       if ple(m, a, x) == impYes:
@@ -964,6 +985,10 @@ proc proveLe*(m: TModel; a, b: PNode): TImplication =
 proc addFactLe*(m: var TModel; a, b: PNode) =
   m.s.add canon(m.o.opLe.buildCall(a, b), m.o)
 
+proc addFactLt*(m: var TModel; a, b: PNode) =
+  let bb = m.o.opAdd.buildCall(b, minusOne())
+  addFactLe(m, a, bb)
+
 proc settype(n: PNode): PType =
   result = newType(tySet, n.typ.owner)
   addSonSkipIntLit(result, n.typ)
@@ -971,36 +996,37 @@ proc settype(n: PNode): PType =
 proc buildOf(it, loc: PNode; o: Operators): PNode =
   var s = newNodeI(nkCurly, it.info, it.len-1)
   s.typ = settype(loc)
-  for i in 0..it.len-2: s.sons[i] = it.sons[i]
+  for i in 0..<it.len-1: s[i] = it[i]
   result = newNodeI(nkCall, it.info, 3)
-  result.sons[0] = newSymNode(o.opContains)
-  result.sons[1] = s
-  result.sons[2] = loc
+  result[0] = newSymNode(o.opContains)
+  result[1] = s
+  result[2] = loc
 
 proc buildElse(n: PNode; o: Operators): PNode =
-  var s = newNodeIT(nkCurly, n.info, settype(n.sons[0]))
-  for i in 1..n.len-2:
-    let branch = n.sons[i]
-    assert branch.kind == nkOfBranch
-    for j in 0..branch.len-2:
-      s.add(branch.sons[j])
+  var s = newNodeIT(nkCurly, n.info, settype(n[0]))
+  for i in 1..<n.len-1:
+    let branch = n[i]
+    assert branch.kind != nkElse
+    if branch.kind == nkOfBranch:
+      for j in 0..<branch.len-1:
+        s.add(branch[j])
   result = newNodeI(nkCall, n.info, 3)
-  result.sons[0] = newSymNode(o.opContains)
-  result.sons[1] = s
-  result.sons[2] = n.sons[0]
+  result[0] = newSymNode(o.opContains)
+  result[1] = s
+  result[2] = n[0]
 
 proc addDiscriminantFact*(m: var TModel, n: PNode) =
   var fact = newNodeI(nkCall, n.info, 3)
-  fact.sons[0] = newSymNode(m.o.opEq)
-  fact.sons[1] = n.sons[0]
-  fact.sons[2] = n.sons[1]
+  fact[0] = newSymNode(m.o.opEq)
+  fact[1] = n[0]
+  fact[2] = n[1]
   m.s.add fact
 
 proc addAsgnFact*(m: var TModel, key, value: PNode) =
   var fact = newNodeI(nkCall, key.info, 3)
-  fact.sons[0] = newSymNode(m.o.opEq)
-  fact.sons[1] = key
-  fact.sons[2] = value
+  fact[0] = newSymNode(m.o.opEq)
+  fact[1] = key
+  fact[2] = value
   m.s.add fact
 
 proc sameSubexprs*(m: TModel; a, b: PNode): bool =
@@ -1014,34 +1040,34 @@ proc sameSubexprs*(m: TModel; a, b: PNode): bool =
   # However, nil checking requires exactly the same mechanism! But for now
   # we simply use sameTree and live with the unsoundness of the analysis.
   var check = newNodeI(nkCall, a.info, 3)
-  check.sons[0] = newSymNode(m.o.opEq)
-  check.sons[1] = a
-  check.sons[2] = b
+  check[0] = newSymNode(m.o.opEq)
+  check[1] = a
+  check[2] = b
   result = m.doesImply(check) == impYes
 
 proc addCaseBranchFacts*(m: var TModel, n: PNode, i: int) =
-  let branch = n.sons[i]
+  let branch = n[i]
   if branch.kind == nkOfBranch:
-    m.s.add buildOf(branch, n.sons[0], m.o)
+    m.s.add buildOf(branch, n[0], m.o)
   else:
     m.s.add n.buildElse(m.o).neg(m.o)
 
 proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
-  if check.sons[1].kind == nkCurly:
+  if check[1].kind == nkCurly:
     result = copyTree(check)
     if access.kind == nkDotExpr:
       var a = copyTree(access)
-      a.sons[1] = check.sons[2]
-      result.sons[2] = a
+      a[1] = check[2]
+      result[2] = a
       # 'access.kind != nkDotExpr' can happen for object constructors
       # which we don't check yet
   else:
     # it is some 'not'
     assert check.getMagic == mNot
-    result = buildProperFieldCheck(access, check.sons[1], o).neg(o)
+    result = buildProperFieldCheck(access, check[1], o).neg(o)
 
 proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef) =
-  for i in 1..n.len-1:
-    let check = buildProperFieldCheck(n.sons[0], n.sons[i], m.o)
+  for i in 1..<n.len:
+    let check = buildProperFieldCheck(n[0], n[i], m.o)
     if check != nil and m.doesImply(check) != impYes:
-      message(conf, n.info, warnProveField, renderTree(n.sons[0])); break
+      message(conf, n.info, warnProveField, renderTree(n[0])); break

compiler/hlo.nim

@@ -12,14 +12,14 @@
 proc hlo(c: PContext, n: PNode): PNode
 
 proc evalPattern(c: PContext, n, orig: PNode): PNode =
-  internalAssert c.config, n.kind == nkCall and n.sons[0].kind == nkSym
+  internalAssert c.config, n.kind == nkCall and n[0].kind == nkSym
   # we need to ensure that the resulting AST is semchecked. However, it's
-  # aweful to semcheck before macro invocation, so we don't and treat
+  # awful to semcheck before macro invocation, so we don't and treat
   # templates and macros as immediate in this context.
   var rule: string
-  if optHints in c.config.options and hintPattern in c.config.notes:
+  if c.config.hasHint(hintPattern):
     rule = renderTree(n, {renderNoComments})
-  let s = n.sons[0].sym
+  let s = n[0].sym
   case s.kind
   of skMacro:
     result = semMacroExpr(c, n, orig, s)
@@ -27,7 +27,7 @@ proc evalPattern(c: PContext, n, orig: PNode): PNode =
     result = semTemplateExpr(c, n, s, {efFromHlo})
   else:
     result = semDirectOp(c, n, {})
-  if optHints in c.config.options and hintPattern in c.config.notes:
+  if c.config.hasHint(hintPattern):
     message(c.config, orig.info, hintPattern, rule & " --> '" &
       renderTree(result, {renderNoComments}) & "'")
 
@@ -50,7 +50,7 @@ proc applyPatterns(c: PContext, n: PNode): PNode =
         c.patterns[i] = nil
         if x.kind == nkStmtList:
           assert x.len == 3
-          x.sons[1] = evalPattern(c, x.sons[1], result)
+          x[1] = evalPattern(c, x[1], result)
           result = flattenStmts(x)
         else:
           result = evalPattern(c, x, result)
@@ -68,17 +68,17 @@ proc hlo(c: PContext, n: PNode): PNode =
     result = n
   else:
     if n.kind in {nkFastAsgn, nkAsgn, nkIdentDefs, nkVarTuple} and
-        n.sons[0].kind == nkSym and
-        {sfGlobal, sfPure} * n.sons[0].sym.flags == {sfGlobal, sfPure}:
+        n[0].kind == nkSym and
+        {sfGlobal, sfPure} * n[0].sym.flags == {sfGlobal, sfPure}:
       # do not optimize 'var g {.global} = re(...)' again!
       return n
     result = applyPatterns(c, n)
     if result == n:
       # no optimization applied, try subtrees:
-      for i in 0 ..< safeLen(result):
-        let a = result.sons[i]
+      for i in 0..<result.safeLen:
+        let a = result[i]
         let h = hlo(c, a)
-        if h != a: result.sons[i] = h
+        if h != a: result[i] = h
     else:
       # perform type checking, so that the replacement still fits:
       if isEmptyType(n.typ) and isEmptyType(result.typ):

compiler/idents.nim

@@ -12,7 +12,7 @@
 # id. This module is essential for the compiler's performance.
 
 import
-  hashes, strutils, wordrecg
+  hashes, wordrecg
 
 type
   TIdObj* = object of RootObj
@@ -87,7 +87,7 @@ proc getIdent*(ic: IdentCache; identifier: cstring, length: int, h: Hash): PIden
   new(result)
   result.h = h
   result.s = newString(length)
-  for i in 0 ..< length: result.s[i] = identifier[i]
+  for i in 0..<length: result.s[i] = identifier[i]
   result.next = ic.buckets[idx]
   ic.buckets[idx] = result
   if id == 0:
@@ -97,11 +97,11 @@ proc getIdent*(ic: IdentCache; identifier: cstring, length: int, h: Hash): PIden
     result.id = id
 
 proc getIdent*(ic: IdentCache; identifier: string): PIdent =
-  result = getIdent(ic, cstring(identifier), len(identifier),
+  result = getIdent(ic, cstring(identifier), identifier.len,
                     hashIgnoreStyle(identifier))
 
 proc getIdent*(ic: IdentCache; identifier: string, h: Hash): PIdent =
-  result = getIdent(ic, cstring(identifier), len(identifier), h)
+  result = getIdent(ic, cstring(identifier), identifier.len, h)
 
 proc newIdentCache*(): IdentCache =
   result = IdentCache()
@@ -110,7 +110,7 @@ proc newIdentCache*(): IdentCache =
   result.idDelegator = result.getIdent":delegator"
   result.emptyIdent = result.getIdent("")
   # initialize the keywords:
-  for s in succ(low(specialWords)) .. high(specialWords):
+  for s in succ(low(specialWords))..high(specialWords):
     result.getIdent(specialWords[s], hashIgnoreStyle(specialWords[s])).id = ord(s)
 
 proc whichKeyword*(id: PIdent): TSpecialWord =

compiler/idgen.nim

@@ -9,7 +9,7 @@
 
 ## This module contains a simple persistent id generator.
 
-import idents, strutils, os, options, pathutils
+import idents, strutils, options, pathutils
 
 var gFrontEndId*: int
 

compiler/importer.nim

@@ -10,13 +10,13 @@
 ## This module implements the symbol importing mechanism.
 
 import
-  intsets, strutils, os, ast, astalgo, msgs, options, idents, lookups,
-  semdata, passes, renderer, modulepaths, sigmatch, lineinfos
+  intsets, ast, astalgo, msgs, options, idents, lookups,
+  semdata, modulepaths, sigmatch, lineinfos, sets
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
   result = initIntSet()
-  for i in 1 ..< n.len:
+  for i in 1..<n.len:
     let ident = lookups.considerQuotedIdent(c, n[i])
     result.incl(ident.id)
 
@@ -27,11 +27,11 @@ proc importPureEnumField*(c: PContext; s: PSym) =
     if checkB == nil:
       strTableAdd(c.pureEnumFields, s)
     else:
-      # mark as ambigous:
+      # mark as ambiguous:
       incl(c.ambiguousSymbols, checkB.id)
       incl(c.ambiguousSymbols, s.id)
 
-proc rawImportSymbol(c: PContext, s: PSym) =
+proc rawImportSymbol(c: PContext, s, origin: PSym) =
   # This does not handle stubs, because otherwise loading on demand would be
   # pointless in practice. So importing stubs is fine here!
   # check if we have already a symbol of the same name:
@@ -47,8 +47,8 @@ proc rawImportSymbol(c: PContext, s: PSym) =
   if s.kind == skType:
     var etyp = s.typ
     if etyp.kind in {tyBool, tyEnum}:
-      for j in 0 ..< sonsLen(etyp.n):
-        var e = etyp.n.sons[j].sym
+      for j in 0..<etyp.n.len:
+        var e = etyp.n[j].sym
         if e.kind != skEnumField:
           internalError(c.config, s.info, "rawImportSymbol")
           # BUGFIX: because of aliases for enums the symbol may already
@@ -63,13 +63,14 @@ proc rawImportSymbol(c: PContext, s: PSym) =
           check = nextIdentIter(it, c.importTable.symbols)
         if e != nil:
           if sfPure notin s.flags:
-            rawImportSymbol(c, e)
+            rawImportSymbol(c, e, origin)
           else:
             importPureEnumField(c, e)
   else:
-    # rodgen assures that converters and patterns are no stubs
     if s.kind == skConverter: addConverter(c, s)
     if hasPattern(s): addPattern(c, s)
+  if s.owner != origin:
+    c.exportIndirections.incl((origin.id, s.id))
 
 proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
   let ident = lookups.considerQuotedIdent(c, n)
@@ -79,19 +80,19 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
   else:
     when false:
       if s.kind == skStub: loadStub(s)
-    if s.kind notin ExportableSymKinds:
-      internalError(c.config, n.info, "importSymbol: 2")
+    let multiImport = s.kind notin ExportableSymKinds or s.kind in skProcKinds
     # for an enumeration we have to add all identifiers
-    case s.kind
-    of skProcKinds:
+    if multiImport:
       # for a overloadable syms add all overloaded routines
       var it: TIdentIter
       var e = initIdentIter(it, fromMod.tab, s.name)
       while e != nil:
         if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3")
-        rawImportSymbol(c, e)
+        if s.kind in ExportableSymKinds:
+          rawImportSymbol(c, e, fromMod)
         e = nextIdentIter(it, fromMod.tab)
-    else: rawImportSymbol(c, s)
+    else:
+      rawImportSymbol(c, s, fromMod)
     suggestSym(c.config, n.info, s, c.graph.usageSym, false)
 
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
@@ -103,14 +104,14 @@ proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
         if s.kind notin ExportableSymKinds:
           internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s)
         if exceptSet.isNil or s.name.id notin exceptSet:
-          rawImportSymbol(c, s)
+          rawImportSymbol(c, s, fromMod)
     s = nextIter(i, fromMod.tab)
 
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
   var exceptSet: IntSet
   importAllSymbolsExcept(c, fromMod, exceptSet)
 
-proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet) =
+proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym) =
   if n.isNil: return
   case n.kind
   of nkExportStmt:
@@ -120,38 +121,43 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet) =
       if s.kind == skModule:
         importAllSymbolsExcept(c, s, exceptSet)
       elif exceptSet.isNil or s.name.id notin exceptSet:
-        rawImportSymbol(c, s)
+        rawImportSymbol(c, s, fromMod)
   of nkExportExceptStmt:
     localError(c.config, n.info, "'export except' not implemented")
   else:
-    for i in 0..safeLen(n)-1:
-      importForwarded(c, n.sons[i], exceptSet)
+    for i in 0..n.safeLen-1:
+      importForwarded(c, n[i], exceptSet, fromMod)
 
 proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
   result = realModule
+  c.unusedImports.add((realModule, n.info))
   if n.kind != nkImportAs: discard
-  elif n.len != 2 or n.sons[1].kind != nkIdent:
+  elif n.len != 2 or n[1].kind != nkIdent:
     localError(c.config, n.info, "module alias must be an identifier")
-  elif n.sons[1].ident.id != realModule.name.id:
+  elif n[1].ident.id != realModule.name.id:
     # some misguided guy will write 'import abc.foo as foo' ...
-    result = createModuleAlias(realModule, n.sons[1].ident, realModule.info,
+    result = createModuleAlias(realModule, n[1].ident, realModule.info,
                                c.config.options)
 
 proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
   let f = checkModuleName(c.config, n)
-  if f != InvalidFileIDX:
+  if f != InvalidFileIdx:
     let L = c.graph.importStack.len
     let recursion = c.graph.importStack.find(f)
     c.graph.importStack.add f
     #echo "adding ", toFullPath(f), " at ", L+1
     if recursion >= 0:
       var err = ""
-      for i in recursion ..< L:
+      for i in recursion..<L:
         if i > recursion: err.add "\n"
         err.add toFullPath(c.config, c.graph.importStack[i]) & " imports " &
                 toFullPath(c.config, c.graph.importStack[i+1])
       c.recursiveDep = err
+
+    discard pushOptionEntry(c)
     result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f))
+    popOptionEntry(c)
+
     #echo "set back to ", L
     c.graph.importStack.setLen(L)
     # we cannot perform this check reliably because of
@@ -172,8 +178,8 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
 proc transformImportAs(c: PContext; n: PNode): PNode =
   if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
     result = newNodeI(nkImportAs, n.info)
-    result.add n.sons[1]
-    result.add n.sons[2]
+    result.add n[1]
+    result.add n[2]
   else:
     result = n
 
@@ -185,12 +191,12 @@ proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
     # ``addDecl`` needs to be done before ``importAllSymbols``!
     addDecl(c, m, it.info) # add symbol to symbol table of module
     importAllSymbolsExcept(c, m, emptySet)
-    #importForwarded(c, m.ast, emptySet)
+    #importForwarded(c, m.ast, emptySet, m)
 
 proc evalImport*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
-  for i in 0 ..< sonsLen(n):
-    let it = n.sons[i]
+  for i in 0..<n.len:
+    let it = n[i]
     if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
       let sep = it[0]
       let dir = it[1]
@@ -200,13 +206,13 @@ proc evalImport*(c: PContext, n: PNode): PNode =
       imp.add sep # dummy entry, replaced in the loop
       for x in it[2]:
         # transform `a/b/[c as d]` to `/a/b/c as d`
-        if x.kind == nkInfix and x.sons[0].ident.s == "as":
+        if x.kind == nkInfix and x[0].ident.s == "as":
           let impAs = copyTree(x)
-          imp.sons[2] = x.sons[1]
-          impAs.sons[1] = imp
+          imp[2] = x[1]
+          impAs[1] = imp
           impMod(c, imp, result)
         else:
-          imp.sons[2] = x
+          imp[2] = x
           impMod(c, imp, result)
     else:
       impMod(c, it, result)
@@ -214,22 +220,22 @@ proc evalImport*(c: PContext, n: PNode): PNode =
 proc evalFrom*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n.sons[0] = transformImportAs(c, n.sons[0])
-  var m = myImportModule(c, n.sons[0], result)
+  n[0] = transformImportAs(c, n[0])
+  var m = myImportModule(c, n[0], result)
   if m != nil:
-    n.sons[0] = newSymNode(m)
+    n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
-    for i in 1 ..< sonsLen(n):
-      if n.sons[i].kind != nkNilLit:
-        importSymbol(c, n.sons[i], m)
+    for i in 1..<n.len:
+      if n[i].kind != nkNilLit:
+        importSymbol(c, n[i], m)
 
 proc evalImportExcept*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n.sons[0] = transformImportAs(c, n.sons[0])
-  var m = myImportModule(c, n.sons[0], result)
+  n[0] = transformImportAs(c, n[0])
+  var m = myImportModule(c, n[0], result)
   if m != nil:
-    n.sons[0] = newSymNode(m)
+    n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
     importAllSymbolsExcept(c, m, readExceptSet(c, n))
-    #importForwarded(c, m.ast, exceptSet)
+    #importForwarded(c, m.ast, exceptSet, m)

compiler/incremental.nim

@@ -12,11 +12,12 @@
 
 const nimIncremental* = defined(nimIncremental)
 
-import options, lineinfos, pathutils
+import options, lineinfos
 
 when nimIncremental:
-  import ast, msgs, intsets, btrees, db_sqlite, std / sha1
+  import ast, msgs, intsets, btrees, db_sqlite, std / sha1, pathutils
   from strutils import parseInt
+  from os import isAbsolute
 
   type
     Writer* = object
@@ -47,7 +48,7 @@ when nimIncremental:
 
   proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: AbsoluteFile): string =
     result = msgs.getHash(conf, fileIdx)
-    if result.len == 0:
+    if result.len == 0 and isAbsolute(string fullpath):
       result = $secureHashFile(string fullpath)
       msgs.setHash(conf, fileIdx, result)
 

compiler/installer.ini

@@ -6,10 +6,10 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv64
+  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv64
   macosx: i386;amd64;powerpc64
   solaris: i386;amd64;sparc;sparc64
-  freebsd: i386;amd64
+  freebsd: i386;amd64;powerpc64;arm;arm64
   netbsd: i386;amd64
   openbsd: i386;amd64
   dragonfly: i386;amd64
@@ -65,6 +65,7 @@ Files: "compiler"
 Files: "doc"
 Files: "doc/html"
 Files: "tools"
+Files: "tools/nim-gdb.py"
 Files: "nimpretty"
 Files: "testament"
 Files: "nimsuggest"
@@ -76,6 +77,7 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/fusion"
 
 Files: "tests"
 
@@ -86,6 +88,9 @@ Files: "bin/nimsuggest.exe"
 Files: "bin/nimble.exe"
 Files: "bin/vccexe.exe"
 Files: "bin/nimgrab.exe"
+Files: "bin/nimpretty.exe"
+Files: "bin/testament.exe"
+Files: "bin/nim-gdb.bat"
 
 Files: "koch.exe"
 Files: "finish.exe"
@@ -114,6 +119,8 @@ Files: "bin/nim"
 [Unix]
 InstallScript: "yes"
 UninstallScript: "yes"
+Files: "bin/nim-gdb"
+Files: "bin/nim-gdb.bash"
 
 
 [InnoSetup]
@@ -136,4 +143,4 @@ licenses: "bin/nim,MIT;lib/*,MIT;"
 
 [nimble]
 pkgName: "compiler"
-pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt"
+pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt;doc/nimdoc.css"

compiler/int128.nim

@@ -0,0 +1,781 @@
+## This module is for compiler internal use only. For reliable error
+## messages and range checks, the compiler needs a data type that can
+## hold all from ``low(BiggestInt)`` to ``high(BiggestUInt)``, This
+## type is for that purpose.
+
+from math import trunc
+
+type
+  Int128* = object
+    udata: array[4,uint32]
+
+template sdata(arg: Int128, idx: int): int32 =
+  # udata and sdata was supposed to be in a union, but unions are
+  # handled incorrectly in the VM.
+  cast[ptr int32](arg.udata[idx].unsafeAddr)[]
+
+# encoding least significant int first (like LittleEndian)
+
+const
+  Zero* = Int128(udata: [0'u32,0,0,0])
+  One* = Int128(udata: [1'u32,0,0,0])
+  Ten* = Int128(udata: [10'u32,0,0,0])
+  Min = Int128(udata: [0'u32,0,0,0x80000000'u32])
+  Max = Int128(udata: [high(uint32),high(uint32),high(uint32),uint32(high(int32))])
+  NegOne* = Int128(udata: [0xffffffff'u32,0xffffffff'u32,0xffffffff'u32,0xffffffff'u32])
+
+template low*(t: typedesc[Int128]): Int128 = Min
+template high*(t: typedesc[Int128]): Int128 = Max
+
+proc `$`*(a: Int128): string
+
+proc toInt128*[T: SomeInteger | bool](arg: T): Int128 =
+  when T is bool: result.sdata(0) = int32(arg)
+  elif T is SomeUnsignedInt:
+    when sizeof(arg) <= 4:
+      result.udata[0] = uint32(arg)
+    else:
+      result.udata[0] = uint32(arg and T(0xffffffff))
+      result.udata[1] = uint32(arg shr 32)
+  elif sizeof(arg) <= 4:
+    result.sdata(0) = int32(arg)
+    if arg < 0: # sign extend
+      result.sdata(1) = -1
+      result.sdata(2) = -1
+      result.sdata(3) = -1
+  else:
+    let tmp = int64(arg)
+    result.udata[0] = uint32(tmp and 0xffffffff)
+    result.sdata(1) = int32(tmp shr 32)
+    if arg < 0: # sign extend
+      result.sdata(2) = -1
+      result.sdata(3) = -1
+
+template isNegative(arg: Int128): bool =
+  arg.sdata(3) < 0
+
+template isNegative(arg: int32): bool =
+  arg < 0
+
+proc bitconcat(a,b: uint32): uint64 =
+  (uint64(a) shl 32) or uint64(b)
+
+proc bitsplit(a: uint64): (uint32,uint32) =
+  (cast[uint32](a shr 32), cast[uint32](a))
+
+proc toInt64*(arg: Int128): int64 =
+  if isNegative(arg):
+    assert(arg.sdata(3) == -1, "out of range")
+    assert(arg.sdata(2) == -1, "out of range")
+  else:
+    assert(arg.sdata(3) == 0, "out of range")
+    assert(arg.sdata(2) == 0, "out of range")
+
+  cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
+
+proc toInt64Checked*(arg: Int128; onError: int64): int64 =
+  if isNegative(arg):
+    if arg.sdata(3) != -1 or arg.sdata(2) != -1:
+      return onError
+  else:
+    if arg.sdata(3) != 0 or arg.sdata(2) != 0:
+      return onError
+  return cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
+
+proc toInt32*(arg: Int128): int32 =
+  if isNegative(arg):
+    assert(arg.sdata(3) == -1, "out of range")
+    assert(arg.sdata(2) == -1, "out of range")
+    assert(arg.sdata(1) == -1, "out of range")
+  else:
+    assert(arg.sdata(3) == 0, "out of range")
+    assert(arg.sdata(2) == 0, "out of range")
+    assert(arg.sdata(1) == 0, "out of range")
+
+  arg.sdata(0)
+
+proc toInt16*(arg: Int128): int16 =
+  if isNegative(arg):
+    assert(arg.sdata(3) == -1, "out of range")
+    assert(arg.sdata(2) == -1, "out of range")
+    assert(arg.sdata(1) == -1, "out of range")
+  else:
+    assert(arg.sdata(3) == 0, "out of range")
+    assert(arg.sdata(2) == 0, "out of range")
+    assert(arg.sdata(1) == 0, "out of range")
+
+  int16(arg.sdata(0))
+
+proc toInt8*(arg: Int128): int8 =
+  if isNegative(arg):
+    assert(arg.sdata(3) == -1, "out of range")
+    assert(arg.sdata(2) == -1, "out of range")
+    assert(arg.sdata(1) == -1, "out of range")
+  else:
+    assert(arg.sdata(3) == 0, "out of range")
+    assert(arg.sdata(2) == 0, "out of range")
+    assert(arg.sdata(1) == 0, "out of range")
+
+  int8(arg.sdata(0))
+
+proc toInt*(arg: Int128): int =
+  when sizeof(int) == 4:
+    cast[int](toInt32(arg))
+  else:
+    cast[int](toInt64(arg))
+
+proc toUInt64*(arg: Int128): uint64 =
+  assert(arg.udata[3] == 0)
+  assert(arg.udata[2] == 0)
+  bitconcat(arg.udata[1], arg.udata[0])
+
+proc toUInt32*(arg: Int128): uint32 =
+  assert(arg.udata[3] == 0)
+  assert(arg.udata[2] == 0)
+  assert(arg.udata[1] == 0)
+  arg.udata[0]
+
+proc toUInt16*(arg: Int128): uint16 =
+  assert(arg.udata[3] == 0)
+  assert(arg.udata[2] == 0)
+  assert(arg.udata[1] == 0)
+  uint16(arg.udata[0])
+
+proc toUInt8*(arg: Int128): uint8 =
+  assert(arg.udata[3] == 0)
+  assert(arg.udata[2] == 0)
+  assert(arg.udata[1] == 0)
+  uint8(arg.udata[0])
+
+proc toUInt*(arg: Int128): uint =
+  when sizeof(int) == 4:
+    cast[uint](toInt32(arg))
+  else:
+    cast[uint](toInt64(arg))
+
+proc castToInt64*(arg: Int128): int64 =
+  ## Conversion to int64 without range check.
+  cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
+
+proc castToUInt64*(arg: Int128): uint64 =
+  ## Conversion to uint64 without range check.
+  cast[uint64](bitconcat(arg.udata[1], arg.udata[0]))
+
+proc addToHex(result: var string; arg: uint32) =
+  for i in 0..<8:
+    let idx = (arg shr ((7-i) * 4)) and 0xf
+    result.add "0123456789abcdef"[idx]
+
+proc addToHex*(result: var string; arg: Int128) =
+  var i = 3
+  while i >= 0:
+    result.addToHex(arg.udata[i])
+    i -= 1
+
+proc toHex*(arg: Int128): string =
+  result.addToHex(arg)
+
+proc inc*(a: var Int128, y: uint32 = 1) =
+  let input = a
+  a.udata[0] += y
+  if unlikely(a.udata[0] < y):
+    a.udata[1].inc
+    if unlikely(a.udata[1] == 0):
+      a.udata[2].inc
+      if unlikely(a.udata[2] == 0):
+        a.udata[3].inc
+        doAssert(a.sdata(3) != low(int32), "overflow")
+
+proc cmp*(a,b: Int128): int =
+  let tmp1 = cmp(a.sdata(3), b.sdata(3))
+  if tmp1 != 0: return tmp1
+  let tmp2 = cmp(a.udata[2], b.udata[2])
+  if tmp2 != 0: return tmp2
+  let tmp3 = cmp(a.udata[1], b.udata[1])
+  if tmp3 != 0: return tmp3
+  let tmp4 = cmp(a.udata[0], b.udata[0])
+  return tmp4
+
+proc `<`*(a,b: Int128): bool =
+  cmp(a,b) < 0
+
+proc `<=`*(a,b: Int128): bool =
+  cmp(a,b) <= 0
+
+proc `==`*(a,b: Int128): bool =
+  if a.udata[0] != b.udata[0]: return false
+  if a.udata[1] != b.udata[1]: return false
+  if a.udata[2] != b.udata[2]: return false
+  if a.udata[3] != b.udata[3]: return false
+  return true
+
+proc inplaceBitnot(a: var Int128) =
+  a.udata[0] = not a.udata[0]
+  a.udata[1] = not a.udata[1]
+  a.udata[2] = not a.udata[2]
+  a.udata[3] = not a.udata[3]
+
+proc bitnot*(a: Int128): Int128 =
+  result.udata[0] = not a.udata[0]
+  result.udata[1] = not a.udata[1]
+  result.udata[2] = not a.udata[2]
+  result.udata[3] = not a.udata[3]
+
+proc bitand*(a,b: Int128): Int128 =
+  result.udata[0] = a.udata[0] and b.udata[0]
+  result.udata[1] = a.udata[1] and b.udata[1]
+  result.udata[2] = a.udata[2] and b.udata[2]
+  result.udata[3] = a.udata[3] and b.udata[3]
+
+proc bitor*(a,b: Int128): Int128 =
+  result.udata[0] = a.udata[0] or b.udata[0]
+  result.udata[1] = a.udata[1] or b.udata[1]
+  result.udata[2] = a.udata[2] or b.udata[2]
+  result.udata[3] = a.udata[3] or b.udata[3]
+
+proc bitxor*(a,b: Int128): Int128 =
+  result.udata[0] = a.udata[0] xor b.udata[0]
+  result.udata[1] = a.udata[1] xor b.udata[1]
+  result.udata[2] = a.udata[2] xor b.udata[2]
+  result.udata[3] = a.udata[3] xor b.udata[3]
+
+proc `shr`*(a: Int128, b: int): Int128 =
+  let b = b and 127
+  if b < 32:
+    result.sdata(3) = a.sdata(3) shr b
+    result.udata[2] = cast[uint32](bitconcat(a.udata[3], a.udata[2]) shr b)
+    result.udata[1] = cast[uint32](bitconcat(a.udata[2], a.udata[1]) shr b)
+    result.udata[0] = cast[uint32](bitconcat(a.udata[1], a.udata[0]) shr b)
+  elif b < 64:
+    if isNegative(a):
+      result.sdata(3) = -1
+    result.sdata(2) = a.sdata(3) shr (b and 31)
+    result.udata[1] = cast[uint32](bitconcat(a.udata[3], a.udata[2]) shr (b and 31))
+    result.udata[0] = cast[uint32](bitconcat(a.udata[2], a.udata[1]) shr (b and 31))
+  elif b < 96:
+    if isNegative(a):
+      result.sdata(3) = -1
+      result.sdata(2) = -1
+    result.sdata(1) = a.sdata(3) shr (b and 31)
+    result.udata[0] = cast[uint32](bitconcat(a.udata[3], a.udata[2]) shr (b and 31))
+  else: # b < 128
+    if isNegative(a):
+      result.sdata(3) = -1
+      result.sdata(2) = -1
+      result.sdata(1) = -1
+    result.sdata(0) = a.sdata(3) shr (b and 31)
+
+proc `shl`*(a: Int128, b: int): Int128 =
+  let b = b and 127
+  if b < 32:
+    result.udata[0] = a.udata[0] shl b
+    result.udata[1] = cast[uint32]((bitconcat(a.udata[1], a.udata[0]) shl b) shr 32)
+    result.udata[2] = cast[uint32]((bitconcat(a.udata[2], a.udata[1]) shl b) shr 32)
+    result.udata[3] = cast[uint32]((bitconcat(a.udata[3], a.udata[2]) shl b) shr 32)
+  elif b < 64:
+    result.udata[0] = 0
+    result.udata[1] = a.udata[0] shl (b and 31)
+    result.udata[2] = cast[uint32]((bitconcat(a.udata[1], a.udata[0]) shl (b and 31)) shr 32)
+    result.udata[3] = cast[uint32]((bitconcat(a.udata[2], a.udata[1]) shl (b and 31)) shr 32)
+  elif b < 96:
+    result.udata[0] = 0
+    result.udata[1] = 0
+    result.udata[2] = a.udata[0] shl (b and 31)
+    result.udata[3] = cast[uint32]((bitconcat(a.udata[1], a.udata[0]) shl (b and 31)) shr 32)
+  else:
+    result.udata[0] = 0
+    result.udata[1] = 0
+    result.udata[2] = 0
+    result.udata[3] = a.udata[0] shl (b and 31)
+
+proc `+`*(a,b: Int128): Int128 =
+  let tmp0 = uint64(a.udata[0]) + uint64(b.udata[0])
+  result.udata[0] = cast[uint32](tmp0)
+  let tmp1 = uint64(a.udata[1]) + uint64(b.udata[1]) + (tmp0 shr 32)
+  result.udata[1] = cast[uint32](tmp1)
+  let tmp2 = uint64(a.udata[2]) + uint64(b.udata[2]) + (tmp1 shr 32)
+  result.udata[2] = cast[uint32](tmp2)
+  let tmp3 = uint64(a.udata[3]) + uint64(b.udata[3]) + (tmp2 shr 32)
+  result.udata[3] = cast[uint32](tmp3)
+
+proc `+=`*(a: var Int128, b: Int128) =
+  a = a + b
+
+proc `-`*(a: Int128): Int128 =
+  result = bitnot(a)
+  result.inc
+
+proc `-`*(a,b: Int128): Int128 =
+  a + (-b)
+
+proc `-=`*(a: var Int128, b: Int128) =
+  a = a - b
+
+proc abs*(a: Int128): Int128 =
+  if isNegative(a):
+    -a
+  else:
+    a
+
+proc abs(a: int32): int =
+  if a < 0: -a else: a
+
+proc `*`(a: Int128, b: uint32): Int128 =
+  let tmp0 = uint64(a.udata[0]) * uint64(b)
+  let tmp1 = uint64(a.udata[1]) * uint64(b)
+  let tmp2 = uint64(a.udata[2]) * uint64(b)
+  let tmp3 = uint64(a.udata[3]) * uint64(b)
+
+  if unlikely(tmp3 > uint64(high(int32))):
+    assert(false, "overflow")
+
+  result.udata[0] = cast[uint32](tmp0)
+  result.udata[1] = cast[uint32](tmp1) + cast[uint32](tmp0 shr 32)
+  result.udata[2] = cast[uint32](tmp2) + cast[uint32](tmp1 shr 32)
+  result.udata[3] = cast[uint32](tmp3) + cast[uint32](tmp2 shr 32)
+
+proc `*`*(a: Int128, b: int32): Int128 =
+  result = a * cast[uint32](abs(b))
+  if b < 0:
+    result = -result
+
+proc `*=`*(a: var Int128, b: int32): Int128 =
+  result = result * b
+
+proc makeInt128(high,low: uint64): Int128 =
+  result.udata[0] = cast[uint32](low)
+  result.udata[1] = cast[uint32](low shr 32)
+  result.udata[2] = cast[uint32](high)
+  result.udata[3] = cast[uint32](high shr 32)
+
+proc high64(a: Int128): uint64 =
+  bitconcat(a.udata[3], a.udata[2])
+
+proc low64(a: Int128): uint64 =
+  bitconcat(a.udata[1], a.udata[0])
+
+proc `*`*(lhs,rhs: Int128): Int128 =
+  let
+    a = cast[uint64](lhs.udata[0])
+    b = cast[uint64](lhs.udata[1])
+    c = cast[uint64](lhs.udata[2])
+    d = cast[uint64](lhs.udata[3])
+
+    e = cast[uint64](rhs.udata[0])
+    f = cast[uint64](rhs.udata[1])
+    g = cast[uint64](rhs.udata[2])
+    h = cast[uint64](rhs.udata[3])
+
+
+  let a32 = cast[uint64](lhs.udata[1])
+  let a00 = cast[uint64](lhs.udata[0])
+  let b32 = cast[uint64](rhs.udata[1])
+  let b00 = cast[uint64](rhs.udata[0])
+
+  result = makeInt128(high64(lhs) * low64(rhs) + low64(lhs) * high64(rhs) + a32 * b32, a00 * b00)
+  result = result + toInt128(a32 * b00) shl 32
+  result = result + toInt128(a00 * b32) shl 32
+
+proc `*=`*(a: var Int128, b: Int128) =
+  a = a * b
+
+import bitops
+
+proc fastLog2*(a: Int128): int =
+  if a.udata[3] != 0:
+    return 96 + fastLog2(a.udata[3])
+  if a.udata[2] != 0:
+    return 64 + fastLog2(a.udata[2])
+  if a.udata[1] != 0:
+    return 32 + fastLog2(a.udata[1])
+  if a.udata[0] != 0:
+    return      fastLog2(a.udata[0])
+
+proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
+  assert(divisor != Zero)
+  let isNegativeA = isNegative(dividend)
+  let isNegativeB = isNegative(divisor)
+
+  var dividend = abs(dividend)
+  let divisor = abs(divisor)
+
+  if divisor > dividend:
+    result.quotient = Zero
+    if isNegativeA:
+      result.remainder = -dividend
+    else:
+      result.remainder = dividend
+    return
+
+  if divisor == dividend:
+    if isNegativeA xor isNegativeB:
+      result.quotient = NegOne
+    else:
+      result.quotient = One
+    result.remainder = Zero
+    return
+
+  var denominator = divisor
+  var quotient = Zero
+
+  # Left aligns the MSB of the denominator and the dividend.
+  let shift = fastLog2(dividend) - fastLog2(denominator)
+  denominator = denominator shl shift
+
+  # Uses shift-subtract algorithm to divide dividend by denominator. The
+  # remainder will be left in dividend.
+  for i in 0..shift:
+    quotient = quotient shl 1
+    if dividend >= denominator:
+      dividend = dividend - denominator
+      quotient = bitor(quotient, One)
+
+    denominator = denominator shr 1
+
+  if isNegativeA xor isNegativeB:
+    result.quotient = -quotient
+  else:
+    result.quotient = quotient
+  if isNegativeA:
+    result.remainder = -dividend
+  else:
+    result.remainder = dividend
+
+proc `div`*(a,b: Int128): Int128 =
+  let (a,b) = divMod(a,b)
+  return a
+
+proc `mod`*(a,b: Int128): Int128 =
+  let (a,b) = divMod(a,b)
+  return b
+
+proc addInt128*(result: var string; value: Int128) =
+  let initialSize = result.len
+  if value == Zero:
+    result.add "0"
+  elif value == low(Int128):
+    result.add "-170141183460469231731687303715884105728"
+  else:
+    let isNegative = isNegative(value)
+    var value = abs(value)
+    while value > Zero:
+      let (quot, rem) = divMod(value, Ten)
+      result.add "0123456789"[rem.toInt64]
+      value = quot
+    if isNegative:
+      result.add '-'
+
+    var i = initialSize
+    var j = high(result)
+    while i < j:
+      swap(result[i], result[j])
+      i += 1
+      j -= 1
+
+proc `$`*(a: Int128): string =
+  result.addInt128(a)
+
+proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =
+  assert(pos < arg.len)
+  assert(arg[pos] in {'-','0'..'9'})
+
+  var isNegative = false
+  var pos = pos
+  if arg[pos] == '-':
+    isNegative = true
+    pos += 1
+
+  result = Zero
+  while pos < arg.len and arg[pos] in '0'..'9':
+    result = result * Ten
+    result.inc(uint32(arg[pos]) - uint32('0'))
+    pos += 1
+
+  if isNegative:
+    result = -result
+
+# fluff
+
+proc `<`*(a: Int128, b: BiggestInt): bool =
+  cmp(a,toInt128(b)) < 0
+
+proc `<`*(a: BiggestInt, b: Int128): bool =
+  cmp(toInt128(a), b) < 0
+
+proc `<=`*(a: Int128, b: BiggestInt): bool =
+  cmp(a,toInt128(b)) <= 0
+
+proc `<=`*(a: BiggestInt, b: Int128): bool =
+  cmp(toInt128(a), b) <= 0
+
+proc `==`*(a: Int128, b: BiggestInt): bool =
+  a == toInt128(b)
+
+proc `==`*(a: BiggestInt, b: Int128): bool =
+  toInt128(a) == b
+
+proc `-`*(a: BiggestInt, b: Int128): Int128 =
+  toInt128(a) - b
+
+proc `-`*(a: Int128, b: BiggestInt): Int128 =
+  a - toInt128(b)
+
+proc `+`*(a: BiggestInt, b: Int128): Int128 =
+  toInt128(a) + b
+
+proc `+`*(a: Int128, b: BiggestInt): Int128 =
+  a + toInt128(b)
+
+proc toFloat64*(arg: Int128): float64 =
+  let isNegative = isNegative(arg)
+  let arg = abs(arg)
+
+  let a = float64(bitconcat(arg.udata[1], arg.udata[0]))
+  let b = float64(bitconcat(arg.udata[3], arg.udata[2]))
+
+  result = a + 18446744073709551616'f64 * b # a + 2^64 * b
+  if isNegative:
+    result = -result
+
+proc ldexp(x: float64, exp: cint): float64 {.importc: "ldexp", header: "<math.h>".}
+
+template bitor(a,b,c: Int128): Int128 = bitor(bitor(a,b), c)
+
+proc toInt128*(arg: float64): Int128 =
+  let isNegative = arg < 0
+  let v0 = ldexp(abs(arg), -100)
+  let w0 = uint64(trunc(v0))
+  let v1 = ldexp(v0 - float64(w0), 50)
+  let w1 = uint64(trunc(v1))
+  let v2 = ldexp(v1 - float64(w1), 50)
+  let w2 = uint64(trunc(v2))
+
+  let res = bitor(toInt128(w0) shl 100, toInt128(w1) shl 50, toInt128(w2))
+  if isNegative:
+    return -res
+  else:
+    return res
+
+proc maskUInt64*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0]
+  result.udata[1] = arg.udata[1]
+  result.udata[2] = 0
+  result.udata[3] = 0
+
+proc maskUInt32*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0]
+  result.udata[1] = 0
+  result.udata[2] = 0
+  result.udata[3] = 0
+
+proc maskUInt16*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0] and 0xffff
+  result.udata[1] = 0
+  result.udata[2] = 0
+  result.udata[3] = 0
+
+proc maskUInt8*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0] and 0xff
+  result.udata[1] = 0
+  result.udata[2] = 0
+  result.udata[3] = 0
+
+proc maskBytes*(arg: Int128, numbytes: int): Int128 {.noinit.} =
+  case numbytes
+  of 1:
+    return maskUInt8(arg)
+  of 2:
+    return maskUInt16(arg)
+  of 4:
+    return maskUInt32(arg)
+  of 8:
+    return maskUInt64(arg)
+  else:
+    assert(false, "masking only implemented for 1, 2, 4 and 8 bytes")
+
+
+
+
+when isMainModule:
+  let (a,b) = divMod(Ten,Ten)
+
+  doAssert $One == "1"
+  doAssert $Ten == "10"
+  doAssert $Zero == "0"
+  let c = parseDecimalInt128("12345678989876543210123456789")
+  doAssert $c == "12345678989876543210123456789"
+
+  var d : array[39, Int128]
+  d[0] =  parseDecimalInt128("1")
+  d[1] =  parseDecimalInt128("10")
+  d[2] =  parseDecimalInt128("100")
+  d[3] =  parseDecimalInt128("1000")
+  d[4] =  parseDecimalInt128("10000")
+  d[5] =  parseDecimalInt128("100000")
+  d[6] =  parseDecimalInt128("1000000")
+  d[7] =  parseDecimalInt128("10000000")
+  d[8] =  parseDecimalInt128("100000000")
+  d[9] =  parseDecimalInt128("1000000000")
+  d[10] = parseDecimalInt128("10000000000")
+  d[11] = parseDecimalInt128("100000000000")
+  d[12] = parseDecimalInt128("1000000000000")
+  d[13] = parseDecimalInt128("10000000000000")
+  d[14] = parseDecimalInt128("100000000000000")
+  d[15] = parseDecimalInt128("1000000000000000")
+  d[16] = parseDecimalInt128("10000000000000000")
+  d[17] = parseDecimalInt128("100000000000000000")
+  d[18] = parseDecimalInt128("1000000000000000000")
+  d[19] = parseDecimalInt128("10000000000000000000")
+  d[20] = parseDecimalInt128("100000000000000000000")
+  d[21] = parseDecimalInt128("1000000000000000000000")
+  d[22] = parseDecimalInt128("10000000000000000000000")
+  d[23] = parseDecimalInt128("100000000000000000000000")
+  d[24] = parseDecimalInt128("1000000000000000000000000")
+  d[25] = parseDecimalInt128("10000000000000000000000000")
+  d[26] = parseDecimalInt128("100000000000000000000000000")
+  d[27] = parseDecimalInt128("1000000000000000000000000000")
+  d[28] = parseDecimalInt128("10000000000000000000000000000")
+  d[29] = parseDecimalInt128("100000000000000000000000000000")
+  d[30] = parseDecimalInt128("1000000000000000000000000000000")
+  d[31] = parseDecimalInt128("10000000000000000000000000000000")
+  d[32] = parseDecimalInt128("100000000000000000000000000000000")
+  d[33] = parseDecimalInt128("1000000000000000000000000000000000")
+  d[34] = parseDecimalInt128("10000000000000000000000000000000000")
+  d[35] = parseDecimalInt128("100000000000000000000000000000000000")
+  d[36] = parseDecimalInt128("1000000000000000000000000000000000000")
+  d[37] = parseDecimalInt128("10000000000000000000000000000000000000")
+  d[38] = parseDecimalInt128("100000000000000000000000000000000000000")
+
+  for i in 0..<d.len:
+    for j in 0..<d.len:
+      doAssert(cmp(d[i], d[j]) == cmp(i,j))
+      if i + j < d.len:
+        doAssert d[i] * d[j] == d[i+j]
+      if i - j >= 0:
+        doAssert d[i] div d[j] == d[i-j]
+
+  var sum: Int128
+
+  for it in d:
+    sum += it
+
+  doAssert $sum == "111111111111111111111111111111111111111"
+
+  for it in d.mitems:
+    it = -it
+
+  for i in 0..<d.len:
+    for j in 0..<d.len:
+      doAssert(cmp(d[i], d[j]) == -cmp(i,j))
+      if i + j < d.len:
+        doAssert d[i] * d[j] == -d[i+j]
+      if i - j >= 0:
+        doAssert d[i] div d[j] == -d[i-j]
+
+  doAssert $high(Int128) == "170141183460469231731687303715884105727"
+  doAssert $low(Int128) == "-170141183460469231731687303715884105728"
+
+  var ma = 100'i64
+  var mb = 13
+
+  doAssert toInt128(ma) * toInt128(0) == toInt128(0)
+  doAssert toInt128(-ma) * toInt128(0) == toInt128(0)
+
+  # sign correctness
+  doAssert divMod(toInt128( ma),toInt128( mb)) == (toInt128( ma div  mb), toInt128( ma mod  mb))
+  doAssert divMod(toInt128(-ma),toInt128( mb)) == (toInt128(-ma div  mb), toInt128(-ma mod  mb))
+  doAssert divMod(toInt128( ma),toInt128(-mb)) == (toInt128( ma div -mb), toInt128( ma mod -mb))
+  doAssert divMod(toInt128(-ma),toInt128(-mb)) == (toInt128(-ma div -mb), toInt128(-ma mod -mb))
+
+  doAssert divMod(toInt128( mb),toInt128( mb)) == (toInt128( mb div  mb), toInt128( mb mod  mb))
+  doAssert divMod(toInt128(-mb),toInt128( mb)) == (toInt128(-mb div  mb), toInt128(-mb mod  mb))
+  doAssert divMod(toInt128( mb),toInt128(-mb)) == (toInt128( mb div -mb), toInt128( mb mod -mb))
+  doAssert divMod(toInt128(-mb),toInt128(-mb)) == (toInt128(-mb div -mb), toInt128(-mb mod -mb))
+
+  doAssert divMod(toInt128( mb),toInt128( ma)) == (toInt128( mb div  ma), toInt128( mb mod  ma))
+  doAssert divMod(toInt128(-mb),toInt128( ma)) == (toInt128(-mb div  ma), toInt128(-mb mod  ma))
+  doAssert divMod(toInt128( mb),toInt128(-ma)) == (toInt128( mb div -ma), toInt128( mb mod -ma))
+  doAssert divMod(toInt128(-mb),toInt128(-ma)) == (toInt128(-mb div -ma), toInt128(-mb mod -ma))
+
+  let e = parseDecimalInt128("70997106675279150998592376708984375")
+
+  let strArray = [
+    # toHex(e shr 0), toHex(e shr 1), toHex(e shr 2), toHex(e shr 3)
+    "000dac6d782d266a37300c32591eee37", "0006d636bc1693351b9806192c8f771b", "00036b1b5e0b499a8dcc030c9647bb8d", "0001b58daf05a4cd46e601864b23ddc6",
+    "0000dac6d782d266a37300c32591eee3", "00006d636bc1693351b9806192c8f771", "000036b1b5e0b499a8dcc030c9647bb8", "00001b58daf05a4cd46e601864b23ddc",
+    "00000dac6d782d266a37300c32591eee", "000006d636bc1693351b9806192c8f77", "0000036b1b5e0b499a8dcc030c9647bb", "000001b58daf05a4cd46e601864b23dd",
+    "000000dac6d782d266a37300c32591ee", "0000006d636bc1693351b9806192c8f7", "00000036b1b5e0b499a8dcc030c9647b", "0000001b58daf05a4cd46e601864b23d",
+    "0000000dac6d782d266a37300c32591e", "00000006d636bc1693351b9806192c8f", "000000036b1b5e0b499a8dcc030c9647", "00000001b58daf05a4cd46e601864b23",
+    "00000000dac6d782d266a37300c32591", "000000006d636bc1693351b9806192c8", "0000000036b1b5e0b499a8dcc030c964", "000000001b58daf05a4cd46e601864b2",
+    "000000000dac6d782d266a37300c3259", "0000000006d636bc1693351b9806192c", "00000000036b1b5e0b499a8dcc030c96", "0000000001b58daf05a4cd46e601864b",
+    "0000000000dac6d782d266a37300c325", "00000000006d636bc1693351b9806192", "000000000036b1b5e0b499a8dcc030c9", "00000000001b58daf05a4cd46e601864",
+    "00000000000dac6d782d266a37300c32", "000000000006d636bc1693351b980619", "0000000000036b1b5e0b499a8dcc030c", "000000000001b58daf05a4cd46e60186",
+    "000000000000dac6d782d266a37300c3", "0000000000006d636bc1693351b98061", "00000000000036b1b5e0b499a8dcc030", "0000000000001b58daf05a4cd46e6018",
+    "0000000000000dac6d782d266a37300c", "00000000000006d636bc1693351b9806", "000000000000036b1b5e0b499a8dcc03", "00000000000001b58daf05a4cd46e601",
+    "00000000000000dac6d782d266a37300", "000000000000006d636bc1693351b980", "0000000000000036b1b5e0b499a8dcc0", "000000000000001b58daf05a4cd46e60",
+    "000000000000000dac6d782d266a3730", "0000000000000006d636bc1693351b98", "00000000000000036b1b5e0b499a8dcc", "0000000000000001b58daf05a4cd46e6",
+    "0000000000000000dac6d782d266a373", "00000000000000006d636bc1693351b9", "000000000000000036b1b5e0b499a8dc", "00000000000000001b58daf05a4cd46e",
+    "00000000000000000dac6d782d266a37", "000000000000000006d636bc1693351b", "0000000000000000036b1b5e0b499a8d", "000000000000000001b58daf05a4cd46",
+    "000000000000000000dac6d782d266a3", "0000000000000000006d636bc1693351", "00000000000000000036b1b5e0b499a8", "0000000000000000001b58daf05a4cd4",
+    "0000000000000000000dac6d782d266a", "00000000000000000006d636bc169335", "000000000000000000036b1b5e0b499a", "00000000000000000001b58daf05a4cd",
+    "00000000000000000000dac6d782d266", "000000000000000000006d636bc16933", "0000000000000000000036b1b5e0b499", "000000000000000000001b58daf05a4c",
+    "000000000000000000000dac6d782d26", "0000000000000000000006d636bc1693", "00000000000000000000036b1b5e0b49", "0000000000000000000001b58daf05a4",
+    "0000000000000000000000dac6d782d2", "00000000000000000000006d636bc169", "000000000000000000000036b1b5e0b4", "00000000000000000000001b58daf05a",
+    "00000000000000000000000dac6d782d", "000000000000000000000006d636bc16", "0000000000000000000000036b1b5e0b", "000000000000000000000001b58daf05",
+    "000000000000000000000000dac6d782", "0000000000000000000000006d636bc1", "00000000000000000000000036b1b5e0", "0000000000000000000000001b58daf0",
+    "0000000000000000000000000dac6d78", "00000000000000000000000006d636bc", "000000000000000000000000036b1b5e", "00000000000000000000000001b58daf",
+    "00000000000000000000000000dac6d7", "000000000000000000000000006d636b", "0000000000000000000000000036b1b5", "000000000000000000000000001b58da",
+    "000000000000000000000000000dac6d", "0000000000000000000000000006d636", "00000000000000000000000000036b1b", "0000000000000000000000000001b58d",
+    "0000000000000000000000000000dac6", "00000000000000000000000000006d63", "000000000000000000000000000036b1", "00000000000000000000000000001b58",
+    "00000000000000000000000000000dac", "000000000000000000000000000006d6", "0000000000000000000000000000036b", "000000000000000000000000000001b5",
+    "000000000000000000000000000000da", "0000000000000000000000000000006d", "00000000000000000000000000000036", "0000000000000000000000000000001b",
+    "0000000000000000000000000000000d", "00000000000000000000000000000006", "00000000000000000000000000000003", "00000000000000000000000000000001",
+    "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000",
+    "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000",
+    "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000",
+  ]
+
+  for i in 0 ..< 128:
+    let str1 = toHex(e shr i)
+    let str2 = strArray[i]
+    doAssert str1 == str2
+
+  let strArray2 = [
+    "000dac6d782d266a37300c32591eee37", "001b58daf05a4cd46e601864b23ddc6e", "0036b1b5e0b499a8dcc030c9647bb8dc", "006d636bc1693351b9806192c8f771b8",
+    "00dac6d782d266a37300c32591eee370", "01b58daf05a4cd46e601864b23ddc6e0", "036b1b5e0b499a8dcc030c9647bb8dc0", "06d636bc1693351b9806192c8f771b80",
+    "0dac6d782d266a37300c32591eee3700", "1b58daf05a4cd46e601864b23ddc6e00", "36b1b5e0b499a8dcc030c9647bb8dc00", "6d636bc1693351b9806192c8f771b800",
+    "dac6d782d266a37300c32591eee37000", "b58daf05a4cd46e601864b23ddc6e000", "6b1b5e0b499a8dcc030c9647bb8dc000", "d636bc1693351b9806192c8f771b8000",
+    "ac6d782d266a37300c32591eee370000", "58daf05a4cd46e601864b23ddc6e0000", "b1b5e0b499a8dcc030c9647bb8dc0000", "636bc1693351b9806192c8f771b80000",
+    "c6d782d266a37300c32591eee3700000", "8daf05a4cd46e601864b23ddc6e00000", "1b5e0b499a8dcc030c9647bb8dc00000", "36bc1693351b9806192c8f771b800000",
+    "6d782d266a37300c32591eee37000000", "daf05a4cd46e601864b23ddc6e000000", "b5e0b499a8dcc030c9647bb8dc000000", "6bc1693351b9806192c8f771b8000000",
+    "d782d266a37300c32591eee370000000", "af05a4cd46e601864b23ddc6e0000000", "5e0b499a8dcc030c9647bb8dc0000000", "bc1693351b9806192c8f771b80000000",
+    "782d266a37300c32591eee3700000000", "f05a4cd46e601864b23ddc6e00000000", "e0b499a8dcc030c9647bb8dc00000000", "c1693351b9806192c8f771b800000000",
+    "82d266a37300c32591eee37000000000", "05a4cd46e601864b23ddc6e000000000", "0b499a8dcc030c9647bb8dc000000000", "1693351b9806192c8f771b8000000000",
+    "2d266a37300c32591eee370000000000", "5a4cd46e601864b23ddc6e0000000000", "b499a8dcc030c9647bb8dc0000000000", "693351b9806192c8f771b80000000000",
+    "d266a37300c32591eee3700000000000", "a4cd46e601864b23ddc6e00000000000", "499a8dcc030c9647bb8dc00000000000", "93351b9806192c8f771b800000000000",
+    "266a37300c32591eee37000000000000", "4cd46e601864b23ddc6e000000000000", "99a8dcc030c9647bb8dc000000000000", "3351b9806192c8f771b8000000000000",
+    "66a37300c32591eee370000000000000", "cd46e601864b23ddc6e0000000000000", "9a8dcc030c9647bb8dc0000000000000", "351b9806192c8f771b80000000000000",
+    "6a37300c32591eee3700000000000000", "d46e601864b23ddc6e00000000000000", "a8dcc030c9647bb8dc00000000000000", "51b9806192c8f771b800000000000000",
+    "a37300c32591eee37000000000000000", "46e601864b23ddc6e000000000000000", "8dcc030c9647bb8dc000000000000000", "1b9806192c8f771b8000000000000000",
+    "37300c32591eee370000000000000000", "6e601864b23ddc6e0000000000000000", "dcc030c9647bb8dc0000000000000000", "b9806192c8f771b80000000000000000",
+    "7300c32591eee3700000000000000000", "e601864b23ddc6e00000000000000000", "cc030c9647bb8dc00000000000000000", "9806192c8f771b800000000000000000",
+    "300c32591eee37000000000000000000", "601864b23ddc6e000000000000000000", "c030c9647bb8dc000000000000000000", "806192c8f771b8000000000000000000",
+    "00c32591eee370000000000000000000", "01864b23ddc6e0000000000000000000", "030c9647bb8dc0000000000000000000", "06192c8f771b80000000000000000000",
+    "0c32591eee3700000000000000000000", "1864b23ddc6e00000000000000000000", "30c9647bb8dc00000000000000000000", "6192c8f771b800000000000000000000",
+    "c32591eee37000000000000000000000", "864b23ddc6e000000000000000000000", "0c9647bb8dc000000000000000000000", "192c8f771b8000000000000000000000",
+    "32591eee370000000000000000000000", "64b23ddc6e0000000000000000000000", "c9647bb8dc0000000000000000000000", "92c8f771b80000000000000000000000",
+    "2591eee3700000000000000000000000", "4b23ddc6e00000000000000000000000", "9647bb8dc00000000000000000000000", "2c8f771b800000000000000000000000",
+    "591eee37000000000000000000000000", "b23ddc6e000000000000000000000000", "647bb8dc000000000000000000000000", "c8f771b8000000000000000000000000",
+    "91eee370000000000000000000000000", "23ddc6e0000000000000000000000000", "47bb8dc0000000000000000000000000", "8f771b80000000000000000000000000",
+    "1eee3700000000000000000000000000", "3ddc6e00000000000000000000000000", "7bb8dc00000000000000000000000000", "f771b800000000000000000000000000",
+    "eee37000000000000000000000000000", "ddc6e000000000000000000000000000", "bb8dc000000000000000000000000000", "771b8000000000000000000000000000",
+    "ee370000000000000000000000000000", "dc6e0000000000000000000000000000", "b8dc0000000000000000000000000000", "71b80000000000000000000000000000",
+    "e3700000000000000000000000000000", "c6e00000000000000000000000000000", "8dc00000000000000000000000000000", "1b800000000000000000000000000000",
+    "37000000000000000000000000000000", "6e000000000000000000000000000000", "dc000000000000000000000000000000", "b8000000000000000000000000000000",
+    "70000000000000000000000000000000", "e0000000000000000000000000000000", "c0000000000000000000000000000000", "80000000000000000000000000000000",
+  ]
+
+  for i in 0 ..< 128:
+    let str1 = toHex(e shl i)
+    let str2 = strArray2[i]
+    doAssert str1 == str2

compiler/jstypes.nim

@@ -7,28 +7,30 @@
 #    distribution, for details about the copyright.
 #
 
+# included from jsgen.nim
+
 ## Type info generation for the JS backend.
 
+proc rope(arg: Int128): Rope = rope($arg)
+
 proc genTypeInfo(p: PProc, typ: PType): Rope
 proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
   var
     s, u: Rope
-    length: int
     field: PSym
     b: PNode
   result = nil
   case n.kind
   of nkRecList:
-    length = sonsLen(n)
-    if length == 1:
-      result = genObjectFields(p, typ, n.sons[0])
+    if n.len == 1:
+      result = genObjectFields(p, typ, n[0])
     else:
       s = nil
-      for i in 0 ..< length:
-        if i > 0: add(s, ", \L")
-        add(s, genObjectFields(p, typ, n.sons[i]))
+      for i in 0..<n.len:
+        if i > 0: s.add(", \L")
+        s.add(genObjectFields(p, typ, n[i]))
       result = ("{kind: 2, len: $1, offset: 0, " &
-          "typ: null, name: null, sons: [$2]}") % [rope(length), s]
+          "typ: null, name: null, sons: [$2]}") % [rope(n.len), s]
   of nkSym:
     field = n.sym
     s = genTypeInfo(p, field.typ)
@@ -37,29 +39,28 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
                    [mangleName(p.module, field), s,
                     makeJSString(field.name.s)]
   of nkRecCase:
-    length = sonsLen(n)
-    if (n.sons[0].kind != nkSym): internalError(p.config, n.info, "genObjectFields")
-    field = n.sons[0].sym
+    if (n[0].kind != nkSym): internalError(p.config, n.info, "genObjectFields")
+    field = n[0].sym
     s = genTypeInfo(p, field.typ)
-    for i in 1 ..< length:
-      b = n.sons[i]           # branch
+    for i in 1..<n.len:
+      b = n[i]           # branch
       u = nil
       case b.kind
       of nkOfBranch:
-        if sonsLen(b) < 2:
+        if b.len < 2:
           internalError(p.config, b.info, "genObjectFields; nkOfBranch broken")
-        for j in 0 .. sonsLen(b) - 2:
-          if u != nil: add(u, ", ")
-          if b.sons[j].kind == nkRange:
-            addf(u, "[$1, $2]", [rope(getOrdValue(b.sons[j].sons[0])),
-                                 rope(getOrdValue(b.sons[j].sons[1]))])
+        for j in 0..<b.len - 1:
+          if u != nil: u.add(", ")
+          if b[j].kind == nkRange:
+            u.addf("[$1, $2]", [rope(getOrdValue(b[j][0])),
+                                 rope(getOrdValue(b[j][1]))])
           else:
-            add(u, rope(getOrdValue(b.sons[j])))
+            u.add(rope(getOrdValue(b[j])))
       of nkElse:
         u = rope(lengthOrd(p.config, field.typ))
       else: internalError(p.config, n.info, "genObjectFields(nkRecCase)")
-      if result != nil: add(result, ", \L")
-      addf(result, "[setConstr($1), $2]",
+      if result != nil: result.add(", \L")
+      result.addf("[setConstr($1), $2]",
            [u, genObjectFields(p, typ, lastSon(b))])
     result = ("{kind: 3, offset: \"$1\", len: $3, " &
         "typ: $2, name: $4, sons: [$5]}") % [
@@ -68,27 +69,27 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
   else: internalError(p.config, n.info, "genObjectFields")
 
 proc objHasTypeField(t: PType): bool {.inline.} =
-  tfInheritable in t.flags or t.sons[0] != nil
+  tfInheritable in t.flags or t[0] != nil
 
 proc genObjectInfo(p: PProc, typ: PType, name: Rope) =
   let kind = if objHasTypeField(typ): tyObject else: tyTuple
   var s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
            "finalizer: null};$n") % [name, rope(ord(kind))]
   prepend(p.g.typeInfo, s)
-  addf(p.g.typeInfo, "var NNI$1 = $2;$n",
+  p.g.typeInfo.addf("var NNI$1 = $2;$n",
        [rope(typ.id), genObjectFields(p, typ, typ.n)])
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
-  if (typ.kind == tyObject) and (typ.sons[0] != nil):
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [name, genTypeInfo(p, typ.sons[0].skipTypes(skipPtrs))])
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
+  if (typ.kind == tyObject) and (typ[0] != nil):
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [name, genTypeInfo(p, typ[0].skipTypes(skipPtrs))])
 
 proc genTupleFields(p: PProc, typ: PType): Rope =
   var s: Rope = nil
-  for i in 0 ..< typ.len:
-    if i > 0: add(s, ", \L")
+  for i in 0..<typ.len:
+    if i > 0: s.add(", \L")
     s.addf("{kind: 1, offset: \"Field$1\", len: 0, " &
            "typ: $2, name: \"Field$1\", sons: null}",
-           [i.rope, genTypeInfo(p, typ.sons[i])])
+           [i.rope, genTypeInfo(p, typ[i])])
   result = ("{kind: 2, len: $1, offset: 0, " &
             "typ: null, name: null, sons: [$2]}") % [rope(typ.len), s]
 
@@ -96,30 +97,29 @@ proc genTupleInfo(p: PProc, typ: PType, name: Rope) =
   var s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
            "finalizer: null};$n") % [name, rope(ord(typ.kind))]
   prepend(p.g.typeInfo, s)
-  addf(p.g.typeInfo, "var NNI$1 = $2;$n",
+  p.g.typeInfo.addf("var NNI$1 = $2;$n",
        [rope(typ.id), genTupleFields(p, typ)])
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
 
 proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
-  let length = sonsLen(typ.n)
   var s: Rope = nil
-  for i in 0 ..< length:
-    if (typ.n.sons[i].kind != nkSym): internalError(p.config, typ.n.info, "genEnumInfo")
-    let field = typ.n.sons[i].sym
-    if i > 0: add(s, ", \L")
+  for i in 0..<typ.n.len:
+    if (typ.n[i].kind != nkSym): internalError(p.config, typ.n.info, "genEnumInfo")
+    let field = typ.n[i].sym
+    if i > 0: s.add(", \L")
     let extName = if field.ast == nil: field.name.s else: field.ast.strVal
-    addf(s, "\"$1\": {kind: 1, offset: $1, typ: $2, name: $3, len: 0, sons: null}",
+    s.addf("\"$1\": {kind: 1, offset: $1, typ: $2, name: $3, len: 0, sons: null}",
          [rope(field.position), name, makeJSString(extName)])
   var n = ("var NNI$1 = {kind: 2, offset: 0, typ: null, " &
-      "name: null, len: $2, sons: {$3}};$n") % [rope(typ.id), rope(length), s]
+      "name: null, len: $2, sons: {$3}};$n") % [rope(typ.id), rope(typ.n.len), s]
   s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
        "finalizer: null};$n") % [name, rope(ord(typ.kind))]
   prepend(p.g.typeInfo, s)
-  add(p.g.typeInfo, n)
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
-  if typ.sons[0] != nil:
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [name, genTypeInfo(p, typ.sons[0])])
+  p.g.typeInfo.add(n)
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
+  if typ[0] != nil:
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [name, genTypeInfo(p, typ[0])])
 
 proc genTypeInfo(p: PProc, typ: PType): Rope =
   let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink, tyOwned})
@@ -127,7 +127,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
   if containsOrIncl(p.g.typeInfoGenerated, t.id): return
   case t.kind
   of tyDistinct:
-    result = genTypeInfo(p, t.sons[0])
+    result = genTypeInfo(p, t[0])
   of tyPointer, tyProc, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
@@ -138,15 +138,15 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-    addf(p.g.typeInfo, "$1.base = $2;$n",
+    p.g.typeInfo.addf("$1.base = $2;$n",
          [result, genTypeInfo(p, t.lastSon)])
   of tyArray:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [result, genTypeInfo(p, t.sons[1])])
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [result, genTypeInfo(p, t[1])])
   of tyEnum: genEnumInfo(p, t, result)
   of tyObject: genObjectInfo(p, t, result)
   of tyTuple: genTupleInfo(p, t, result)

compiler/lambdalifting.nim

@@ -10,9 +10,9 @@
 # This file implements lambda lifting for the transformator.
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs,
+  intsets, strutils, options, ast, astalgo, msgs,
   idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos,
-  transf
+  transf, liftdestructors
 
 discard """
   The basic approach is that captured vars need to be put on the heap and
@@ -128,8 +128,8 @@ proc newCall(a: PSym, b: PNode): PNode =
 
 proc createClosureIterStateType*(g: ModuleGraph; iter: PSym): PType =
   var n = newNodeI(nkRange, iter.info)
-  addSon(n, newIntNode(nkIntLit, -1))
-  addSon(n, newIntNode(nkIntLit, 0))
+  n.add newIntNode(nkIntLit, -1)
+  n.add newIntNode(nkIntLit, 0)
   result = newType(tyRange, iter)
   result.n = n
   var intType = nilOrSysInt(g)
@@ -148,27 +148,27 @@ proc createEnvObj(g: ModuleGraph; owner: PSym; info: TLineInfo): PType =
 
 proc getClosureIterResult*(g: ModuleGraph; iter: PSym): PSym =
   if resultPos < iter.ast.len:
-    result = iter.ast.sons[resultPos].sym
+    result = iter.ast[resultPos].sym
   else:
     # XXX a bit hacky:
     result = newSym(skResult, getIdent(g.cache, ":result"), iter, iter.info, {})
-    result.typ = iter.typ.sons[0]
+    result.typ = iter.typ[0]
     incl(result.flags, sfUsed)
     iter.ast.add newSymNode(result)
 
 proc addHiddenParam(routine: PSym, param: PSym) =
   assert param.kind == skParam
-  var params = routine.ast.sons[paramsPos]
+  var params = routine.ast[paramsPos]
   # -1 is correct here as param.position is 0 based but we have at position 0
   # some nkEffect node:
   param.position = routine.typ.n.len-1
-  addSon(params, newSymNode(param))
+  params.add newSymNode(param)
   #incl(routine.typ.flags, tfCapturesEnv)
   assert sfFromGeneric in param.flags
   #echo "produced environment: ", param.id, " for ", routine.id
 
 proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
-  let params = routine.ast.sons[paramsPos]
+  let params = routine.ast[paramsPos]
   let hidden = lastSon(params)
   if hidden.kind == nkSym and hidden.sym.kind == skParam and hidden.sym.name.s == paramName:
     result = hidden.sym
@@ -179,7 +179,7 @@ proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
     result = routine
 
 proc getEnvParam*(routine: PSym): PSym =
-  let params = routine.ast.sons[paramsPos]
+  let params = routine.ast[paramsPos]
   let hidden = lastSon(params)
   if hidden.kind == nkSym and hidden.sym.name.s == paramName:
     result = hidden.sym
@@ -206,8 +206,8 @@ proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode =
   # which is however the only case when we generate an assignment in the first
   # place.
   result = newNodeI(nkAsgn, info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
 
 proc makeClosure*(g: ModuleGraph; prc: PSym; env: PNode; info: TLineInfo): PNode =
   result = newNodeIT(nkClosure, info, prc.typ)
@@ -218,6 +218,10 @@ proc makeClosure*(g: ModuleGraph; prc: PSym; env: PNode; info: TLineInfo): PNode
     if env.skipConv.kind == nkClosure:
       localError(g.config, info, "internal error: taking closure of closure")
     result.add(env)
+  #if isClosureIterator(result.typ):
+  createTypeBoundOps(g, nil, result.typ, info)
+  if tfHasAsgn in result.typ.flags or optSeqDestructors in g.config.globalOptions:
+    prc.flags.incl sfInjectDestructors
 
 proc interestingIterVar(s: PSym): bool {.inline.} =
   # XXX optimization: Only lift the variable if it lives across
@@ -233,6 +237,12 @@ proc liftingHarmful(conf: ConfigRef; owner: PSym): bool {.inline.} =
   let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
   result = conf.cmd == cmdCompileToJS and not isCompileTime
 
+proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; owner: PSym) =
+  createTypeBoundOps(g, nil, refType.lastSon, info)
+  createTypeBoundOps(g, nil, refType, info)
+  if tfHasAsgn in refType.flags or optSeqDestructors in g.config.globalOptions:
+    owner.flags.incl sfInjectDestructors
+
 proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env))
   if liftingHarmful(g.config, owner): return n
@@ -245,7 +255,7 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   var env: PNode
   if owner.isIterator:
     let it = getHiddenParam(g, owner)
-    addUniqueField(it.typ.sons[0], hp, g.cache)
+    addUniqueField(it.typ.skipTypes({tyOwned})[0], hp, g.cache)
     env = indirectAccess(newSymNode(it), hp, hp.info)
   else:
     let e = newSym(skLet, iter.name, owner, n.info)
@@ -257,11 +267,12 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
     result.add(v)
   # add 'new' statement:
   result.add newCall(getSysSym(g, n.info, "internalNew"), env)
+  createTypeBoundOpsLL(g, env.typ, n.info, owner)
   result.add makeClosure(g, iter, env, n.info)
 
 proc freshVarForClosureIter*(g: ModuleGraph; s, owner: PSym): PNode =
   let envParam = getHiddenParam(g, owner)
-  let obj = envParam.typ.skipTypes({tyOwned, tyRef})
+  let obj = envParam.typ.skipTypes({tyOwned, tyRef, tyPtr})
   addField(obj, s, g.cache)
 
   var access = newSymNode(envParam)
@@ -291,7 +302,7 @@ type
   DetectionPass = object
     processed, capturedVars: IntSet
     ownerToType: Table[int, PType]
-    somethingToDo, noDestructors: bool
+    somethingToDo: bool
     graph: ModuleGraph
 
 proc initDetectionPass(g: ModuleGraph; fn: PSym): DetectionPass =
@@ -321,9 +332,10 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
     c.ownerToType[owner.id] = result
 
 proc asOwnedRef(c: DetectionPass; t: PType): PType =
-  if optNimV2 in c.graph.config.globalOptions:
+  if optOwnedRefs in c.graph.config.globalOptions:
     assert t.kind == tyRef
     result = newType(tyOwned, t.owner)
+    result.flags.incl tfHasOwned
     result.rawAddSon t
   else:
     result = t
@@ -332,16 +344,17 @@ proc getEnvTypeForOwnerUp(c: var DetectionPass; owner: PSym;
                           info: TLineInfo): PType =
   var r = c.getEnvTypeForOwner(owner, info)
   result = newType(tyPtr, owner)
-  rawAddSon(result, r.skipTypes({tyOwned, tyRef}))
+  rawAddSon(result, r.skipTypes({tyOwned, tyRef, tyPtr}))
 
 proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
   let refObj = c.getEnvTypeForOwner(dest, info) # getHiddenParam(dest).typ
-  let obj = refObj.skipTypes({tyOwned, tyRef})
+  let obj = refObj.skipTypes({tyOwned, tyRef, tyPtr})
   # The assumption here is that gcDestructors means we cannot deal
   # with cycles properly, so it's better to produce a weak ref (=ptr) here.
-  # This seems to be generally correct but since it's a bit risky it's only
-  # enabled for gcDestructors.
-  let fieldType = if c.graph.config.selectedGc == gcDestructors:
+  # This seems to be generally correct but since it's a bit risky it's disabled
+  # for now.
+  # XXX This is wrong for the 'hamming' test, so remove this logic again.
+  let fieldType = if isDefined(c.graph.config, "nimCycleBreaker"):
                     c.getEnvTypeForOwnerUp(dep, info) #getHiddenParam(dep).typ
                   else:
                     c.getEnvTypeForOwner(dep, info)
@@ -351,11 +364,18 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
   let upIdent = getIdent(c.graph.cache, upName)
   let upField = lookupInRecord(obj.n, upIdent)
   if upField != nil:
-    if upField.typ.skipTypes({tyOwned, tyRef}) != fieldType.skipTypes({tyOwned, tyRef}):
+    if upField.typ.skipTypes({tyOwned, tyRef, tyPtr}) != fieldType.skipTypes({tyOwned, tyRef, tyPtr}):
       localError(c.graph.config, dep.info, "internal error: up references do not agree")
+
+    when false:
+      if c.graph.config.selectedGC == gcDestructors and sfCursor notin upField.flags:
+        localError(c.graph.config, dep.info, "internal error: up reference is not a .cursor")
   else:
     let result = newSym(skField, upIdent, obj.owner, obj.owner.info)
     result.typ = fieldType
+    when false:
+      if c.graph.config.selectedGC == gcDestructors:
+        result.flags.incl sfCursor
     rawAddField(obj, result)
 
 discard """
@@ -412,8 +432,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     if innerProc:
       if s.isIterator: c.somethingToDo = true
       if not c.processed.containsOrIncl(s.id):
-        let body = transformBody(c.graph, s, cache = true,
-                                 noDestructors = c.noDestructors)
+        let body = transformBody(c.graph, s, cache = true)
         detectCapturedVars(body, s, c)
     let ow = s.skipGenericOwner
     if ow == owner:
@@ -422,8 +441,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
         addClosureParam(c, owner, n.info)
         if interestingIterVar(s):
           if not c.capturedVars.containsOrIncl(s.id):
-            let obj = getHiddenParam(c.graph, owner).typ.skipTypes({tyOwned, tyRef})
-            #let obj = c.getEnvTypeForOwner(s.owner).skipTypes({tyOwned, tyRef})
+            let obj = getHiddenParam(c.graph, owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
+            #let obj = c.getEnvTypeForOwner(s.owner).skipTypes({tyOwned, tyRef, tyPtr})
 
             if s.name.id == getIdent(c.graph.cache, ":state").id:
               obj.n[0].sym.id = -s.id
@@ -448,8 +467,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       #echo "capturing ", n.info
       # variable 's' is actually captured:
       if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id):
-        let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef})
-        #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef})
+        let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef, tyPtr})
+        #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
         addField(obj, s, c.graph.cache)
       # create required upFields:
       var w = owner.skipGenericOwner
@@ -474,7 +493,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
           w = up
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit,
      nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef,
-     nkConverterDef, nkMacroDef, nkFuncDef, nkCommentStmt:
+     nkConverterDef, nkMacroDef, nkFuncDef, nkCommentStmt, nkTypeOfExpr:
     discard
   of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil:
@@ -490,6 +509,7 @@ type
     processed: IntSet
     envVars: Table[int, PNode]
     inContainer: int
+    unownedEnvVars: Table[int, PNode] # only required for --newruntime
 
 proc initLiftingPass(fn: PSym): LiftingPass =
   result.processed = initIntSet()
@@ -503,7 +523,7 @@ proc accessViaEnvParam(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   if not envParam.isNil:
     var access = newSymNode(envParam)
     while true:
-      let obj = access.typ.sons[0]
+      let obj = access.typ[0]
       assert obj.kind == tyObject
       let field = getFieldFromObj(obj, s)
       if field != nil:
@@ -514,56 +534,75 @@ proc accessViaEnvParam(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   localError(g.config, n.info, "internal error: environment misses: " & s.name.s)
   result = n
 
-proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType): PNode =
-  var v = newSym(skVar, getIdent(cache, envName), owner, owner.info)
-  incl(v.flags, sfShadowed)
+proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo): PNode =
+  var v = newSym(skVar, getIdent(cache, envName), owner, info)
+  v.flags = {sfShadowed, sfGeneratedOp}
   v.typ = typ
   result = newSymNode(v)
   when false:
     if owner.kind == skIterator and owner.typ.callConv == ccClosure:
       let it = getHiddenParam(owner)
-      addUniqueField(it.typ.sons[0], v)
+      addUniqueField(it.typ[0], v)
       result = indirectAccess(newSymNode(it), v, v.info)
     else:
       result = newSymNode(v)
 
 proc setupEnvVar(owner: PSym; d: DetectionPass;
-                 c: var LiftingPass): PNode =
+                 c: var LiftingPass; info: TLineInfo): PNode =
   if owner.isIterator:
     return getHiddenParam(d.graph, owner).newSymNode
-  result = c.envvars.getOrDefault(owner.id)
+  result = c.envVars.getOrDefault(owner.id)
   if result.isNil:
     let envVarType = d.ownerToType.getOrDefault(owner.id)
     if envVarType.isNil:
       localError d.graph.config, owner.info, "internal error: could not determine closure type"
-    result = newEnvVar(d.graph.cache, owner, asOwnedRef(d, envVarType))
+    result = newEnvVar(d.graph.cache, owner, asOwnedRef(d, envVarType), info)
     c.envVars[owner.id] = result
+    if optOwnedRefs in d.graph.config.globalOptions:
+      var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), owner, info)
+      v.flags = {sfShadowed, sfGeneratedOp}
+      v.typ = envVarType
+      c.unownedEnvVars[owner.id] = newSymNode(v)
 
 proc getUpViaParam(g: ModuleGraph; owner: PSym): PNode =
   let p = getHiddenParam(g, owner)
   result = p.newSymNode
   if owner.isIterator:
-    let upField = lookupInRecord(p.typ.skipTypes({tyOwned, tyRef}).n, getIdent(g.cache, upName))
+    let upField = lookupInRecord(p.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(g.cache, upName))
     if upField == nil:
       localError(g.config, owner.info, "could not find up reference for closure iter")
     else:
       result = rawIndirectAccess(result, upField, p.info)
 
 proc rawClosureCreation(owner: PSym;
-                        d: DetectionPass; c: var LiftingPass): PNode =
+                        d: DetectionPass; c: var LiftingPass;
+                        info: TLineInfo): PNode =
   result = newNodeI(nkStmtList, owner.info)
 
   var env: PNode
   if owner.isIterator:
     env = getHiddenParam(d.graph, owner).newSymNode
   else:
-    env = setupEnvVar(owner, d, c)
+    env = setupEnvVar(owner, d, c, info)
     if env.kind == nkSym:
       var v = newNodeI(nkVarSection, env.info)
       addVar(v, env)
       result.add(v)
+      if optOwnedRefs in d.graph.config.globalOptions:
+        let unowned = c.unownedEnvVars[owner.id]
+        assert unowned != nil
+        addVar(v, unowned)
+
     # add 'new' statement:
     result.add(newCall(getSysSym(d.graph, env.info, "internalNew"), env))
+    if optOwnedRefs in d.graph.config.globalOptions:
+      let unowned = c.unownedEnvVars[owner.id]
+      assert unowned != nil
+      let env2 = copyTree(env)
+      env2.typ = unowned.typ
+      result.add newAsgnStmt(unowned, env2, env.info)
+      createTypeBoundOpsLL(d.graph, unowned.typ, env.info, owner)
+
     # add assignment statements for captured parameters:
     for i in 1..<owner.typ.n.len:
       let local = owner.typ.n[i].sym
@@ -571,11 +610,14 @@ proc rawClosureCreation(owner: PSym;
         let fieldAccess = indirectAccess(env, local, env.info)
         # add ``env.param = param``
         result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info))
+        createTypeBoundOps(d.graph, nil, fieldAccess.typ, env.info)
+        if tfHasAsgn in fieldAccess.typ.flags or optSeqDestructors in d.graph.config.globalOptions:
+          owner.flags.incl sfInjectDestructors
 
-  let upField = lookupInRecord(env.typ.skipTypes({tyOwned, tyRef}).n, getIdent(d.graph.cache, upName))
+  let upField = lookupInRecord(env.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
     let up = getUpViaParam(d.graph, owner)
-    if up != nil and upField.typ.skipTypes({tyOwned, tyRef}) == up.typ.skipTypes({tyOwned, tyRef}):
+    if up != nil and upField.typ.skipTypes({tyOwned, tyRef, tyPtr}) == up.typ.skipTypes({tyOwned, tyRef, tyPtr}):
       result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info),
                  up, env.info))
     #elif oldenv != nil and oldenv.typ == upField.typ:
@@ -583,6 +625,18 @@ proc rawClosureCreation(owner: PSym;
     #             oldenv, env.info))
     else:
       localError(d.graph.config, env.info, "internal error: cannot create up reference")
+  # we are not in the sem'check phase anymore! so pass 'nil' for the PContext
+  # and hope for the best:
+  createTypeBoundOpsLL(d.graph, env.typ, owner.info, owner)
+
+proc finishClosureCreation(owner: PSym; d: DetectionPass; c: LiftingPass;
+                           info: TLineInfo; res: PNode) =
+  if optOwnedRefs in d.graph.config.globalOptions:
+    let unowned = c.unownedEnvVars[owner.id]
+    assert unowned != nil
+    let nilLit = newNodeIT(nkNilLit, info, unowned.typ)
+    res.add newAsgnStmt(unowned, nilLit, info)
+    createTypeBoundOpsLL(d.graph, unowned.typ, info, owner)
 
 proc closureCreationForIter(iter: PNode;
                             d: DetectionPass; c: var LiftingPass): PNode =
@@ -594,7 +648,7 @@ proc closureCreationForIter(iter: PNode;
   var vnode: PNode
   if owner.isIterator:
     let it = getHiddenParam(d.graph, owner)
-    addUniqueField(it.typ.skipTypes({tyOwned, tyRef}), v, d.graph.cache)
+    addUniqueField(it.typ.skipTypes({tyOwned, tyRef, tyPtr}), v, d.graph.cache)
     vnode = indirectAccess(newSymNode(it), v, v.info)
   else:
     vnode = v.newSymNode
@@ -602,11 +656,12 @@ proc closureCreationForIter(iter: PNode;
     addVar(vs, vnode)
     result.add(vs)
   result.add(newCall(getSysSym(d.graph, iter.info, "internalNew"), vnode))
+  createTypeBoundOpsLL(d.graph, vnode.typ, iter.info, owner)
 
-  let upField = lookupInRecord(v.typ.skipTypes({tyOwned, tyRef}).n, getIdent(d.graph.cache, upName))
+  let upField = lookupInRecord(v.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
-    let u = setupEnvVar(owner, d, c)
-    if u.typ.skipTypes({tyOwned, tyRef}) == upField.typ.skipTypes({tyOwned, tyRef}):
+    let u = setupEnvVar(owner, d, c, iter.info)
+    if u.typ.skipTypes({tyOwned, tyRef, tyPtr}) == upField.typ.skipTypes({tyOwned, tyRef, tyPtr}):
       result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info),
                  u, iter.info))
     else:
@@ -615,8 +670,10 @@ proc closureCreationForIter(iter: PNode;
 
 proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
                      c: var LiftingPass): PNode =
-  let access = setupEnvVar(owner, d, c)
-  let obj = access.typ.skipTypes({tyOwned, tyRef})
+  var access = setupEnvVar(owner, d, c, n.info)
+  if optOwnedRefs in d.graph.config.globalOptions:
+    access = c.unownedEnvVars[owner.id]
+  let obj = access.typ.skipTypes({tyOwned, tyRef, tyPtr})
   let field = getFieldFromObj(obj, n.sym)
   if field != nil:
     result = rawIndirectAccess(access, field, n.info)
@@ -625,7 +682,7 @@ proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
     result = n
 
 proc getStateField*(g: ModuleGraph; owner: PSym): PSym =
-  getHiddenParam(g, owner).typ.skipTypes({tyOwned, tyRef}).n.sons[0].sym
+  getHiddenParam(g, owner).typ.skipTypes({tyOwned, tyRef, tyPtr}).n[0].sym
 
 proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
                       c: var LiftingPass): PNode
@@ -641,7 +698,7 @@ proc symToClosure(n: PNode; owner: PSym; d: DetectionPass;
     result = closureCreationForIter(n, d, c)
   elif s.skipGenericOwner == owner:
     # direct dependency, so use the outer's env variable:
-    result = makeClosure(d.graph, s, setupEnvVar(owner, d, c), n.info)
+    result = makeClosure(d.graph, s, setupEnvVar(owner, d, c, n.info), n.info)
   else:
     let available = getHiddenParam(d.graph, owner)
     let wanted = getHiddenParam(d.graph, s).typ
@@ -650,7 +707,7 @@ proc symToClosure(n: PNode; owner: PSym; d: DetectionPass;
     while true:
       if access.typ == wanted:
         return makeClosure(d.graph, s, access, n.info)
-      let obj = access.typ.skipTypes({tyOwned, tyRef})
+      let obj = access.typ.skipTypes({tyOwned, tyRef, tyPtr})
       let upField = lookupInRecord(obj.n, getIdent(d.graph.cache, upName))
       if upField == nil:
         localError(d.graph.config, n.info, "internal error: no environment found")
@@ -669,12 +726,13 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         #  echo renderTree(s.getBody, {renderIds})
         let oldInContainer = c.inContainer
         c.inContainer = 0
-        var body = transformBody(d.graph, s)
+        var body = transformBody(d.graph, s, cache = false)
         body = liftCapturedVars(body, s, d, c)
-        if c.envvars.getOrDefault(s.id).isNil:
+        if c.envVars.getOrDefault(s.id).isNil:
           s.transformedBody = body
         else:
-          s.transformedBody = newTree(nkStmtList, rawClosureCreation(s, d, c), body)
+          s.transformedBody = newTree(nkStmtList, rawClosureCreation(s, d, c, n.info), body)
+          finishClosureCreation(s, d, c, n.info, s.transformedBody)
         c.inContainer = oldInContainer
 
       if s.typ.callConv == ccClosure:
@@ -693,13 +751,13 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
     discard
   of nkClosure:
     if n[1].kind == nkNilLit:
-      n.sons[0] = liftCapturedVars(n[0], owner, d, c)
-      let x = n.sons[0].skipConv
+      n[0] = liftCapturedVars(n[0], owner, d, c)
+      let x = n[0].skipConv
       if x.kind == nkClosure:
         #localError(n.info, "internal error: closure to closure created")
         # now we know better, so patch it:
-        n.sons[0] = x.sons[0]
-        n.sons[1] = x.sons[1]
+        n[0] = x[0]
+        n[1] = x[1]
   of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil and n[namePos].kind == nkSym:
       let oldInContainer = c.inContainer
@@ -710,26 +768,29 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
       c.inContainer = oldInContainer
   of nkHiddenStdConv:
     if n.len == 2:
-      n.sons[1] = liftCapturedVars(n[1], owner, d, c)
+      n[1] = liftCapturedVars(n[1], owner, d, c)
       if n[1].kind == nkClosure: result = n[1]
   of nkReturnStmt:
     if n[0].kind in {nkAsgn, nkFastAsgn}:
       # we have a `result = result` expression produced by the closure
       # transform, let's not touch the LHS in order to make the lifting pass
       # correct when `result` is lifted
-      n[0].sons[1] = liftCapturedVars(n[0].sons[1], owner, d, c)
-    else: assert n[0].kind == nkEmpty
+      n[0][1] = liftCapturedVars(n[0][1], owner, d, c)
+    else:
+      n[0] = liftCapturedVars(n[0], owner, d, c)
+  of nkTypeOfExpr:
+    result = n
   else:
     if owner.isIterator:
       if nfLL in n.flags:
         # special case 'when nimVm' due to bug #3636:
-        n.sons[1] = liftCapturedVars(n[1], owner, d, c)
+        n[1] = liftCapturedVars(n[1], owner, d, c)
         return
 
     let inContainer = n.kind in {nkObjConstr, nkBracket}
     if inContainer: inc c.inContainer
     for i in 0..<n.len:
-      n.sons[i] = liftCapturedVars(n[i], owner, d, c)
+      n[i] = liftCapturedVars(n[i], owner, d, c)
     if inContainer: dec c.inContainer
 
 # ------------------ old stuff -------------------------------------------
@@ -776,16 +837,15 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType): PNo
   # pretend 'fn' is a closure iterator for the analysis:
   let oldKind = fn.kind
   let oldCC = fn.typ.callConv
-  fn.kind = skIterator
+  fn.transitionRoutineSymKind(skIterator)
   fn.typ.callConv = ccClosure
   d.ownerToType[fn.id] = ptrType
   detectCapturedVars(body, fn, d)
   result = liftCapturedVars(body, fn, d, c)
-  fn.kind = oldKind
+  fn.transitionRoutineSymKind(oldKind)
   fn.typ.callConv = oldCC
 
-proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
-                  noDestructors: bool): PNode =
+proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PNode =
   # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
 
@@ -801,7 +861,6 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
     tooEarly = true
   else:
     var d = initDetectionPass(g, fn)
-    d.noDestructors = noDestructors
     detectCapturedVars(body, fn, d)
     if not d.somethingToDo and fn.isIterator:
       addClosureParam(d, fn, body.info)
@@ -810,8 +869,9 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
       var c = initLiftingPass(fn)
       result = liftCapturedVars(body, fn, d, c)
       # echo renderTree(result, {renderIds})
-      if c.envvars.getOrDefault(fn.id) != nil:
-        result = newTree(nkStmtList, rawClosureCreation(fn, d, c), result)
+      if c.envVars.getOrDefault(fn.id) != nil:
+        result = newTree(nkStmtList, rawClosureCreation(fn, d, c, body.info), result)
+        finishClosureCreation(fn, d, c, body.info, result)
     else:
       result = body
     #if fn.name.s == "get2":
@@ -854,11 +914,10 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
         ...
     """
   if liftingHarmful(g.config, owner): return body
-  var L = body.len
-  if not (body.kind == nkForStmt and body[L-2].kind in nkCallKinds):
+  if not (body.kind == nkForStmt and body[^2].kind in nkCallKinds):
     localError(g.config, body.info, "ignored invalid for loop")
     return body
-  var call = body[L-2]
+  var call = body[^2]
 
   result = newNodeI(nkStmtList, body.info)
 
@@ -879,37 +938,39 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
     result.add(v)
     # add 'new' statement:
     result.add(newCall(getSysSym(g, env.info, "internalNew"), env.newSymNode))
+    createTypeBoundOpsLL(g, env.typ, body.info, owner)
+
   elif op.kind == nkStmtListExpr:
     let closure = op.lastSon
     if closure.kind == nkClosure:
-      call.sons[0] = closure
-      for i in 0 .. op.len-2:
+      call[0] = closure
+      for i in 0..<op.len-1:
         result.add op[i]
 
   var loopBody = newNodeI(nkStmtList, body.info, 3)
   var whileLoop = newNodeI(nkWhileStmt, body.info, 2)
-  whileLoop.sons[0] = newIntTypeNode(nkIntLit, 1, getSysType(g, body.info, tyBool))
-  whileLoop.sons[1] = loopBody
+  whileLoop[0] = newIntTypeNode(1, getSysType(g, body.info, tyBool))
+  whileLoop[1] = loopBody
   result.add whileLoop
 
   # setup loopBody:
   # gather vars in a tuple:
   var v2 = newNodeI(nkLetSection, body.info)
-  var vpart = newNodeI(if L == 3: nkIdentDefs else: nkVarTuple, body.info)
-  for i in 0 .. L-3:
+  var vpart = newNodeI(if body.len == 3: nkIdentDefs else: nkVarTuple, body.info)
+  for i in 0..<body.len-2:
     if body[i].kind == nkSym:
-      body[i].sym.kind = skLet
-    addSon(vpart, body[i])
+      body[i].sym.transitionToLet()
+    vpart.add body[i]
 
-  addSon(vpart, newNodeI(nkEmpty, body.info)) # no explicit type
+  vpart.add newNodeI(nkEmpty, body.info) # no explicit type
   if not env.isNil:
-    call.sons[0] = makeClosure(g, call.sons[0].sym, env.newSymNode, body.info)
-  addSon(vpart, call)
-  addSon(v2, vpart)
+    call[0] = makeClosure(g, call[0].sym, env.newSymNode, body.info)
+  vpart.add call
+  v2.add vpart
 
-  loopBody.sons[0] = v2
+  loopBody[0] = v2
   var bs = newNodeI(nkBreakState, body.info)
-  bs.addSon(call.sons[0])
+  bs.add call[0]
 
   let ibs = newNodeI(nkIfStmt, body.info)
   let elifBranch = newNodeI(nkElifBranch, body.info)
@@ -921,5 +982,5 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
   elifBranch.add(br)
   ibs.add(elifBranch)
 
-  loopBody.sons[1] = ibs
-  loopBody.sons[2] = body[L-1]
+  loopBody[1] = ibs
+  loopBody[2] = body[^1]

compiler/layouter.nim

@@ -9,13 +9,10 @@
 
 ## Layouter for nimpretty.
 
-import idents, lexer, lineinfos, llstream, options, msgs, strutils,
-  pathutils
-from os import changeFileExt
-from sequtils import delete
+import idents, lexer, lineinfos, llstream, options, msgs, strutils, pathutils
 
 const
-  MaxLineLen = 80
+  MinLineLen = 15
 
 type
   SplitKind = enum
@@ -24,10 +21,15 @@ type
   SemicolonKind = enum
     detectSemicolonKind, useSemicolon, dontTouch
 
-  LayoutToken = enum
-    ltSpaces, ltNewline, ltTab,
+  LayoutToken* = enum
+    ltSpaces,
+    ltCrucialNewline, ## a semantically crucial newline (indentation!)
+    ltSplittingNewline, ## newline used for splitting up long
+                        ## expressions (like after a comma or a binary operator)
+    ltTab,
+    ltOptionalNewline, ## optional newline introduced by nimpretty
     ltComment, ltLit, ltKeyword, ltExportMarker, ltIdent,
-    ltOther, ltOpr,
+    ltOther, ltOpr, ltSomeParLe, ltSomeParRi,
     ltBeginSection, ltEndSection
 
   Emitter* = object
@@ -36,18 +38,19 @@ type
     lastTok: TTokType
     inquote, lastTokWasTerse: bool
     semicolons: SemicolonKind
-    col, lastLineNumber, lineSpan, indentLevel, indWidth*: int
+    col, lastLineNumber, lineSpan, indentLevel, indWidth*, inSection: int
     keepIndents*: int
     doIndentMore*: int
-    kinds: seq[LayoutToken]
-    tokens: seq[string]
+    kinds*: seq[LayoutToken]
+    tokens*: seq[string]
     indentStack: seq[int]
     fixedUntil: int # marks where we must not go in the content
     altSplitPos: array[SplitKind, int] # alternative split positions
+    maxLineLen*: int
 
 proc openEmitter*(em: var Emitter, cache: IdentCache;
                   config: ConfigRef, fileIdx: FileIndex) =
-  let fullPath = Absolutefile config.toFullPath(fileIdx)
+  let fullPath = AbsoluteFile config.toFullPath(fileIdx)
   if em.indWidth == 0:
     em.indWidth = getIndentWidth(fileIdx, llStreamOpen(fullPath, fmRead),
                                 cache, config)
@@ -63,17 +66,20 @@ proc openEmitter*(em: var Emitter, cache: IdentCache;
 
 proc computeMax(em: Emitter; pos: int): int =
   var p = pos
+  var extraSpace = 0
   result = 0
   while p < em.tokens.len and em.kinds[p] != ltEndSection:
     var lhs = 0
     var lineLen = 0
     var foundTab = false
     while p < em.tokens.len and em.kinds[p] != ltEndSection:
-      if em.kinds[p] == ltNewline:
-        if foundTab and lineLen <= MaxLineLen: result = max(result, lhs)
+      if em.kinds[p] in {ltCrucialNewline, ltSplittingNewline}:
+        if foundTab and lineLen <= em.maxLineLen:
+          result = max(result, lhs + extraSpace)
         inc p
         break
       if em.kinds[p] == ltTab:
+        extraSpace = if em.kinds[p-1] == ltSpaces: 0 else: 1
         foundTab = true
       else:
         if not foundTab:
@@ -84,46 +90,17 @@ proc computeMax(em: Emitter; pos: int): int =
 proc computeRhs(em: Emitter; pos: int): int =
   var p = pos
   result = 0
-  while p < em.tokens.len and em.kinds[p] != ltNewline:
+  while p < em.tokens.len and em.kinds[p] notin {ltCrucialNewline, ltSplittingNewline}:
     inc result, em.tokens[p].len
     inc p
 
-proc closeEmitter*(em: var Emitter) =
-  let outFile = em.config.absOutFile
+proc isLongEnough(lineLen, startPos, endPos: int): bool =
+  result = lineLen > MinLineLen and endPos > startPos + 4
 
-  var content = newStringOfCap(16_000)
-  var maxLhs = 0
-  var lineLen = 0
-  var lineBegin = 0
-  for i in 0..em.tokens.high:
-    case em.kinds[i]
-    of ltBeginSection:
-      maxLhs = computeMax(em, lineBegin)
-    of ltEndSection:
-      maxLhs = 0
-    of ltTab:
-      if maxLhs == 0 or computeRhs(em, i)+maxLhs > MaxLineLen:
-        content.add ' '
-      else:
-        let spaces = max(maxLhs - lineLen + 1, 1)
-        for j in 1..spaces: content.add ' '
-    of ltNewline:
-      content.add em.tokens[i]
-      lineLen = 0
-      lineBegin = i+1
-    else:
-      content.add em.tokens[i]
-      inc lineLen, em.tokens[i].len
-
-  if fileExists(outFile) and readFile(outFile.string) == content:
-    discard "do nothing, see #9499"
-    return
-  var f = llStreamOpen(outFile, fmWrite)
-  if f == nil:
-    rawMessage(em.config, errGenerated, "cannot open file: " & outFile.string)
-    return
-  f.llStreamWrite content
-  llStreamClose(f)
+proc findNewline(em: Emitter; p, lineLen: var int) =
+  while p < em.tokens.len and em.kinds[p] notin {ltCrucialNewline, ltSplittingNewline}:
+    inc lineLen, em.tokens[p].len
+    inc p
 
 proc countNewlines(s: string): int =
   result = 0
@@ -137,17 +114,177 @@ proc calcCol(em: var Emitter; s: string) =
     dec i
     inc em.col
 
+proc optionalIsGood(em: var Emitter; pos, currentLen: int): bool =
+  let ourIndent = em.tokens[pos].len
+  var p = pos+1
+  var lineLen = 0
+  em.findNewline(p, lineLen)
+  if p == pos+1: # optionalNewline followed by another newline
+    result = false
+  elif em.kinds[p-1] == ltComment and currentLen+lineLen < em.maxLineLen+MinLineLen:
+    result = false
+  elif p+1 < em.tokens.len and em.kinds[p+1] == ltSpaces and
+      em.kinds[p-1] == ltOptionalNewline:
+    if em.tokens[p+1].len == ourIndent:
+      # concatenate lines with the same indententation
+      var nlPos = p
+      var lineLenTotal = lineLen
+      inc p
+      em.findNewline(p, lineLenTotal)
+      if isLongEnough(lineLenTotal, nlPos, p):
+        em.kinds[nlPos] = ltOptionalNewline
+        if em.kinds[nlPos+1] == ltSpaces:
+          # inhibit extra spaces when concatenating two lines
+          em.tokens[nlPos+1] = if em.tokens[nlPos-2] == ",": " " else: ""
+      result = true
+    elif em.tokens[p+1].len < ourIndent:
+      result = isLongEnough(lineLen, pos, p)
+  elif em.kinds[pos+1] in {ltOther, ltSomeParLe, ltSomeParRi}: # note: pos+1, not p+1
+    result = false
+  else:
+    result = isLongEnough(lineLen, pos, p)
+
+proc lenOfNextTokens(em: Emitter; pos: int): int =
+  result = 0
+  for i in 1..<em.tokens.len-pos:
+    if em.kinds[pos+i] in {ltCrucialNewline, ltSplittingNewline, ltOptionalNewline}: break
+    inc result, em.tokens[pos+i].len
+
+proc guidingInd(em: Emitter; pos: int): int =
+  var i = pos - 1
+  while i >= 0 and em.kinds[i] != ltSomeParLe:
+    dec i
+  while i+1 <= em.kinds.high and em.kinds[i] != ltSomeParRi:
+    if em.kinds[i] == ltSplittingNewline and em.kinds[i+1] == ltSpaces:
+      return em.tokens[i+1].len
+    inc i
+  result = -1
+
+proc renderTokens*(em: var Emitter): string =
+  ## Render Emitter tokens to a string of code
+  template defaultCase() =
+    content.add em.tokens[i]
+    inc lineLen, em.tokens[i].len
+  var content = newStringOfCap(16_000)
+  var maxLhs = 0
+  var lineLen = 0
+  var lineBegin = 0
+  var openPars = 0
+  var i = 0
+  while i <= em.tokens.high:
+    when defined(debug):
+      echo (token: em.tokens[i], kind: em.kinds[i])
+    case em.kinds[i]
+    of ltBeginSection:
+      maxLhs = computeMax(em, lineBegin)
+    of ltEndSection:
+      maxLhs = 0
+      lineBegin = i+1
+    of ltTab:
+      if i >= 2 and em.kinds[i-2] in {ltCrucialNewline, ltSplittingNewline} and
+          em.kinds[i-1] in {ltCrucialNewline, ltSplittingNewline, ltSpaces}:
+        # a previous section has ended
+        maxLhs = 0
+
+      if maxLhs == 0:
+        if em.kinds[i-1] != ltSpaces:
+          content.add em.tokens[i]
+          inc lineLen, em.tokens[i].len
+      else:
+        # pick the shorter indentation token:
+        var spaces = maxLhs - lineLen
+        if spaces < em.tokens[i].len or computeRhs(em, i+1)+maxLhs <= em.maxLineLen+MinLineLen:
+          if spaces <= 0 and content[^1] notin {' ', '\L'}: spaces = 1
+          for j in 1..spaces: content.add ' '
+          inc lineLen, spaces
+        else:
+          content.add em.tokens[i]
+          inc lineLen, em.tokens[i].len
+    of ltCrucialNewline, ltSplittingNewline:
+      content.add em.tokens[i]
+      lineLen = 0
+      lineBegin = i+1
+    of ltOptionalNewline:
+      let totalLineLen = lineLen + lenOfNextTokens(em, i)
+      if totalLineLen > em.maxLineLen and optionalIsGood(em, i, lineLen):
+        if i-1 >= 0 and em.kinds[i-1] == ltSpaces:
+          let spaces = em.tokens[i-1].len
+          content.setLen(content.len - spaces)
+        content.add "\L"
+        let guide = if openPars > 0: guidingInd(em, i) else: -1
+        if guide >= 0:
+          content.add repeat(' ', guide)
+          lineLen = guide
+        else:
+          content.add em.tokens[i]
+          lineLen = em.tokens[i].len
+        lineBegin = i+1
+        if i+1 < em.kinds.len and em.kinds[i+1] == ltSpaces:
+          # inhibit extra spaces at the start of a new line
+          inc i
+    of ltLit:
+      let lineSpan = countNewlines(em.tokens[i])
+      if lineSpan > 0:
+        em.calcCol(em.tokens[i])
+        lineLen = em.col
+      else:
+        inc lineLen, em.tokens[i].len
+      content.add em.tokens[i]
+    of ltSomeParLe:
+      inc openPars
+      defaultCase()
+    of ltSomeParRi:
+      doAssert openPars > 0
+      dec openPars
+      defaultCase()
+    else:
+      defaultCase()
+    inc i
+
+  return content
+
+proc writeOut*(em: Emitter, content: string)  =
+  ## Write to disk
+  let outFile = em.config.absOutFile
+  if fileExists(outFile) and readFile(outFile.string) == content:
+    discard "do nothing, see #9499"
+    return
+  var f = llStreamOpen(outFile, fmWrite)
+  if f == nil:
+    rawMessage(em.config, errGenerated, "cannot open file: " & outFile.string)
+    return
+  f.llStreamWrite content
+  llStreamClose(f)
+
+proc closeEmitter*(em: var Emitter) =
+  ## Renders emitter tokens and write to a file
+  let content = renderTokens(em)
+  em.writeOut(content)
+
 proc wr(em: var Emitter; x: string; lt: LayoutToken) =
   em.tokens.add x
   em.kinds.add lt
   inc em.col, x.len
   assert em.tokens.len == em.kinds.len
 
-proc wrNewline(em: var Emitter) =
+proc wrNewline(em: var Emitter; kind = ltCrucialNewline) =
   em.tokens.add "\L"
-  em.kinds.add ltNewline
+  em.kinds.add kind
   em.col = 0
 
+proc newlineWasSplitting*(em: var Emitter) =
+  if em.kinds.len >= 3 and em.kinds[^3] == ltCrucialNewline:
+    em.kinds[^3] = ltSplittingNewline
+
+#[
+Splitting newlines can occur:
+- after commas, semicolon, '[', '('.
+- after binary operators, '='.
+- after ':' type
+
+We only need parser support for the "after type" case.
+]#
+
 proc wrSpaces(em: var Emitter; spaces: int) =
   if spaces > 0:
     wr(em, strutils.repeat(' ', spaces), ltSpaces)
@@ -158,8 +295,19 @@ proc wrSpace(em: var Emitter) =
 proc wrTab(em: var Emitter) =
   wr(em, " ", ltTab)
 
-proc beginSection*(em: var Emitter) = wr(em, "", ltBeginSection)
-proc endSection*(em: var Emitter) = wr(em, "", ltEndSection)
+proc beginSection*(em: var Emitter) =
+  let pos = max(0, em.tokens.len-2)
+  em.tokens.insert "", pos
+  em.kinds.insert ltBeginSection, pos
+  inc em.inSection
+
+#wr(em, "", ltBeginSection)
+proc endSection*(em: var Emitter) =
+  em.tokens.insert "", em.tokens.len-2
+  em.kinds.insert ltEndSection, em.kinds.len-2
+  dec em.inSection
+
+#wr(em, "", ltEndSection)
 
 proc removeSpaces(em: var Emitter) =
   while em.kinds.len > 0 and em.kinds[^1] == ltSpaces:
@@ -168,87 +316,72 @@ proc removeSpaces(em: var Emitter) =
     setLen(em.kinds, em.kinds.len-1)
     dec em.col, tokenLen
 
-template goodCol(col): bool = col in 40..MaxLineLen
 
 const
   openPars = {tkParLe, tkParDotLe,
-              tkBracketLe, tkBracketLeColon, tkCurlyDotLe,
-              tkCurlyLe}
+              tkBracketLe, tkBracketDotLe, tkBracketLeColon,
+              tkCurlyDotLe, tkCurlyLe}
   closedPars = {tkParRi, tkParDotRi,
-              tkBracketRi, tkCurlyDotRi,
-              tkCurlyRi}
+                tkBracketRi, tkBracketDotRi,
+                tkCurlyDotRi, tkCurlyRi}
 
-  splitters = openPars + {tkComma, tkSemicolon}
+  splitters = openPars + {tkComma, tkSemiColon} # do not add 'tkColon' here!
   oprSet = {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
             tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor}
 
-template rememberSplit(kind) =
-  if goodCol(em.col):
-    em.altSplitPos[kind] = em.tokens.len
+template goodCol(col): bool = col >= em.maxLineLen div 2
 
 template moreIndent(em): int =
-  (if em.doIndentMore > 0: em.indWidth*2 else: em.indWidth)
+  if em.doIndentMore > 0: em.indWidth*2 else: em.indWidth
 
-proc softLinebreak(em: var Emitter, lit: string) =
-  # XXX Use an algorithm that is outlined here:
-  # https://llvm.org/devmtg/2013-04/jasper-slides.pdf
-  # +2 because we blindly assume a comma or ' &' might follow
-  if not em.inquote and em.col+lit.len+2 >= MaxLineLen:
-    if em.lastTok in splitters:
-      # bug #10295, check first if even more indentation would help:
-      let spaces = em.indentLevel+moreIndent(em)
-      if spaces < em.col:
-        removeSpaces em
-        wrNewline(em)
-        em.col = 0
-        wrSpaces em, spaces
-    else:
-      # search backwards for a good split position:
-      for a in mitems(em.altSplitPos):
-        if a > em.fixedUntil:
-          var spaces = 0
-          while a+spaces < em.kinds.len and em.kinds[a+spaces] == ltSpaces:
-            inc spaces
-          if spaces > 0:
-            delete(em.tokens, a, a+spaces-1)
-            delete(em.kinds, a, a+spaces-1)
-          em.kinds.insert(ltNewline, a)
-          em.tokens.insert("\L", a)
-          em.kinds.insert(ltSpaces, a+1)
-          em.tokens.insert(repeat(' ', em.indentLevel+moreIndent(em)), a+1)
-          # recompute em.col:
-          var i = em.kinds.len-1
-          em.col = 0
-          while i >= 0 and em.kinds[i] != ltNewline:
-            inc em.col, em.tokens[i].len
-            dec i
-          # mark position as "already split here"
-          a = -1
-          break
+template rememberSplit(kind) =
+  if goodCol(em.col) and not em.inquote:
+    let spaces = em.indentLevel+moreIndent(em)
+    if spaces < em.col and spaces > 0:
+      wr(em, strutils.repeat(' ', spaces), ltOptionalNewline)
+    #em.altSplitPos[kind] = em.tokens.len
 
-proc emitMultilineComment(em: var Emitter, lit: string, col: int) =
+proc emitMultilineComment(em: var Emitter, lit: string, col: int; dontIndent: bool) =
   # re-align every line in the multi-line comment:
   var i = 0
-  var lastIndent = em.indentStack[^1]
+  var lastIndent = if em.keepIndents > 0: em.indentLevel else: em.indentStack[^1]
   var b = 0
+  var dontIndent = dontIndent
+  var hasEmptyLine = false
   for commentLine in splitLines(lit):
-    let stripped = commentLine.strip()
-    var a = 0
-    while a < commentLine.len and commentLine[a] == ' ': inc a
-    if i == 0:
-      discard
-    elif stripped.len == 0:
-      wrNewline em
+    if i == 0 and (commentLine.endsWith("\\") or commentLine.endsWith("[")):
+      dontIndent = true
+      wr em, commentLine, ltComment
+    elif dontIndent:
+      if i > 0: wrNewline em
+      wr em, commentLine, ltComment
     else:
-      if a > lastIndent:
-        b += em.indWidth
-        lastIndent = a
-      elif a < lastIndent:
-        b -= em.indWidth
-        lastIndent = a
-      wrNewline em
-      wrSpaces em, col + b
-    wr em, stripped, ltComment
+      let stripped = commentLine.strip()
+      if i == 0:
+        if em.kinds.len > 0 and em.kinds[^1] != ltTab:
+          wr(em, "", ltTab)
+      elif stripped.len == 0:
+        wrNewline em
+        hasEmptyLine = true
+      else:
+        var a = 0
+        while a < commentLine.len and commentLine[a] == ' ': inc a
+
+        if a > lastIndent:
+          b += em.indWidth
+          lastIndent = a
+        elif a < lastIndent:
+          b -= em.indWidth
+          lastIndent = a
+        wrNewline em
+        if not hasEmptyLine or col + b < 15:
+          if col + b > 0:
+            wr(em, repeat(' ', col+b), ltTab)
+          else:
+            wr(em, "", ltTab)
+        else:
+          wr(em, repeat(' ', a), ltSpaces)
+      wr em, stripped, ltComment
     inc i
 
 proc lastChar(s: string): char =
@@ -257,20 +390,20 @@ proc lastChar(s: string): char =
 proc endsInWhite(em: Emitter): bool =
   var i = em.tokens.len-1
   while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection}: dec(i)
-  result = if i >= 0: em.kinds[i] in {ltSpaces, ltNewline, ltTab} else: true
+  result = if i >= 0: em.kinds[i] in {ltSpaces, ltCrucialNewline, ltSplittingNewline, ltTab} else: true
 
 proc endsInNewline(em: Emitter): bool =
   var i = em.tokens.len-1
   while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection, ltSpaces}: dec(i)
-  result = if i >= 0: em.kinds[i] in {ltNewline, ltTab} else: true
+  result = if i >= 0: em.kinds[i] in {ltCrucialNewline, ltSplittingNewline, ltTab} else: true
 
 proc endsInAlpha(em: Emitter): bool =
   var i = em.tokens.len-1
   while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection}: dec(i)
   result = if i >= 0: em.tokens[i].lastChar in SymChars+{'_'} else: false
 
-proc emitComment(em: var Emitter; tok: TToken) =
-  let col = em.col
+proc emitComment(em: var Emitter; tok: TToken; dontIndent: bool) =
+  var col = em.col
   let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA, tok.commentOffsetB)
   em.lineSpan = countNewlines(lit)
   if em.lineSpan > 0: calcCol(em, lit)
@@ -281,7 +414,8 @@ proc emitComment(em: var Emitter; tok: TToken) =
   else:
     if not endsInWhite(em):
       wrTab em
-    emitMultilineComment(em, lit, col)
+      inc col
+    emitMultilineComment(em, lit, col, dontIndent)
 
 proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
   template wasExportMarker(em): bool =
@@ -303,19 +437,25 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     return
 
   var preventComment = false
-  if tok.tokType == tkComment and tok.line == em.lastLineNumber and tok.indent >= 0:
+  if tok.tokType == tkComment and tok.line == em.lastLineNumber:
     # we have an inline comment so handle it before the indentation token:
-    emitComment(em, tok)
+    emitComment(em, tok, dontIndent = (em.inSection == 0))
     preventComment = true
     em.fixedUntil = em.tokens.high
 
   elif tok.indent >= 0:
+    var newlineKind = ltCrucialNewline
     if em.keepIndents > 0:
       em.indentLevel = tok.indent
-    elif (em.lastTok in (splitters + oprSet) and tok.tokType notin closedPars):
+    elif (em.lastTok in (splitters + oprSet) and
+        tok.tokType notin (closedPars - {tkBracketDotRi})):
+      if tok.tokType in openPars and tok.indent > em.indentStack[^1]:
+        while em.indentStack[^1] < tok.indent:
+          em.indentStack.add(em.indentStack[^1] + em.indWidth)
       # aka: we are in an expression context:
-      let alignment = tok.indent - em.indentStack[^1]
+      let alignment = max(tok.indent - em.indentStack[^1], 0)
       em.indentLevel = alignment + em.indentStack.high * em.indWidth
+      newlineKind = ltSplittingNewline
     else:
       if tok.indent > em.indentStack[^1]:
         em.indentStack.add tok.indent
@@ -335,7 +475,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     ]#
     # remove trailing whitespace:
     removeSpaces em
-    wrNewline em
+    wrNewline em, newlineKind
     for i in 2..tok.line - em.lastLineNumber: wrNewline(em)
     wrSpaces em, em.indentLevel
     em.fixedUntil = em.tokens.high
@@ -346,20 +486,15 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     if endsInAlpha(em):
       wrSpace em
     elif not em.inquote and not endsInWhite(em) and
-        em.lastTok notin openPars and not em.lastTokWasTerse:
+        em.lastTok notin (openPars+{tkOpr, tkDotDot}) and not em.lastTokWasTerse:
       #and tok.tokType in oprSet
       wrSpace em
 
     if not em.inquote:
       wr(em, TokTypeToStr[tok.tokType], ltKeyword)
-
-      case tok.tokType
-      of tkAnd: rememberSplit(splitAnd)
-      of tkOr: rememberSplit(splitOr)
-      of tkIn, tkNotin:
+      if tok.tokType in {tkAnd, tkOr, tkIn, tkNotin}:
         rememberSplit(splitIn)
         wrSpace em
-      else: discard
     else:
       # keywords in backticks are not normalized:
       wr(em, tok.ident.s, ltIdent)
@@ -367,22 +502,19 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
   of tkColon:
     wr(em, TokTypeToStr[tok.tokType], ltOther)
     wrSpace em
-  of tkSemicolon, tkComma:
+  of tkSemiColon, tkComma:
     wr(em, TokTypeToStr[tok.tokType], ltOther)
     rememberSplit(splitComma)
     wrSpace em
-  of tkParDotLe, tkParLe, tkBracketDotLe, tkBracketLe,
-     tkCurlyLe, tkCurlyDotLe, tkBracketLeColon:
-    if tok.strongSpaceA > 0 and not em.endsInWhite and not em.wasExportMarker:
+  of openPars:
+    if tok.strongSpaceA > 0 and not em.endsInWhite and
+        (not em.wasExportMarker or tok.tokType == tkCurlyDotLe):
       wrSpace em
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, TokTypeToStr[tok.tokType], ltSomeParLe)
     rememberSplit(splitParLe)
-  of tkParRi,
-     tkBracketRi, tkCurlyRi,
-     tkBracketDotRi,
-     tkCurlyDotRi,
-     tkParDotRi,
-     tkColonColon:
+  of closedPars:
+    wr(em, TokTypeToStr[tok.tokType], ltSomeParRi)
+  of tkColonColon:
     wr(em, TokTypeToStr[tok.tokType], ltOther)
   of tkDot:
     lastTokWasTerse = true
@@ -392,8 +524,8 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     wr(em, TokTypeToStr[tok.tokType], ltOther)
     if not em.inquote: wrSpace(em)
   of tkOpr, tkDotDot:
-    if ((tok.strongSpaceA == 0 and tok.strongSpaceB == 0) or em.inquote) and
-      tok.ident.s notin ["<", ">", "<=", ">=", "==", "!="]:
+    if em.inquote or ((tok.strongSpaceA == 0 and tok.strongSpaceB == 0) and
+        tok.ident.s notin ["<", ">", "<=", ">=", "==", "!="]):
       # bug #9504: remember to not spacify a keyword:
       lastTokWasTerse = true
       # if not surrounded by whitespace, don't produce any whitespace either:
@@ -413,10 +545,9 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     em.inquote = not em.inquote
   of tkComment:
     if not preventComment:
-      emitComment(em, tok)
+      emitComment(em, tok, dontIndent = false)
   of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, tkGTripleStrLit, tkCharLit:
     let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
-    softLinebreak(em, lit)
     if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
     em.lineSpan = countNewlines(lit)
     if em.lineSpan > 0: calcCol(em, lit)
@@ -424,7 +555,6 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
   of tkEof: discard
   else:
     let lit = if tok.ident != nil: tok.ident.s else: tok.literal
-    softLinebreak(em, lit)
     if endsInAlpha(em): wrSpace(em)
     wr em, lit, ltIdent
 
@@ -439,6 +569,11 @@ proc endsWith(em: Emitter; k: varargs[string]): bool =
     if em.tokens[em.tokens.len - k.len + i] != k[i]: return false
   return true
 
+proc rfind(em: Emitter, t: string): int =
+  for i in 1..5:
+    if em.tokens[^i] == t:
+      return i
+
 proc starWasExportMarker*(em: var Emitter) =
   if em.endsWith(" ", "*", " "):
     setLen(em.tokens, em.tokens.len-3)
@@ -449,11 +584,13 @@ proc starWasExportMarker*(em: var Emitter) =
 
 proc commaWasSemicolon*(em: var Emitter) =
   if em.semicolons == detectSemicolonKind:
-    em.semicolons = if em.endsWith(",", " "): dontTouch else: useSemicolon
-  if em.semicolons == useSemicolon and em.endsWith(",", " "):
-    em.tokens[em.tokens.len-2] = ";"
+    em.semicolons = if em.rfind(";") > 0: useSemicolon else: dontTouch
+  if em.semicolons == useSemicolon:
+    let commaPos = em.rfind(",")
+    if commaPos > 0:
+      em.tokens[^commaPos] = ";"
 
 proc curlyRiWasPragma*(em: var Emitter) =
   if em.endsWith("}"):
-    em.tokens[em.tokens.len-1] = ".}"
+    em.tokens[^1] = ".}"
     inc em.col

compiler/lexer.nim

@@ -62,6 +62,7 @@ type
     tkComma, tkSemiColon,
     tkColon, tkColonColon, tkEquals, tkDot, tkDotDot, tkBracketLeColon,
     tkOpr, tkComment, tkAccent,
+    # these are fake tokens used by renderer.nim
     tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
 
   TTokTypes* = set[TTokType]
@@ -134,7 +135,7 @@ type
   TErrorHandler* = proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string)
   TLexer* = object of TBaseLexer
     fileIdx*: FileIndex
-    indentAhead*: int         # if > 0 an indendation has already been read
+    indentAhead*: int         # if > 0 an indentation has already been read
                               # this is needed because scanning comments
                               # needs so much look-ahead
     currLineIndent*: int
@@ -146,10 +147,6 @@ type
       previousToken: TLineInfo
     config*: ConfigRef
 
-when defined(nimpretty):
-  var
-    gIndentationWidth*: int
-
 proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
   result = newLineInfo(L.fileIdx, tok.line, tok.col)
   when defined(nimpretty):
@@ -173,7 +170,7 @@ proc isNimIdentifier*(s: string): bool =
       inc(i)
     result = true
 
-proc tokToStr*(tok: TToken): string =
+proc `$`*(tok: TToken): string =
   case tok.tokType
   of tkIntLit..tkInt64Lit: result = $tok.iNumber
   of tkFloatLit..tkFloat64Lit: result = $tok.fNumber
@@ -188,11 +185,11 @@ proc tokToStr*(tok: TToken): string =
 
 proc prettyTok*(tok: TToken): string =
   if isKeyword(tok.tokType): result = "keyword " & tok.ident.s
-  else: result = tokToStr(tok)
+  else: result = $tok
 
 proc printTok*(conf: ConfigRef; tok: TToken) =
   msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" &
-      TokTypeToStr[tok.tokType] & " " & tokToStr(tok))
+      TokTypeToStr[tok.tokType] & " " & $tok)
 
 proc initToken*(L: var TToken) =
   L.tokType = tkInvalid
@@ -223,7 +220,7 @@ proc fillToken(L: var TToken) =
 proc openLexer*(lex: var TLexer, fileIdx: FileIndex, inputstream: PLLStream;
                  cache: IdentCache; config: ConfigRef) =
   openBaseLexer(lex, inputstream)
-  lex.fileIdx = fileidx
+  lex.fileIdx = fileIdx
   lex.indentAhead = -1
   lex.currLineIndent = 0
   inc(lex.lineNumber, inputstream.lineOffset)
@@ -308,11 +305,11 @@ template tokenEndPrevious(tok, pos) =
     tok.offsetB = L.offsetBase + pos
 
 template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
-  add(t.literal, replacementChar)
+  t.literal.add(replacementChar)
   inc(L.bufpos)
 
 template eatChar(L: var TLexer, t: var TToken) =
-  add(t.literal, L.buf[L.bufpos])
+  t.literal.add(L.buf[L.bufpos])
   inc(L.bufpos)
 
 proc getNumber(L: var TLexer, result: var TToken) =
@@ -321,7 +318,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
     result = 0
     while true:
       if L.buf[pos] in chars:
-        add(tok.literal, L.buf[pos])
+        tok.literal.add(L.buf[pos])
         inc(pos)
         inc(result)
       else:
@@ -332,14 +329,14 @@ proc getNumber(L: var TLexer, result: var TToken) =
             "only single underscores may occur in a token and token may not " &
             "end with an underscore: e.g. '1__1' and '1_' are invalid")
           break
-        add(tok.literal, '_')
+        tok.literal.add('_')
         inc(pos)
     L.bufpos = pos
 
   proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
     var pos = L.bufpos              # use registers for pos, buf
     while L.buf[pos] in chars:
-      add(tok.literal, L.buf[pos])
+      tok.literal.add(L.buf[pos])
       inc(pos)
     L.bufpos = pos
 
@@ -355,12 +352,12 @@ proc getNumber(L: var TLexer, result: var TToken) =
     # We must verify +/- specifically so that we're not past the literal
     if  L.buf[L.bufpos] in {'+', '-'} and
         L.buf[L.bufpos - 1] in {'e', 'E'}:
-      add(t.literal, L.buf[L.bufpos])
+      t.literal.add(L.buf[L.bufpos])
       inc(L.bufpos)
       matchChars(L, t, literalishChars)
     if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
       inc(L.bufpos)
-      add(t.literal, L.buf[L.bufpos])
+      t.literal.add(L.buf[L.bufpos])
       matchChars(L, t, {'0'..'9'})
     L.bufpos = msgPos
     lexMessage(L, msgKind, msg % t.literal)
@@ -379,7 +376,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
   result.literal = ""
   result.base = base10
   startpos = L.bufpos
-  tokenBegin(result, startPos)
+  tokenBegin(result, startpos)
 
   # First stage: find out base, make verifications, build token literal string
   # {'c', 'C'} is added for deprecation reasons to provide a clear error message
@@ -534,13 +531,13 @@ proc getNumber(L: var TLexer, result: var TToken) =
 
       case result.tokType
       of tkIntLit, tkInt64Lit: result.iNumber = xi
-      of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi))))
-      of tkInt16Lit: result.iNumber = BiggestInt(int16(toU16(int(xi))))
-      of tkInt32Lit: result.iNumber = BiggestInt(int32(toU32(int64(xi))))
+      of tkInt8Lit: result.iNumber = ashr(xi shl 56, 56)
+      of tkInt16Lit: result.iNumber = ashr(xi shl 48, 48)
+      of tkInt32Lit: result.iNumber = ashr(xi shl 32, 32)
       of tkUIntLit, tkUInt64Lit: result.iNumber = xi
-      of tkUInt8Lit: result.iNumber = BiggestInt(cast[uint8](toU8(int(xi))))
-      of tkUInt16Lit: result.iNumber = BiggestInt(cast[uint16](toU16(int(xi))))
-      of tkUInt32Lit: result.iNumber = BiggestInt(cast[uint32](toU32(int64(xi))))
+      of tkUInt8Lit: result.iNumber = xi and 0xff
+      of tkUInt16Lit: result.iNumber = xi and 0xffff
+      of tkUInt32Lit: result.iNumber = xi and 0xffffffff
       of tkFloat32Lit:
         result.fNumber = (cast[PFloat32](addr(xi)))[]
         # note: this code is endian neutral!
@@ -569,7 +566,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
       case result.tokType
       of floatTypes:
         result.fNumber = parseFloat(result.literal)
-      of tkUint64Lit:
+      of tkUInt64Lit, tkUIntLit:
         var iNumber: uint64
         var len: int
         try:
@@ -691,51 +688,51 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
     if L.config.oldNewlines:
       if tok.tokType == tkCharLit:
         lexMessage(L, errGenerated, "\\n not allowed in character literal")
-      add(tok.literal, L.config.target.tnl)
+      tok.literal.add(L.config.target.tnl)
     else:
-      add(tok.literal, '\L')
+      tok.literal.add('\L')
     inc(L.bufpos)
   of 'p', 'P':
     if tok.tokType == tkCharLit:
       lexMessage(L, errGenerated, "\\p not allowed in character literal")
-    add(tok.literal, L.config.target.tnl)
+    tok.literal.add(L.config.target.tnl)
     inc(L.bufpos)
   of 'r', 'R', 'c', 'C':
-    add(tok.literal, CR)
+    tok.literal.add(CR)
     inc(L.bufpos)
   of 'l', 'L':
-    add(tok.literal, LF)
+    tok.literal.add(LF)
     inc(L.bufpos)
   of 'f', 'F':
-    add(tok.literal, FF)
+    tok.literal.add(FF)
     inc(L.bufpos)
   of 'e', 'E':
-    add(tok.literal, ESC)
+    tok.literal.add(ESC)
     inc(L.bufpos)
   of 'a', 'A':
-    add(tok.literal, BEL)
+    tok.literal.add(BEL)
     inc(L.bufpos)
   of 'b', 'B':
-    add(tok.literal, BACKSPACE)
+    tok.literal.add(BACKSPACE)
     inc(L.bufpos)
   of 'v', 'V':
-    add(tok.literal, VT)
+    tok.literal.add(VT)
     inc(L.bufpos)
   of 't', 'T':
-    add(tok.literal, '\t')
+    tok.literal.add('\t')
     inc(L.bufpos)
   of '\'', '\"':
-    add(tok.literal, L.buf[L.bufpos])
+    tok.literal.add(L.buf[L.bufpos])
     inc(L.bufpos)
   of '\\':
-    add(tok.literal, '\\')
+    tok.literal.add('\\')
     inc(L.bufpos)
   of 'x', 'X':
     inc(L.bufpos)
     var xi = 0
     handleHexChar(L, xi, 1)
     handleHexChar(L, xi, 2)
-    add(tok.literal, chr(xi))
+    tok.literal.add(chr(xi))
   of 'u', 'U':
     if tok.tokType == tkCharLit:
       lexMessage(L, errGenerated, "\\u not allowed in character literal")
@@ -765,14 +762,14 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
       lexMessage(L, warnOctalEscape)
     var xi = 0
     handleDecChars(L, xi)
-    if (xi <= 255): add(tok.literal, chr(xi))
+    if (xi <= 255): tok.literal.add(chr(xi))
     else: lexMessage(L, errGenerated, "invalid character constant")
   else: lexMessage(L, errGenerated, "invalid character constant")
 
 proc newString(s: cstring, len: int): string =
   ## XXX, how come there is no support for this?
   result = newString(len)
-  for i in 0 ..< len:
+  for i in 0..<len:
     result[i] = s[i]
 
 proc handleCRLF(L: var TLexer, pos: int): int =
@@ -819,12 +816,12 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
           tokenEndIgnore(tok, pos+2)
           L.bufpos = pos + 3 # skip the three """
           break
-        add(tok.literal, '\"')
+        tok.literal.add('\"')
         inc(pos)
       of CR, LF:
         tokenEndIgnore(tok, pos)
         pos = handleCRLF(L, pos)
-        add(tok.literal, "\n")
+        tok.literal.add("\n")
       of nimlexbase.EndOfFile:
         tokenEndIgnore(tok, pos)
         var line2 = L.lineNumber
@@ -834,7 +831,7 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
         L.bufpos = pos
         break
       else:
-        add(tok.literal, L.buf[pos])
+        tok.literal.add(L.buf[pos])
         inc(pos)
   else:
     # ordinary string literal
@@ -845,7 +842,7 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
       if c == '\"':
         if mode != normal and L.buf[pos+1] == '\"':
           inc(pos, 2)
-          add(tok.literal, '"')
+          tok.literal.add('"')
         else:
           tokenEndIgnore(tok, pos)
           inc(pos) # skip '"'
@@ -859,7 +856,7 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
         getEscapedChar(L, tok)
         pos = L.bufpos
       else:
-        add(tok.literal, c)
+        tok.literal.add(c)
         inc(pos)
     L.bufpos = pos
 
@@ -868,7 +865,9 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
   inc(L.bufpos)               # skip '
   var c = L.buf[L.bufpos]
   case c
-  of '\0'..pred(' '), '\'': lexMessage(L, errGenerated, "invalid character literal")
+  of '\0'..pred(' '), '\'':
+    lexMessage(L, errGenerated, "invalid character literal")
+    tok.literal = $c
   of '\\': getEscapedChar(L, tok)
   else:
     tok.literal = $c
@@ -882,6 +881,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
   var h: Hash = 0
   var pos = L.bufpos
   tokenBegin(tok, pos)
+  var suspicious = false
   while true:
     var c = L.buf[pos]
     case c
@@ -892,21 +892,26 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
       c = chr(ord(c) + (ord('a') - ord('A'))) # toLower()
       h = h !& ord(c)
       inc(pos)
+      suspicious = true
     of '_':
       if L.buf[pos+1] notin SymChars:
         lexMessage(L, errGenerated, "invalid token: trailing underscore")
         break
       inc(pos)
+      suspicious = true
     else: break
   tokenEnd(tok, pos-1)
   h = !$h
   tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
-  L.bufpos = pos
   if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
       (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
     tok.tokType = tkSymbol
   else:
     tok.tokType = TTokType(tok.ident.id + ord(tkSymbol))
+    if suspicious and {optStyleHint, optStyleError} * L.config.globalOptions != {}:
+      lintReport(L.config, getLineInfo(L), tok.ident.s.normalize, tok.ident.s)
+  L.bufpos = pos
+
 
 proc endOperator(L: var TLexer, tok: var TToken, pos: int,
                  hash: Hash) {.inline.} =
@@ -943,15 +948,14 @@ proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
 
   case tok.tokType
   of tkOpr:
-    let L = tok.ident.s.len
     let relevantChar = tok.ident.s[0]
 
     # arrow like?
-    if L > 1 and tok.ident.s[L-1] == '>' and
-      tok.ident.s[L-2] in {'-', '~', '='}: return considerStrongSpaces(1)
+    if tok.ident.s.len > 1 and tok.ident.s[^1] == '>' and
+      tok.ident.s[^2] in {'-', '~', '='}: return considerStrongSpaces(1)
 
     template considerAsgn(value: untyped) =
-      result = if tok.ident.s[L-1] == '=': 1 else: value
+      result = if tok.ident.s[^1] == '=': 1 else: value
 
     case relevantChar
     of '$', '^': considerAsgn(10)
@@ -1077,7 +1081,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
     var lastBackslash = -1
     while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
       if L.buf[pos] == '\\': lastBackslash = pos+1
-      add(tok.literal, L.buf[pos])
+      tok.literal.add(L.buf[pos])
       inc(pos)
     tokenEndIgnore(tok, pos)
     pos = handleCRLF(L, pos)
@@ -1119,7 +1123,7 @@ proc skip(L: var TLexer, tok: var TToken) =
       inc(pos)
       inc(tok.strongSpaceA)
     of '\t':
-      if not L.allowTabs: lexMessagePos(L, errGenerated, pos, "tabulators are not allowed")
+      if not L.allowTabs: lexMessagePos(L, errGenerated, pos, "tabs are not allowed, use spaces instead")
       inc(pos)
     of CR, LF:
       tokenEndPrevious(tok, pos)
@@ -1174,8 +1178,6 @@ proc skip(L: var TLexer, tok: var TToken) =
       tok.commentOffsetB = L.offsetBase + pos - 1
       tok.tokType = tkComment
       tok.indent = commentIndent
-    if gIndentationWidth <= 0:
-      gIndentationWidth = tok.indent
 
 proc rawGetTok*(L: var TLexer, tok: var TToken) =
   template atTokenEnd() {.dirty.} =
@@ -1334,10 +1336,13 @@ proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
   var tok: TToken
   initToken(tok)
   openLexer(lex, fileIdx, inputstream, cache, config)
-  while true:
+  var prevToken = tkEof
+  while tok.tokType != tkEof:
     rawGetTok(lex, tok)
-    result = tok.indent
-    if result > 0 or tok.tokType == tkEof: break
+    if tok.indent > 0 and prevToken in {tkColon, tkEquals, tkType, tkConst, tkLet, tkVar, tkUsing}:
+      result = tok.indent
+      if result > 0: break
+    prevToken = tok.tokType
   closeLexer(lex)
 
 proc getPrecedence*(ident: PIdent): int =
@@ -1346,7 +1351,7 @@ proc getPrecedence*(ident: PIdent): int =
   initToken(tok)
   tok.ident = ident
   tok.tokType =
-    if tok.ident.id in ord(tokKeywordLow) - ord(tkSymbol) .. ord(tokKeywordHigh) - ord(tkSymbol):
+    if tok.ident.id in ord(tokKeywordLow) - ord(tkSymbol)..ord(tokKeywordHigh) - ord(tkSymbol):
       TTokType(tok.ident.id + ord(tkSymbol))
     else: tkOpr
   getPrecedence(tok, false)

compiler/liftlocals.nim

@@ -10,7 +10,7 @@
 ## This module implements the '.liftLocals' pragma.
 
 import
-  intsets, strutils, options, ast, astalgo, msgs,
+  strutils, options, ast, msgs,
   idents, renderer, types, lowerings, lineinfos
 
 from pragmas import getPragmaVal
@@ -30,10 +30,10 @@ proc lookupOrAdd(c: var Ctx; s: PSym; info: TLineInfo): PNode =
   let field = addUniqueField(c.objType, s, c.cache)
   var deref = newNodeI(nkHiddenDeref, info)
   deref.typ = c.objType
-  add(deref, newSymNode(c.partialParam, info))
+  deref.add(newSymNode(c.partialParam, info))
   result = newNodeI(nkDotExpr, info)
-  add(result, deref)
-  add(result, newSymNode(field))
+  result.add(deref)
+  result.add(newSymNode(field))
   result.typ = field.typ
 
 proc liftLocals(n: PNode; i: int; c: var Ctx) =
@@ -44,12 +44,12 @@ proc liftLocals(n: PNode; i: int; c: var Ctx) =
       n[i] = lookupOrAdd(c, it.sym, it.info)
   of procDefs, nkTypeSection: discard
   else:
-    for i in 0 ..< it.safeLen:
+    for i in 0..<it.safeLen:
       liftLocals(it, i, c)
 
 proc lookupParam(params, dest: PNode): PSym =
   if dest.kind != nkIdent: return nil
-  for i in 1 ..< params.len:
+  for i in 1..<params.len:
     if params[i].kind == nkSym and params[i].sym.name.id == dest.ident.id:
       return params[i].sym
 

compiler/lineinfos.nim

@@ -13,7 +13,7 @@
 import ropes, tables, pathutils
 
 const
-  explanationsBaseUrl* = "https://nim-lang.org/docs/manual"
+  explanationsBaseUrl* = "https://nim-lang.org/docs"
 
 type
   TMsgKind* = enum
@@ -23,6 +23,7 @@ type
     errGeneralParseError,
     errNewSectionExpected,
     errInvalidDirectiveX,
+    errProveInit,
     errGenerated,
     errUser,
     warnCannotOpenFile,
@@ -33,12 +34,20 @@ type
     warnFieldXNotSupported, warnCommentXIgnored,
     warnTypelessParam,
     warnUseBase, warnWriteToForeignHeap, warnUnsafeCode,
+    warnUnusedImportX,
+    warnInheritFromException,
     warnEachIdentIsTuple,
-    warnProveInit, warnProveField, warnProveIndex, warnGcUnsafe, warnGcUnsafe2,
+    warnUnsafeSetLen,
+    warnUnsafeDefault,
+    warnProveInit, warnProveField, warnProveIndex,
+    warnStaticIndexCheck, warnGcUnsafe, warnGcUnsafe2,
     warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed,
-    warnInconsistentSpacing, warnCaseTransition, warnUser,
+    warnInconsistentSpacing, warnCaseTransition, warnCycleCreated,
+    warnObservableStores,
+    warnUser,
     hintSuccess, hintSuccessX, hintCC,
-    hintLineTooLong, hintXDeclaredButNotUsed, hintConvToBaseNotNeeded,
+    hintLineTooLong, hintXDeclaredButNotUsed,
+    hintConvToBaseNotNeeded,
     hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled,
     hintProcessing, hintCodeBegin, hintCodeEnd, hintConf, hintPath,
     hintConditionAlwaysTrue, hintConditionAlwaysFalse, hintName, hintPattern,
@@ -59,6 +68,7 @@ const
     errGeneralParseError: "general parse error",
     errNewSectionExpected: "new section expected",
     errInvalidDirectiveX: "invalid directive: '$1'",
+    errProveInit: "Cannot prove that '$1' is initialized.",
     errGenerated: "$1",
     errUser: "$1",
     warnCannotOpenFile: "cannot open '$1'",
@@ -78,10 +88,17 @@ const
     warnUseBase: "use {.base.} for base methods; baseless methods are deprecated",
     warnWriteToForeignHeap: "write to foreign heap",
     warnUnsafeCode: "unsafe code: '$1'",
+    warnUnusedImportX: "imported and not used: '$1'",
+    warnInheritFromException: "inherit from a more precise exception type like ValueError, " &
+      "IOError or OSError. If these don't suit, inherit from CatchableError or Defect.",
     warnEachIdentIsTuple: "each identifier is a tuple",
+    warnUnsafeSetLen: "setLen can potentially expand the sequence, " &
+                      "but the element type '$1' doesn't have a valid default value",
+    warnUnsafeDefault: "The '$1' type doesn't have a valid default value",
     warnProveInit: "Cannot prove that '$1' is initialized. This will become a compile time error in the future.",
     warnProveField: "cannot prove that field '$1' is accessible",
     warnProveIndex: "cannot prove index '$1' is valid",
+    warnStaticIndexCheck: "$1",
     warnGcUnsafe: "not GC-safe: '$1'",
     warnGcUnsafe2: "$1",
     warnUninit: "'$1' might not have been initialized",
@@ -91,10 +108,13 @@ const
     warnResultShadowed: "Special variable 'result' is shadowed.",
     warnInconsistentSpacing: "Number of spaces around '$#' is not consistent",
     warnCaseTransition: "Potential object case transition, instantiate new object instead",
+    warnCycleCreated: "$1",
+    warnObservableStores: "observable stores to '$1'",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
-    hintSuccessX: "operation successful ($# lines compiled; $# sec total; $#; $#)",
-    hintCC: "CC: \'$1\'", # unused
+    # keep in sync with `testament.isSuccess`
+    hintSuccessX: "$loc LOC; $sec sec; $mem; $build build; proj: $project; out: $output",
+    hintCC: "CC: $1",
     hintLineTooLong: "line too long",
     hintXDeclaredButNotUsed: "'$1' is declared but not used",
     hintConvToBaseNotNeeded: "conversion to base object is not needed",
@@ -108,10 +128,10 @@ const
     hintPath: "added path: '$1'",
     hintConditionAlwaysTrue: "condition is always true: '$1'",
     hintConditionAlwaysFalse: "condition is always false: '$1'",
-    hintName: "name should be: '$1'",
+    hintName: "$1",
     hintPattern: "$1",
     hintExecuting: "$1",
-    hintLinking: "",
+    hintLinking: "$1",
     hintDependency: "$1",
     hintSource: "$1",
     hintPerformance: "$1",
@@ -133,14 +153,19 @@ const
     "LanguageXNotSupported", "FieldXNotSupported",
     "CommentXIgnored",
     "TypelessParam", "UseBase", "WriteToForeignHeap",
-    "UnsafeCode", "EachIdentIsTuple",
-    "ProveInit", "ProveField", "ProveIndex", "GcUnsafe", "GcUnsafe2", "Uninit",
+    "UnsafeCode", "UnusedImport", "InheritFromException",
+    "EachIdentIsTuple",
+    "UnsafeSetLen", "UnsafeDefault",
+    "ProveInit", "ProveField", "ProveIndex",
+    "IndexCheck", "GcUnsafe", "GcUnsafe2", "Uninit",
     "GcMem", "Destructor", "LockLevel", "ResultShadowed",
-    "Spacing", "CaseTransition", "User"]
+    "Spacing", "CaseTransition", "CycleCreated",
+    "ObservableStores", "User"]
 
   HintsToStr* = [
     "Success", "SuccessX", "CC", "LineTooLong",
-    "XDeclaredButNotUsed", "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded",
+    "XDeclaredButNotUsed",
+    "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded",
     "ExprAlwaysX", "QuitCalled", "Processing", "CodeBegin", "CodeEnd", "Conf",
     "Path", "CondTrue", "CondFalse", "Name", "Pattern", "Exec", "Link", "Dependency",
     "Source", "Performance", "StackTrace", "GCStats", "GlobalVar", "ExpandMacro",
@@ -193,7 +218,7 @@ type
                                #   used for better error messages and
                                #   embedding the original source in the
                                #   generated code
-    dirtyfile*: AbsoluteFile   # the file that is actually read into memory
+    dirtyFile*: AbsoluteFile   # the file that is actually read into memory
                                # and parsed; usually "" but is used
                                # for 'nimsuggest'
     hash*: string              # the checksum of the file
@@ -221,7 +246,7 @@ type
   TErrorOutputs* = set[TErrorOutput]
 
   ERecoverableError* = object of ValueError
-  ESuggestDone* = object of Exception
+  ESuggestDone* = object of ValueError
 
 proc `==`*(a, b: FileIndex): bool {.borrow.}
 
@@ -229,19 +254,17 @@ proc raiseRecoverableError*(msg: string) {.noinline.} =
   raise newException(ERecoverableError, msg)
 
 const
-  InvalidFileIDX* = FileIndex(-1)
-
-proc unknownLineInfo*(): TLineInfo =
-  result.line = uint16(0)
-  result.col = int16(-1)
-  result.fileIndex = InvalidFileIDX
+  InvalidFileIdx* = FileIndex(-1)
+  unknownLineInfo* = TLineInfo(line: 0, col: -1, fileIndex: InvalidFileIdx)
 
 type
   Severity* {.pure.} = enum ## VS Code only supports these three
     Hint, Warning, Error
 
-const trackPosInvalidFileIdx* = FileIndex(-2) # special marker so that no suggestions
-                                   # are produced within comments and string literals
+const
+  trackPosInvalidFileIdx* = FileIndex(-2) # special marker so that no suggestions
+                                          # are produced within comments and string literals
+  commandLineIdx* = FileIndex(-3)
 
 type
   MsgConfig* = object ## does not need to be stored in the incremental cache
@@ -259,7 +282,7 @@ type
 
 proc initMsgConfig*(): MsgConfig =
   result.msgContext = @[]
-  result.lastError = unknownLineInfo()
+  result.lastError = unknownLineInfo
   result.filenameToIndexTbl = initTable[string, FileIndex]()
   result.fileInfos = @[]
   result.errorOutputs = {eStdOut, eStdErr}

compiler/linter.nim

@@ -9,11 +9,9 @@
 
 ## This module implements the style checker.
 
-import
-  strutils, os, intsets, strtabs
+import strutils
 
-import options, ast, astalgo, msgs, semdata, ropes, idents,
-  lineinfos, pathutils
+import options, ast, msgs, idents, lineinfos, wordrecg
 
 const
   Letters* = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF', '_'}
@@ -22,33 +20,6 @@ proc identLen*(line: string, start: int): int =
   while start+result < line.len and line[start+result] in Letters:
     inc result
 
-type
-  StyleCheck* {.pure.} = enum None, Warn, Auto
-
-var
-  gOverWrite* = true
-  gStyleCheck*: StyleCheck
-  gCheckExtern*, gOnlyMainfile*: bool
-
-proc overwriteFiles*(conf: ConfigRef) =
-  let doStrip = options.getConfigVar(conf, "pretty.strip").normalize == "on"
-  for i in 0 .. high(conf.m.fileInfos):
-    if conf.m.fileInfos[i].dirty and
-        (not gOnlyMainfile or FileIndex(i) == conf.projectMainIdx):
-      let newFile = if gOverWrite: conf.m.fileInfos[i].fullpath
-                    else: conf.m.fileInfos[i].fullpath.changeFileExt(".pretty.nim")
-      try:
-        var f = open(newFile.string, fmWrite)
-        for line in conf.m.fileInfos[i].lines:
-          if doStrip:
-            f.write line.strip(leading = false, trailing = true)
-          else:
-            f.write line
-          f.write(conf.m.fileInfos[i], "\L")
-        f.close
-      except IOError:
-        rawMessage(conf, errGenerated, "cannot open file: " & newFile.string)
-
 proc `=~`(s: string, a: openArray[string]): bool =
   for x in a:
     if s.startsWith(x): return true
@@ -66,10 +37,10 @@ proc beautifyName(s: string, k: TSymKind): string =
     # Types should start with a capital unless builtins like 'int' etc.:
     if s =~ ["int", "uint", "cint", "cuint", "clong", "cstring", "string",
              "char", "byte", "bool", "openArray", "seq", "array", "void",
-             "pointer", "float", "csize", "cdouble", "cchar", "cschar",
+             "pointer", "float", "csize", "csize_t", "cdouble", "cchar", "cschar",
              "cshort", "cu", "nil", "typedesc", "auto", "any",
              "range", "openarray", "varargs", "set", "cfloat", "ref", "ptr",
-             "untyped", "typed", "static", "sink", "lent", "type"]:
+             "untyped", "typed", "static", "sink", "lent", "type", "owned"]:
       result.add s[i]
     else:
       result.add toUpperAscii(s[i])
@@ -98,47 +69,25 @@ proc beautifyName(s: string, k: TSymKind): string =
       result.add s[i]
     inc i
 
-proc differ*(line: string, a, b: int, x: string): bool =
-  let y = line[a..b]
-  result = cmpIgnoreStyle(y, x) == 0 and y != x
+proc differ*(line: string, a, b: int, x: string): string =
+  proc substrEq(s: string, pos, last: int, substr: string): bool =
+    var i = 0
+    while i < substr.len and pos+i <= last and s[pos+i] == substr[i]:
+      inc i
+    return i == substr.len
 
-proc replaceInFile(conf: ConfigRef; info: TLineInfo; newName: string) =
-  let line = conf.m.fileInfos[info.fileIndex.int].lines[info.line.int-1]
-  var first = min(info.col.int, line.len)
-  if first < 0: return
-  #inc first, skipIgnoreCase(line, "proc ", first)
-  while first > 0 and line[first-1] in Letters: dec first
-  if first < 0: return
-  if line[first] == '`': inc first
+  let last = min(b, line.len)
 
-  let last = first+identLen(line, first)-1
-  if differ(line, first, last, newName):
-    # last-first+1 != newName.len or
-    var x = line.substr(0, first-1) & newName & line.substr(last+1)
-    system.shallowCopy(conf.m.fileInfos[info.fileIndex.int].lines[info.line.int-1], x)
-    conf.m.fileInfos[info.fileIndex.int].dirty = true
-
-proc lintReport(conf: ConfigRef; info: TLineInfo, beau: string) =
-  if optStyleError in conf.globalOptions:
-    localError(conf, info, "name should be: '$1'" % beau)
-  else:
-    message(conf, info, hintName, beau)
+  result = ""
+  if not substrEq(line, a, b, x):
+    let y = line[a..b]
+    if cmpIgnoreStyle(y, x) == 0:
+      result = y
 
 proc checkStyle(conf: ConfigRef; cache: IdentCache; info: TLineInfo, s: string, k: TSymKind; sym: PSym) =
   let beau = beautifyName(s, k)
   if s != beau:
-    if gStyleCheck == StyleCheck.Auto:
-      sym.name = getIdent(cache, beau)
-      replaceInFile(conf, info, beau)
-    else:
-      lintReport(conf, info, beau)
-
-proc styleCheckDefImpl(conf: ConfigRef; cache: IdentCache; info: TLineInfo; s: PSym; k: TSymKind) =
-  # operators stay as they are:
-  if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
-  if k in {skType, skGenericParam} and sfAnon in s.flags: return
-  if {sfImportc, sfExportc} * s.flags == {} or gCheckExtern:
-    checkStyle(conf, cache, info, s.name.s, k, s)
+    lintReport(conf, info, beau, s)
 
 proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
   # operators stay as they are:
@@ -146,35 +95,47 @@ proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
   if k in {skType, skGenericParam} and sfAnon in s.flags: return
   if s.typ != nil and s.typ.kind == tyTypeDesc: return
   if {sfImportc, sfExportc} * s.flags != {}: return
+  if optStyleCheck notin s.options: return
   let beau = beautifyName(s.name.s, k)
   if s.name.s != beau:
-    lintReport(conf, info, beau)
+    lintReport(conf, info, beau, s.name.s)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
   if {optStyleHint, optStyleError} * conf.globalOptions != {}:
     nep1CheckDefImpl(conf, info, s, k)
-  when defined(nimfix):
-    if gStyleCheck != StyleCheck.None: styleCheckDefImpl(conf, cache, info, s, k)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) =
   styleCheckDef(conf, info, s, s.kind)
 template styleCheckDef*(conf: ConfigRef; s: PSym) =
   styleCheckDef(conf, s.info, s, s.kind)
 
-proc styleCheckUseImpl(conf: ConfigRef; info: TLineInfo; s: PSym) =
+proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string =
+  let line = sourceLine(conf, info)
+  var first = min(info.col.int, line.len)
+  if first < 0: return
+  #inc first, skipIgnoreCase(line, "proc ", first)
+  while first > 0 and line[first-1] in Letters: dec first
+  if first < 0: return
+  if first+1 < line.len and line[first] == '`': inc first
+
+  let last = first+identLen(line, first)-1
+  result = differ(line, first, last, newName)
+
+proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) =
   if info.fileIndex.int < 0: return
   # we simply convert it to what it looks like in the definition
   # for consistency
 
   # operators stay as they are:
-  if s.kind in {skResult, skTemp} or s.name.s[0] notin Letters:
+  if s.kind == skTemp or s.name.s[0] notin Letters or sfAnon in s.flags:
     return
-  if s.kind in {skType, skGenericParam} and sfAnon in s.flags: return
+
   let newName = s.name.s
+  let oldName = differs(conf, info, newName)
+  if oldName.len > 0:
+    lintReport(conf, info, newName, oldName)
 
-  replaceInFile(conf, info, newName)
-  #if newName == "File": writeStackTrace()
-
-template styleCheckUse*(info: TLineInfo; s: PSym) =
-  when defined(nimfix):
-    if gStyleCheck != StyleCheck.None: styleCheckUseImpl(conf, info, s)
+proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
+  let wanted = canonPragmaSpelling(w)
+  if pragmaName != wanted:
+    lintReport(conf, info, wanted, pragmaName)

compiler/llstream.nim

@@ -10,11 +10,13 @@
 ## Low-level streams for high performance.
 
 import
-  strutils, pathutils
+  pathutils
 
-# support '-d:useGnuReadline' for backwards compatibility:
-when not defined(windows) and (defined(useGnuReadline) or defined(useLinenoise)):
-  import rdstdin
+# support `useGnuReadline`, `useLinenoise` for backwards compatibility
+const hasRstdin = (defined(nimUseLinenoise) or defined(useLinenoise) or defined(useGnuReadline)) and
+  not defined(windows)
+
+when hasRstdin: import rdstdin
 
 type
   TLLRepl* = proc (s: PLLStream, buf: pointer, bufLen: int): int
@@ -97,7 +99,7 @@ proc continueLine(line: string, inTripleString: bool): bool {.inline.} =
 
 proc countTriples(s: string): int =
   var i = 0
-  while i < s.len:
+  while i+2 < s.len:
     if s[i] == '"' and s[i+1] == '"' and s[i+2] == '"':
       inc result
       inc i, 2
@@ -109,12 +111,12 @@ proc llReadFromStdin(s: PLLStream, buf: pointer, bufLen: int): int =
   var line = newStringOfCap(120)
   var triples = 0
   while readLineFromStdin(if s.s.len == 0: ">>> " else: "... ", line):
-    add(s.s, line)
-    add(s.s, "\n")
+    s.s.add(line)
+    s.s.add("\n")
     inc triples, countTriples(line)
     if not continueLine(line, (triples and 1) == 1): break
   inc(s.lineOffset)
-  result = min(bufLen, len(s.s) - s.rd)
+  result = min(bufLen, s.s.len - s.rd)
   if result > 0:
     copyMem(buf, addr(s.s[s.rd]), result)
     inc(s.rd, result)
@@ -124,7 +126,7 @@ proc llStreamRead*(s: PLLStream, buf: pointer, bufLen: int): int =
   of llsNone:
     result = 0
   of llsString:
-    result = min(bufLen, len(s.s) - s.rd)
+    result = min(bufLen, s.s.len - s.rd)
     if result > 0:
       copyMem(buf, addr(s.s[0 + s.rd]), result)
       inc(s.rd, result)
@@ -139,7 +141,7 @@ proc llStreamReadLine*(s: PLLStream, line: var string): bool =
   of llsNone:
     result = true
   of llsString:
-    while s.rd < len(s.s):
+    while s.rd < s.s.len:
       case s.s[s.rd]
       of '\x0D':
         inc(s.rd)
@@ -149,9 +151,9 @@ proc llStreamReadLine*(s: PLLStream, line: var string): bool =
         inc(s.rd)
         break
       else:
-        add(line, s.s[s.rd])
+        line.add(s.s[s.rd])
         inc(s.rd)
-    result = line.len > 0 or s.rd < len(s.s)
+    result = line.len > 0 or s.rd < s.s.len
   of llsFile:
     result = readLine(s.f, line)
   of llsStdIn:
@@ -162,8 +164,8 @@ proc llStreamWrite*(s: PLLStream, data: string) =
   of llsNone, llsStdIn:
     discard
   of llsString:
-    add(s.s, data)
-    inc(s.wr, len(data))
+    s.s.add(data)
+    inc(s.wr, data.len)
   of llsFile:
     write(s.f, data)
 
@@ -177,7 +179,7 @@ proc llStreamWrite*(s: PLLStream, data: char) =
   of llsNone, llsStdIn:
     discard
   of llsString:
-    add(s.s, data)
+    s.s.add(data)
     inc(s.wr)
   of llsFile:
     c = data
@@ -189,7 +191,7 @@ proc llStreamWrite*(s: PLLStream, buf: pointer, buflen: int) =
     discard
   of llsString:
     if buflen > 0:
-      setLen(s.s, len(s.s) + buflen)
+      setLen(s.s, s.s.len + buflen)
       copyMem(addr(s.s[0 + s.wr]), buf, buflen)
       inc(s.wr, buflen)
   of llsFile:
@@ -204,7 +206,7 @@ proc llStreamReadAll*(s: PLLStream): string =
   of llsString:
     if s.rd == 0: result = s.s
     else: result = substr(s.s, s.rd)
-    s.rd = len(s.s)
+    s.rd = s.s.len
   of llsFile:
     result = newString(bufSize)
     var bytes = readBuffer(s.f, addr(result[0]), bufSize)

compiler/lookups.nim

@@ -11,7 +11,7 @@
 
 import
   intsets, ast, astalgo, idents, semdata, types, msgs, options,
-  renderer, wordrecg, idgen, nimfix/prettybase, lineinfos, strutils
+  renderer, nimfix/prettybase, lineinfos, strutils
 
 proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope)
 
@@ -36,11 +36,11 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
   of nkAccQuoted:
     case n.len
     of 0: handleError(n, origin)
-    of 1: result = considerQuotedIdent(c, n.sons[0], origin)
+    of 1: result = considerQuotedIdent(c, n[0], origin)
     else:
       var id = ""
       for i in 0..<n.len:
-        let x = n.sons[i]
+        let x = n[i]
         case x.kind
         of nkIdent: id.add(x.ident.s)
         of nkSym: id.add(x.sym.name.s)
@@ -49,7 +49,7 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
       result = getIdent(c.cache, id)
   of nkOpenSymChoice, nkClosedSymChoice:
     if n[0].kind == nkSym:
-      result = n.sons[0].sym.name
+      result = n[0].sym.name
     else:
       handleError(n, origin)
   else:
@@ -100,16 +100,15 @@ proc searchInScopes*(c: PContext, s: PIdent): PSym =
     if result != nil: return
   result = nil
 
-when declared(echo):
-  proc debugScopes*(c: PContext; limit=0) {.deprecated.} =
-    var i = 0
-    for scope in walkScopes(c.currentScope):
-      echo "scope ", i
-      for h in 0 .. high(scope.symbols.data):
-        if scope.symbols.data[h] != nil:
-          echo scope.symbols.data[h].name.s
-      if i == limit: break
-      inc i
+proc debugScopes*(c: PContext; limit=0) {.deprecated.} =
+  var i = 0
+  for scope in walkScopes(c.currentScope):
+    echo "scope ", i
+    for h in 0..high(scope.symbols.data):
+      if scope.symbols.data[h] != nil:
+        echo scope.symbols.data[h].name.s
+    if i == limit: break
+    inc i
 
 proc searchInScopes*(c: PContext, s: PIdent, filter: TSymKinds): PSym =
   for scope in walkScopes(c.currentScope):
@@ -124,7 +123,7 @@ proc errorSym*(c: PContext, n: PNode): PSym =
   ## creates an error symbol to avoid cascading errors (for IDE support)
   var m = n
   # ensure that 'considerQuotedIdent' can't fail:
-  if m.kind == nkDotExpr: m = m.sons[1]
+  if m.kind == nkDotExpr: m = m[1]
   let ident = if m.kind in {nkIdent, nkSym, nkAccQuoted}:
       considerQuotedIdent(c, m)
     else:
@@ -233,7 +232,7 @@ proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) =
   addInterfaceDeclAux(c, sym)
 
 when defined(nimfix):
-  # when we cannot find the identifier, retry with a changed identifer:
+  # when we cannot find the identifier, retry with a changed identifier:
   proc altSpelling(x: PIdent): PIdent =
     case x.s[0]
     of 'A'..'Z': result = getIdent(toLowerAscii(x.s[0]) & x.s.substr(1))
@@ -326,29 +325,29 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
       errorUseQualifier(c, n.info, n.sym)
   of nkDotExpr:
     result = nil
-    var m = qualifiedLookUp(c, n.sons[0], (flags*{checkUndeclared})+{checkModule})
+    var m = qualifiedLookUp(c, n[0], (flags*{checkUndeclared})+{checkModule})
     if m != nil and m.kind == skModule:
       var ident: PIdent = nil
-      if n.sons[1].kind == nkIdent:
-        ident = n.sons[1].ident
-      elif n.sons[1].kind == nkAccQuoted:
-        ident = considerQuotedIdent(c, n.sons[1])
+      if n[1].kind == nkIdent:
+        ident = n[1].ident
+      elif n[1].kind == nkAccQuoted:
+        ident = considerQuotedIdent(c, n[1])
       if ident != nil:
         if m == c.module:
           result = strTableGet(c.topLevelScope.symbols, ident).skipAlias(n, c.config)
         else:
           result = strTableGet(m.tab, ident).skipAlias(n, c.config)
         if result == nil and checkUndeclared in flags:
-          fixSpelling(n.sons[1], ident, searchInScopes)
-          errorUndeclaredIdentifier(c, n.sons[1].info, ident.s)
-          result = errorSym(c, n.sons[1])
-      elif n.sons[1].kind == nkSym:
-        result = n.sons[1].sym
+          fixSpelling(n[1], ident, searchInScopes)
+          errorUndeclaredIdentifier(c, n[1].info, ident.s)
+          result = errorSym(c, n[1])
+      elif n[1].kind == nkSym:
+        result = n[1].sym
       elif checkUndeclared in flags and
-           n.sons[1].kind notin {nkOpenSymChoice, nkClosedSymChoice}:
-        localError(c.config, n.sons[1].info, "identifier expected, but got: " &
-                   renderTree(n.sons[1]))
-        result = errorSym(c, n.sons[1])
+           n[1].kind notin {nkOpenSymChoice, nkClosedSymChoice}:
+        localError(c.config, n[1].info, "identifier expected, but got: " &
+                   renderTree(n[1]))
+        result = errorSym(c, n[1])
   else:
     result = nil
   when false:
@@ -372,13 +371,13 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
     o.mode = oimDone
   of nkDotExpr:
     o.mode = oimOtherModule
-    o.m = qualifiedLookUp(c, n.sons[0], {checkUndeclared, checkModule})
+    o.m = qualifiedLookUp(c, n[0], {checkUndeclared, checkModule})
     if o.m != nil and o.m.kind == skModule:
       var ident: PIdent = nil
-      if n.sons[1].kind == nkIdent:
-        ident = n.sons[1].ident
-      elif n.sons[1].kind == nkAccQuoted:
-        ident = considerQuotedIdent(c, n.sons[1], n)
+      if n[1].kind == nkIdent:
+        ident = n[1].ident
+      elif n[1].kind == nkAccQuoted:
+        ident = considerQuotedIdent(c, n[1], n)
       if ident != nil:
         if o.m == c.module:
           # a module may access its private members:
@@ -388,12 +387,12 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         else:
           result = initIdentIter(o.it, o.m.tab, ident).skipAlias(n, c.config)
       else:
-        noidentError(c.config, n.sons[1], n)
-        result = errorSym(c, n.sons[1])
+        noidentError(c.config, n[1], n)
+        result = errorSym(c, n[1])
   of nkClosedSymChoice, nkOpenSymChoice:
     o.mode = oimSymChoice
     if n[0].kind == nkSym:
-      result = n.sons[0].sym
+      result = n[0].sym
     else:
       o.mode = oimDone
       return nil
@@ -430,8 +429,8 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
   of oimOtherModule:
     result = nextIdentIter(o.it, o.m.tab).skipAlias(n, c.config)
   of oimSymChoice:
-    if o.symChoiceIndex < sonsLen(n):
-      result = n.sons[o.symChoiceIndex].sym
+    if o.symChoiceIndex < n.len:
+      result = n[o.symChoiceIndex].sym
       incl(o.inSymChoice, result.id)
       inc o.symChoiceIndex
     elif n.kind == nkOpenSymChoice:
@@ -439,19 +438,19 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
       o.mode = oimSymChoiceLocalLookup
       o.scope = c.currentScope
       result = firstIdentExcluding(o.it, o.scope.symbols,
-                                   n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
       while result == nil:
         o.scope = o.scope.parent
         if o.scope == nil: break
         result = firstIdentExcluding(o.it, o.scope.symbols,
-                                     n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                     n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
   of oimSymChoiceLocalLookup:
     result = nextIdentExcluding(o.it, o.scope.symbols, o.inSymChoice).skipAlias(n, c.config)
     while result == nil:
       o.scope = o.scope.parent
       if o.scope == nil: break
       result = firstIdentExcluding(o.it, o.scope.symbols,
-                                   n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
 
   when false:
     if result != nil and result.kind == skStub: loadStub(result)

compiler/lowerings.nim

@@ -14,44 +14,50 @@ const
 
 import ast, astalgo, types, idents, magicsys, msgs, options, modulegraphs,
   lineinfos
-from trees import getMagic
 
 proc newDeref*(n: PNode): PNode {.inline.} =
-  result = newNodeIT(nkHiddenDeref, n.info, n.typ.sons[0])
-  addSon(result, n)
+  result = newNodeIT(nkHiddenDeref, n.info, n.typ[0])
+  result.add n
 
 proc newTupleAccess*(g: ModuleGraph; tup: PNode, i: int): PNode =
   if tup.kind == nkHiddenAddr:
-    result = newNodeIT(nkHiddenAddr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar}))
-    result.addSon(newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar}).sons[i]))
-    addSon(result[0], tup[0])
+    result = newNodeIT(nkHiddenAddr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar, tyLent}))
+    result.add newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar, tyLent})[i])
+    result[0].add tup[0]
     var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
     lit.intVal = i
-    addSon(result[0], lit)
+    result[0].add lit
   else:
     result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(
-                       abstractInst).sons[i])
-    addSon(result, copyTree(tup))
+                       abstractInst)[i])
+    result.add copyTree(tup)
     var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
     lit.intVal = i
-    addSon(result, lit)
+    result.add lit
 
 proc addVar*(father, v: PNode) =
   var vpart = newNodeI(nkIdentDefs, v.info, 3)
-  vpart.sons[0] = v
-  vpart.sons[1] = newNodeI(nkEmpty, v.info)
-  vpart.sons[2] = vpart[1]
-  addSon(father, vpart)
+  vpart[0] = v
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = vpart[1]
+  father.add vpart
 
-proc newAsgnStmt(le, ri: PNode): PNode =
+proc addVar*(father, v, value: PNode) =
+  var vpart = newNodeI(nkIdentDefs, v.info, 3)
+  vpart[0] = v
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = value
+  father.add vpart
+
+proc newAsgnStmt*(le, ri: PNode): PNode =
   result = newNodeI(nkAsgn, le.info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
 
-proc newFastAsgnStmt(le, ri: PNode): PNode =
+proc newFastAsgnStmt*(le, ri: PNode): PNode =
   result = newNodeI(nkFastAsgn, le.info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
 
 proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   assert n.kind == nkVarTuple
@@ -64,13 +70,13 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
 
   var v = newNodeI(nkVarSection, value.info)
   let tempAsNode = newSymNode(temp)
-  v.addVar(tempAsNode)
+  v.addVar(tempAsNode, value)
   result.add(v)
 
-  result.add newAsgnStmt(tempAsNode, value)
-  for i in 0 .. n.len-3:
-    if n.sons[i].kind == nkSym: v.addVar(n.sons[i])
-    result.add newAsgnStmt(n.sons[i], newTupleAccess(g, tempAsNode, i))
+  for i in 0..<n.len-2:
+    let val = newTupleAccess(g, tempAsNode, i)
+    if n[i].kind == nkSym: v.addVar(n[i], val)
+    else: result.add newAsgnStmt(n[i], val)
 
 proc evalOnce*(g: ModuleGraph; value: PNode; owner: PSym): PNode =
   ## Turns (value) into (let tmp = value; tmp) so that 'value' can be re-used
@@ -90,10 +96,10 @@ proc evalOnce*(g: ModuleGraph; value: PNode; owner: PSym): PNode =
 
 proc newTupleAccessRaw*(tup: PNode, i: int): PNode =
   result = newNodeI(nkBracketExpr, tup.info)
-  addSon(result, copyTree(tup))
+  result.add copyTree(tup)
   var lit = newNodeI(nkIntLit, tup.info)
   lit.intVal = i
-  addSon(result, lit)
+  result.add lit
 
 proc newTryFinally*(body, final: PNode): PNode =
   result = newTree(nkHiddenTryStmt, body, newTree(nkFinally, final))
@@ -107,31 +113,31 @@ proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   let tempAsNode = newSymNode(temp) #newIdentNode(getIdent(genPrefix & $temp.id), value.info)
 
   var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
-  vpart.sons[0] = tempAsNode
-  vpart.sons[1] = newNodeI(nkEmpty, value.info)
-  vpart.sons[2] = value
-  addSon(v, vpart)
+  vpart[0] = tempAsNode
+  vpart[1] = newNodeI(nkEmpty, value.info)
+  vpart[2] = value
+  v.add vpart
   result.add(v)
 
-  let lhs = n.sons[0]
-  for i in 0 .. lhs.len-1:
-    result.add newAsgnStmt(lhs.sons[i], newTupleAccessRaw(tempAsNode, i))
+  let lhs = n[0]
+  for i in 0..<lhs.len:
+    result.add newAsgnStmt(lhs[i], newTupleAccessRaw(tempAsNode, i))
 
 proc lowerSwap*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   result = newNodeI(nkStmtList, n.info)
   # note: cannot use 'skTemp' here cause we really need the copy for the VM :-(
   var temp = newSym(skVar, getIdent(g.cache, genPrefix), owner, n.info, owner.options)
-  temp.typ = n.sons[1].typ
+  temp.typ = n[1].typ
   incl(temp.flags, sfFromGeneric)
 
   var v = newNodeI(nkVarSection, n.info)
   let tempAsNode = newSymNode(temp)
 
   var vpart = newNodeI(nkIdentDefs, v.info, 3)
-  vpart.sons[0] = tempAsNode
-  vpart.sons[1] = newNodeI(nkEmpty, v.info)
-  vpart.sons[2] = n[1]
-  addSon(v, vpart)
+  vpart[0] = tempAsNode
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = n[1]
+  v.add vpart
 
   result.add(v)
   result.add newFastAsgnStmt(n[1], n[2])
@@ -151,45 +157,53 @@ proc createObj*(g: ModuleGraph; owner: PSym, info: TLineInfo; final=true): PType
   s.typ = result
   result.sym = s
 
+template fieldCheck {.dirty.} =
+  when false:
+    if tfCheckedForDestructor in obj.flags:
+      echo "missed field ", field.name.s
+      writeStackTrace()
+
 proc rawAddField*(obj: PType; field: PSym) =
   assert field.kind == skField
-  field.position = sonsLen(obj.n)
-  addSon(obj.n, newSymNode(field))
+  field.position = obj.n.len
+  obj.n.add newSymNode(field)
+  propagateToOwner(obj, field.typ)
+  fieldCheck()
 
 proc rawIndirectAccess*(a: PNode; field: PSym; info: TLineInfo): PNode =
   # returns a[].field as a node
   assert field.kind == skField
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
-  addSon(deref, a)
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc rawDirectAccess*(obj, field: PSym): PNode =
   # returns a.field as a node
   assert field.kind == skField
   result = newNodeI(nkDotExpr, field.info)
-  addSon(result, newSymNode obj)
-  addSon(result, newSymNode field)
+  result.add newSymNode(obj)
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc lookupInRecord(n: PNode, id: int): PSym =
   result = nil
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      result = lookupInRecord(n.sons[i], id)
+    for i in 0..<n.len:
+      result = lookupInRecord(n[i], id)
       if result != nil: return
   of nkRecCase:
-    if n.sons[0].kind != nkSym: return
-    result = lookupInRecord(n.sons[0], id)
+    if n[0].kind != nkSym: return
+    result = lookupInRecord(n[0], id)
     if result != nil: return
-    for i in 1 ..< sonsLen(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = lookupInRecord(lastSon(n.sons[i]), id)
+        result = lookupInRecord(lastSon(n[i]), id)
         if result != nil: return
       else: discard
   of nkSym:
@@ -205,8 +219,11 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache) =
   let t = skipIntLit(s.typ)
   field.typ = t
   assert t.kind != tyTyped
-  field.position = sonsLen(obj.n)
-  addSon(obj.n, newSymNode(field))
+  propagateToOwner(obj, t)
+  field.position = obj.n.len
+  field.flags = s.flags * {sfCursor}
+  obj.n.add newSymNode(field)
+  fieldCheck()
 
 proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache): PSym {.discardable.} =
   result = lookupInRecord(obj.n, s.id)
@@ -217,45 +234,46 @@ proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache): PSym {.discardable
     let t = skipIntLit(s.typ)
     field.typ = t
     assert t.kind != tyTyped
-    field.position = sonsLen(obj.n)
-    addSon(obj.n, newSymNode(field))
+    propagateToOwner(obj, t)
+    field.position = obj.n.len
+    obj.n.add newSymNode(field)
     result = field
 
-proc newDotExpr(obj, b: PSym): PNode =
+proc newDotExpr*(obj, b: PSym): PNode =
   result = newNodeI(nkDotExpr, obj.info)
   let field = lookupInRecord(obj.typ.n, b.id)
   assert field != nil, b.name.s
-  addSon(result, newSymNode(obj))
-  addSon(result, newSymNode(field))
+  result.add newSymNode(obj)
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc indirectAccess*(a: PNode, b: int, info: TLineInfo): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
   while true:
     assert t.kind == tyObject
     field = lookupInRecord(t.n, b)
     if field != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
   #if field == nil:
   #  echo "FIELD ", b
   #  debug deref.typ
   assert field != nil
-  addSon(deref, a)
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
-proc indirectAccess(a: PNode, b: string, info: TLineInfo; cache: IdentCache): PNode =
+proc indirectAccess*(a: PNode, b: string, info: TLineInfo; cache: IdentCache): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
   let bb = getIdent(cache, b)
@@ -263,17 +281,17 @@ proc indirectAccess(a: PNode, b: string, info: TLineInfo; cache: IdentCache): PN
     assert t.kind == tyObject
     field = getSymFromList(t.n, bb)
     if field != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
   #if field == nil:
   #  echo "FIELD ", b
   #  debug deref.typ
   assert field != nil
-  addSon(deref, a)
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc getFieldFromObj*(t: PType; v: PSym): PSym =
@@ -283,7 +301,7 @@ proc getFieldFromObj*(t: PType; v: PSym): PSym =
     assert t.kind == tyObject
     result = lookupInRecord(t.n, v.id)
     if result != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
 
@@ -294,19 +312,19 @@ proc indirectAccess*(a: PNode, b: PSym, info: TLineInfo): PNode =
 proc indirectAccess*(a, b: PSym, info: TLineInfo): PNode =
   result = indirectAccess(newSymNode(a), b, info)
 
-proc genAddrOf*(n: PNode): PNode =
+proc genAddrOf*(n: PNode, typeKind = tyPtr): PNode =
   result = newNodeI(nkAddr, n.info, 1)
-  result.sons[0] = n
-  result.typ = newType(tyPtr, n.typ.owner)
+  result[0] = n
+  result.typ = newType(typeKind, n.typ.owner)
   result.typ.rawAddSon(n.typ)
 
 proc genDeref*(n: PNode; k = nkHiddenDeref): PNode =
   result = newNodeIT(k, n.info,
-                     n.typ.skipTypes(abstractInst).sons[0])
+                     n.typ.skipTypes(abstractInst)[0])
   result.add n
 
 proc callCodegenProc*(g: ModuleGraph; name: string;
-                      info: TLineInfo = unknownLineInfo();
+                      info: TLineInfo = unknownLineInfo;
                       arg1, arg2, arg3, optionalArgs: PNode = nil): PNode =
   result = newNodeI(nkCall, info)
   let sym = magicsys.getCompilerProc(g, name)
@@ -318,240 +336,9 @@ proc callCodegenProc*(g: ModuleGraph; name: string;
     if arg2 != nil: result.add arg2
     if arg3 != nil: result.add arg3
     if optionalArgs != nil:
-      for i in 1..optionalArgs.len-3:
+      for i in 1..<optionalArgs.len-2:
         result.add optionalArgs[i]
-    result.typ = sym.typ.sons[0]
-
-proc callProc(a: PNode): PNode =
-  result = newNodeI(nkCall, a.info)
-  result.add a
-  result.typ = a.typ.sons[0]
-
-# we have 4 cases to consider:
-# - a void proc --> nothing to do
-# - a proc returning GC'ed memory --> requires a flowVar
-# - a proc returning non GC'ed memory --> pass as hidden 'var' parameter
-# - not in a parallel environment --> requires a flowVar for memory safety
-type
-  TSpawnResult* = enum
-    srVoid, srFlowVar, srByVar
-  TFlowVarKind = enum
-    fvInvalid # invalid type T for 'FlowVar[T]'
-    fvGC      # FlowVar of a GC'ed type
-    fvBlob    # FlowVar of a blob type
-
-proc spawnResult*(t: PType; inParallel: bool): TSpawnResult =
-  if t.isEmptyType: srVoid
-  elif inParallel and not containsGarbageCollectedRef(t): srByVar
-  else: srFlowVar
-
-proc flowVarKind(t: PType): TFlowVarKind =
-  if t.skipTypes(abstractInst).kind in {tyRef, tyString, tySequence}: fvGC
-  elif containsGarbageCollectedRef(t): fvInvalid
-  else: fvBlob
-
-proc typeNeedsNoDeepCopy(t: PType): bool =
-  var t = t.skipTypes(abstractInst)
-  # for the tconvexhull example (and others) we're a bit lax here and pretend
-  # seqs and strings are *by value* only and 'shallow' doesn't exist!
-  if t.kind == tyString: return true
-  # note that seq[T] is fine, but 'var seq[T]' is not, so we need to skip 'var'
-  # for the stricter check and likewise we can skip 'seq' for a less
-  # strict check:
-  if t.kind in {tyVar, tyLent, tySequence}: t = t.lastSon
-  result = not containsGarbageCollectedRef(t)
-
-proc hoistExpr*(varSection, expr: PNode, name: PIdent, owner: PSym): PSym =
-  result = newSym(skLet, name, owner, varSection.info, owner.options)
-  result.flags.incl sfHoisted
-  result.typ = expr.typ
-
-  var varDef = newNodeI(nkIdentDefs, varSection.info, 3)
-  varDef.sons[0] = newSymNode(result)
-  varDef.sons[1] = newNodeI(nkEmpty, varSection.info)
-  varDef.sons[2] = expr
-
-  varSection.add varDef
-
-proc addLocalVar(g: ModuleGraph; varSection, varInit: PNode; owner: PSym; typ: PType;
-                 v: PNode; useShallowCopy=false): PSym =
-  result = newSym(skTemp, getIdent(g.cache, genPrefix), owner, varSection.info,
-                  owner.options)
-  result.typ = typ
-  incl(result.flags, sfFromGeneric)
-
-  var vpart = newNodeI(nkIdentDefs, varSection.info, 3)
-  vpart.sons[0] = newSymNode(result)
-  vpart.sons[1] = newNodeI(nkEmpty, varSection.info)
-  vpart.sons[2] = if varInit.isNil: v else: vpart[1]
-  varSection.add vpart
-  if varInit != nil:
-    if useShallowCopy and typeNeedsNoDeepCopy(typ):
-      varInit.add newFastAsgnStmt(newSymNode(result), v)
-    else:
-      let deepCopyCall = newNodeI(nkCall, varInit.info, 3)
-      deepCopyCall.sons[0] = newSymNode(getSysMagic(g, varSection.info, "deepCopy", mDeepCopy))
-      deepCopyCall.sons[1] = newSymNode(result)
-      deepCopyCall.sons[2] = v
-      varInit.add deepCopyCall
-
-discard """
-We generate roughly this:
-
-proc f_wrapper(thread, args) =
-  barrierEnter(args.barrier)  # for parallel statement
-  var a = args.a # thread transfer; deepCopy or shallowCopy or no copy
-                 # depending on whether we're in a 'parallel' statement
-  var b = args.b
-  var fv = args.fv
-
-  fv.owner = thread # optional
-  nimArgsPassingDone() # signal parent that the work is done
-  #
-  args.fv.blob = f(a, b, ...)
-  nimFlowVarSignal(args.fv)
-
-  # - or -
-  f(a, b, ...)
-  barrierLeave(args.barrier)  # for parallel statement
-
-stmtList:
-  var scratchObj
-  scratchObj.a = a
-  scratchObj.b = b
-
-  nimSpawn(f_wrapper, addr scratchObj)
-  scratchObj.fv # optional
-
-"""
-
-proc createWrapperProc(g: ModuleGraph; f: PNode; threadParam, argsParam: PSym;
-                       varSection, varInit, call, barrier, fv: PNode;
-                       spawnKind: TSpawnResult): PSym =
-  var body = newNodeI(nkStmtList, f.info)
-  body.flags.incl nfTransf # do not transform further
-
-  var threadLocalBarrier: PSym
-  if barrier != nil:
-    var varSection2 = newNodeI(nkVarSection, barrier.info)
-    threadLocalBarrier = addLocalVar(g, varSection2, nil, argsParam.owner,
-                                     barrier.typ, barrier)
-    body.add varSection2
-    body.add callCodegenProc(g, "barrierEnter", threadLocalBarrier.info,
-      threadLocalBarrier.newSymNode)
-  var threadLocalProm: PSym
-  if spawnKind == srByVar:
-    threadLocalProm = addLocalVar(g, varSection, nil, argsParam.owner, fv.typ, fv)
-  elif fv != nil:
-    internalAssert g.config, fv.typ.kind == tyGenericInst
-    threadLocalProm = addLocalVar(g, varSection, nil, argsParam.owner, fv.typ, fv)
-  body.add varSection
-  body.add varInit
-  if fv != nil and spawnKind != srByVar:
-    # generate:
-    #   fv.owner = threadParam
-    body.add newAsgnStmt(indirectAccess(threadLocalProm.newSymNode,
-      "owner", fv.info, g.cache), threadParam.newSymNode)
-
-  body.add callCodegenProc(g, "nimArgsPassingDone", threadParam.info,
-    threadParam.newSymNode)
-  if spawnKind == srByVar:
-    body.add newAsgnStmt(genDeref(threadLocalProm.newSymNode), call)
-  elif fv != nil:
-    let fk = fv.typ.sons[1].flowVarKind
-    if fk == fvInvalid:
-      localError(g.config, f.info, "cannot create a flowVar of type: " &
-        typeToString(fv.typ.sons[1]))
-    body.add newAsgnStmt(indirectAccess(threadLocalProm.newSymNode,
-      if fk == fvGC: "data" else: "blob", fv.info, g.cache), call)
-    if fk == fvGC:
-      let incRefCall = newNodeI(nkCall, fv.info, 2)
-      incRefCall.sons[0] = newSymNode(getSysMagic(g, fv.info, "GCref", mGCref))
-      incRefCall.sons[1] = indirectAccess(threadLocalProm.newSymNode,
-                                          "data", fv.info, g.cache)
-      body.add incRefCall
-    if barrier == nil:
-      # by now 'fv' is shared and thus might have beeen overwritten! we need
-      # to use the thread-local view instead:
-      body.add callCodegenProc(g, "nimFlowVarSignal", threadLocalProm.info,
-        threadLocalProm.newSymNode)
-  else:
-    body.add call
-  if barrier != nil:
-    body.add callCodegenProc(g, "barrierLeave", threadLocalBarrier.info,
-      threadLocalBarrier.newSymNode)
-
-  var params = newNodeI(nkFormalParams, f.info)
-  params.add newNodeI(nkEmpty, f.info)
-  params.add threadParam.newSymNode
-  params.add argsParam.newSymNode
-
-  var t = newType(tyProc, threadParam.owner)
-  t.rawAddSon nil
-  t.rawAddSon threadParam.typ
-  t.rawAddSon argsParam.typ
-  t.n = newNodeI(nkFormalParams, f.info)
-  t.n.add newNodeI(nkEffectList, f.info)
-  t.n.add threadParam.newSymNode
-  t.n.add argsParam.newSymNode
-
-  let name = (if f.kind == nkSym: f.sym.name.s else: genPrefix) & "Wrapper"
-  result = newSym(skProc, getIdent(g.cache, name), argsParam.owner, f.info,
-                  argsParam.options)
-  let emptyNode = newNodeI(nkEmpty, f.info)
-  result.ast = newProcNode(nkProcDef, f.info, body = body,
-      params = params, name = newSymNode(result), pattern = emptyNode,
-      genericParams = emptyNode, pragmas = emptyNode,
-      exceptions = emptyNode)
-  result.typ = t
-
-proc createCastExpr(argsParam: PSym; objType: PType): PNode =
-  result = newNodeI(nkCast, argsParam.info)
-  result.add newNodeI(nkEmpty, argsParam.info)
-  result.add newSymNode(argsParam)
-  result.typ = newType(tyPtr, objType.owner)
-  result.typ.rawAddSon(objType)
-
-proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType; scratchObj: PSym,
-                             castExpr, call,
-                             varSection, varInit, result: PNode) =
-  let formals = n[0].typ.n
-  let tmpName = getIdent(g.cache, genPrefix)
-  for i in 1 ..< n.len:
-    # we pick n's type here, which hopefully is 'tyArray' and not
-    # 'tyOpenArray':
-    var argType = n[i].typ.skipTypes(abstractInst)
-    if i < formals.len and formals[i].typ.kind in {tyVar, tyLent}:
-      localError(g.config, n[i].info, "'spawn'ed function cannot have a 'var' parameter")
-    #elif containsTyRef(argType):
-    #  localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure")
-
-    let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
-    var field = newSym(skField, fieldname, objType.owner, n.info, g.config.options)
-    field.typ = argType
-    objType.addField(field, g.cache)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[i])
-
-    let temp = addLocalVar(g, varSection, varInit, objType.owner, argType,
-                           indirectAccess(castExpr, field, n.info))
-    call.add(newSymNode(temp))
-
-proc getRoot*(n: PNode): PSym =
-  ## ``getRoot`` takes a *path* ``n``. A path is an lvalue expression
-  ## like ``obj.x[i].y``. The *root* of a path is the symbol that can be
-  ## determined as the owner; ``obj`` in the example.
-  case n.kind
-  of nkSym:
-    if n.sym.kind in {skVar, skResult, skTemp, skLet, skForVar}:
-      result = n.sym
-  of nkDotExpr, nkBracketExpr, nkHiddenDeref, nkDerefExpr,
-      nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
-    result = getRoot(n.sons[0])
-  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-    result = getRoot(n.sons[1])
-  of nkCallKinds:
-    if getMagic(n) == mSlice: result = getRoot(n.sons[1])
-  else: discard
+    result.typ = sym.typ[0]
 
 proc newIntLit*(g: ModuleGraph; info: TLineInfo; value: BiggestInt): PNode =
   result = nkIntLit.newIntNode(value)
@@ -559,212 +346,30 @@ proc newIntLit*(g: ModuleGraph; info: TLineInfo; value: BiggestInt): PNode =
 
 proc genHigh*(g: ModuleGraph; n: PNode): PNode =
   if skipTypes(n.typ, abstractVar).kind == tyArray:
-    result = newIntLit(g, n.info, lastOrd(g.config, skipTypes(n.typ, abstractVar)))
+    result = newIntLit(g, n.info, toInt64(lastOrd(g.config, skipTypes(n.typ, abstractVar))))
   else:
     result = newNodeI(nkCall, n.info, 2)
     result.typ = getSysType(g, n.info, tyInt)
-    result.sons[0] = newSymNode(getSysMagic(g, n.info, "high", mHigh))
-    result.sons[1] = n
+    result[0] = newSymNode(getSysMagic(g, n.info, "high", mHigh))
+    result[1] = n
 
 proc genLen*(g: ModuleGraph; n: PNode): PNode =
   if skipTypes(n.typ, abstractVar).kind == tyArray:
-    result = newIntLit(g, n.info, lastOrd(g.config, skipTypes(n.typ, abstractVar)) + 1)
+    result = newIntLit(g, n.info, toInt64(lastOrd(g.config, skipTypes(n.typ, abstractVar)) + 1))
   else:
     result = newNodeI(nkCall, n.info, 2)
     result.typ = getSysType(g, n.info, tyInt)
-    result.sons[0] = newSymNode(getSysMagic(g, n.info, "len", mLengthSeq))
-    result.sons[1] = n
+    result[0] = newSymNode(getSysMagic(g, n.info, "len", mLengthSeq))
+    result[1] = n
 
-proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; scratchObj: PSym;
-                             castExpr, call,
-                             varSection, varInit, result: PNode) =
-  let formals = n[0].typ.n
-  let tmpName = getIdent(g.cache, genPrefix)
-  # we need to copy the foreign scratch object fields into local variables
-  # for correctness: These are called 'threadLocal' here.
-  for i in 1 ..< n.len:
-    let n = n[i]
-    let argType = skipTypes(if i < formals.len: formals[i].typ else: n.typ,
-                            abstractInst)
-    #if containsTyRef(argType):
-    #  localError(n.info, "'spawn'ed function cannot refer to 'ref'/closure")
+proc hoistExpr*(varSection, expr: PNode, name: PIdent, owner: PSym): PSym =
+  result = newSym(skLet, name, owner, varSection.info, owner.options)
+  result.flags.incl sfHoisted
+  result.typ = expr.typ
 
-    let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
-    var field = newSym(skField, fieldname, objType.owner, n.info, g.config.options)
+  var varDef = newNodeI(nkIdentDefs, varSection.info, 3)
+  varDef[0] = newSymNode(result)
+  varDef[1] = newNodeI(nkEmpty, varSection.info)
+  varDef[2] = expr
 
-    if argType.kind in {tyVarargs, tyOpenArray}:
-      # important special case: we always create a zero-copy slice:
-      let slice = newNodeI(nkCall, n.info, 4)
-      slice.typ = n.typ
-      slice.sons[0] = newSymNode(createMagic(g, "slice", mSlice))
-      slice.sons[0].typ = getSysType(g, n.info, tyInt) # fake type
-      var fieldB = newSym(skField, tmpName, objType.owner, n.info, g.config.options)
-      fieldB.typ = getSysType(g, n.info, tyInt)
-      objType.addField(fieldB, g.cache)
-
-      if getMagic(n) == mSlice:
-        let a = genAddrOf(n[1])
-        field.typ = a.typ
-        objType.addField(field, g.cache)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
-
-        var fieldA = newSym(skField, tmpName, objType.owner, n.info, g.config.options)
-        fieldA.typ = getSysType(g, n.info, tyInt)
-        objType.addField(fieldA, g.cache)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldA), n[2])
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), n[3])
-
-        let threadLocal = addLocalVar(g, varSection,nil, objType.owner, fieldA.typ,
-                                      indirectAccess(castExpr, fieldA, n.info),
-                                      useShallowCopy=true)
-        slice.sons[2] = threadLocal.newSymNode
-      else:
-        let a = genAddrOf(n)
-        field.typ = a.typ
-        objType.addField(field, g.cache)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), genHigh(g, n))
-
-        slice.sons[2] = newIntLit(g, n.info, 0)
-      # the array itself does not need to go through a thread local variable:
-      slice.sons[1] = genDeref(indirectAccess(castExpr, field, n.info))
-
-      let threadLocal = addLocalVar(g, varSection,nil, objType.owner, fieldB.typ,
-                                    indirectAccess(castExpr, fieldB, n.info),
-                                    useShallowCopy=true)
-      slice.sons[3] = threadLocal.newSymNode
-      call.add slice
-    elif (let size = computeSize(g.config, argType); size < 0 or size > 16) and
-        n.getRoot != nil:
-      # it is more efficient to pass a pointer instead:
-      let a = genAddrOf(n)
-      field.typ = a.typ
-      objType.addField(field, g.cache)
-      result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
-      let threadLocal = addLocalVar(g, varSection,nil, objType.owner, field.typ,
-                                    indirectAccess(castExpr, field, n.info),
-                                    useShallowCopy=true)
-      call.add(genDeref(threadLocal.newSymNode))
-    else:
-      # boring case
-      field.typ = argType
-      objType.addField(field, g.cache)
-      result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n)
-      let threadLocal = addLocalVar(g, varSection, varInit,
-                                    objType.owner, field.typ,
-                                    indirectAccess(castExpr, field, n.info),
-                                    useShallowCopy=true)
-      call.add(threadLocal.newSymNode)
-
-proc wrapProcForSpawn*(g: ModuleGraph; owner: PSym; spawnExpr: PNode; retType: PType;
-                       barrier, dest: PNode = nil): PNode =
-  # if 'barrier' != nil, then it is in a 'parallel' section and we
-  # generate quite different code
-  let n = spawnExpr[^2]
-  let spawnKind = spawnResult(retType, barrier!=nil)
-  case spawnKind
-  of srVoid:
-    internalAssert g.config, dest == nil
-    result = newNodeI(nkStmtList, n.info)
-  of srFlowVar:
-    internalAssert g.config, dest == nil
-    result = newNodeIT(nkStmtListExpr, n.info, retType)
-  of srByVar:
-    if dest == nil: localError(g.config, n.info, "'spawn' must not be discarded")
-    result = newNodeI(nkStmtList, n.info)
-
-  if n.kind notin nkCallKinds:
-    localError(g.config, n.info, "'spawn' takes a call expression")
-    return
-  if optThreadAnalysis in g.config.globalOptions:
-    if {tfThread, tfNoSideEffect} * n[0].typ.flags == {}:
-      localError(g.config, n.info, "'spawn' takes a GC safe call expression")
-  var
-    threadParam = newSym(skParam, getIdent(g.cache, "thread"), owner, n.info, g.config.options)
-    argsParam = newSym(skParam, getIdent(g.cache, "args"), owner, n.info, g.config.options)
-  block:
-    let ptrType = getSysType(g, n.info, tyPointer)
-    threadParam.typ = ptrType
-    argsParam.typ = ptrType
-    argsParam.position = 1
-
-  var objType = createObj(g, owner, n.info)
-  incl(objType.flags, tfFinal)
-  let castExpr = createCastExpr(argsParam, objType)
-
-  var scratchObj = newSym(skVar, getIdent(g.cache, "scratch"), owner, n.info, g.config.options)
-  block:
-    scratchObj.typ = objType
-    incl(scratchObj.flags, sfFromGeneric)
-    var varSectionB = newNodeI(nkVarSection, n.info)
-    varSectionB.addVar(scratchObj.newSymNode)
-    result.add varSectionB
-
-  var call = newNodeIT(nkCall, n.info, n.typ)
-  var fn = n.sons[0]
-  # templates and macros are in fact valid here due to the nature of
-  # the transformation:
-  if fn.kind == nkClosure:
-    localError(g.config, n.info, "closure in spawn environment is not allowed")
-  if not (fn.kind == nkSym and fn.sym.kind in {skProc, skTemplate, skMacro,
-                                               skFunc, skMethod, skConverter}):
-    # for indirect calls we pass the function pointer in the scratchObj
-    var argType = n[0].typ.skipTypes(abstractInst)
-    var field = newSym(skField, getIdent(g.cache, "fn"), owner, n.info, g.config.options)
-    field.typ = argType
-    objType.addField(field, g.cache)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0])
-    fn = indirectAccess(castExpr, field, n.info)
-  elif fn.kind == nkSym and fn.sym.kind == skIterator:
-    localError(g.config, n.info, "iterator in spawn environment is not allowed")
-  elif fn.typ.callConv == ccClosure:
-    localError(g.config, n.info, "closure in spawn environment is not allowed")
-
-  call.add(fn)
-  var varSection = newNodeI(nkVarSection, n.info)
-  var varInit = newNodeI(nkStmtList, n.info)
-  if barrier.isNil:
-    setupArgsForConcurrency(g, n, objType, scratchObj, castExpr, call,
-                            varSection, varInit, result)
-  else:
-    setupArgsForParallelism(g, n, objType, scratchObj, castExpr, call,
-                            varSection, varInit, result)
-
-  var barrierAsExpr: PNode = nil
-  if barrier != nil:
-    let typ = newType(tyPtr, owner)
-    typ.rawAddSon(magicsys.getCompilerProc(g, "Barrier").typ)
-    var field = newSym(skField, getIdent(g.cache, "barrier"), owner, n.info, g.config.options)
-    field.typ = typ
-    objType.addField(field, g.cache)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), barrier)
-    barrierAsExpr = indirectAccess(castExpr, field, n.info)
-
-  var fvField, fvAsExpr: PNode = nil
-  if spawnKind == srFlowVar:
-    var field = newSym(skField, getIdent(g.cache, "fv"), owner, n.info, g.config.options)
-    field.typ = retType
-    objType.addField(field, g.cache)
-    fvField = newDotExpr(scratchObj, field)
-    fvAsExpr = indirectAccess(castExpr, field, n.info)
-    # create flowVar:
-    result.add newFastAsgnStmt(fvField, callProc(spawnExpr[^1]))
-    if barrier == nil:
-      result.add callCodegenProc(g, "nimFlowVarCreateSemaphore", fvField.info,
-        fvField)
-
-  elif spawnKind == srByVar:
-    var field = newSym(skField, getIdent(g.cache, "fv"), owner, n.info, g.config.options)
-    field.typ = newType(tyPtr, objType.owner)
-    field.typ.rawAddSon(retType)
-    objType.addField(field, g.cache)
-    fvAsExpr = indirectAccess(castExpr, field, n.info)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), genAddrOf(dest))
-
-  let wrapper = createWrapperProc(g, fn, threadParam, argsParam,
-                                  varSection, varInit, call,
-                                  barrierAsExpr, fvAsExpr, spawnKind)
-  result.add callCodegenProc(g, "nimSpawn" & $spawnExpr.len, wrapper.info,
-    wrapper.newSymNode, genAddrOf(scratchObj.newSymNode), nil, spawnExpr)
-
-  if spawnKind == srFlowVar: result.add fvField
+  varSection.add varDef

compiler/macrocacheimpl.nim

@@ -9,7 +9,7 @@
 
 ## This module implements helpers for the macro cache.
 
-import lineinfos, ast, modulegraphs, vmdef, magicsys
+import lineinfos, ast, modulegraphs, vmdef
 
 proc recordInc*(c: PCtx; info: TLineInfo; key: string; by: BiggestInt) =
   var recorded = newNodeI(nkCommentStmt, info)

compiler/magicsys.nim

@@ -10,7 +10,7 @@
 # Built-in types and compilerprocs are registered here.
 
 import
-  ast, astalgo, hashes, msgs, platform, nversion, times, idents,
+  ast, astalgo, msgs, platform, idents,
   modulegraphs, lineinfos
 
 export createMagic
@@ -40,7 +40,7 @@ proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSy
   while r != nil:
     if r.magic == m:
       # prefer the tyInt variant:
-      if r.typ.sons[0] != nil and r.typ.sons[0].kind == tyInt: return r
+      if r.typ[0] != nil and r.typ[0].kind == tyInt: return r
       result = r
     r = nextIdentIter(ti, g.systemModule.tab)
   if result != nil: return result
@@ -68,7 +68,7 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
     of tyUInt64: result = sysTypeFromName("uint64")
     of tyFloat: result = sysTypeFromName("float")
     of tyFloat32: result = sysTypeFromName("float32")
-    of tyFloat64: return sysTypeFromName("float64")
+    of tyFloat64: result = sysTypeFromName("float64")
     of tyFloat128: result = sysTypeFromName("float128")
     of tyBool: result = sysTypeFromName("bool")
     of tyChar: result = sysTypeFromName("char")
@@ -79,7 +79,9 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
     else: internalError(g.config, "request for typekind: " & $kind)
     g.sysTypes[kind] = result
   if result.kind != kind:
-    internalError(g.config, "wanted: " & $kind & " got: " & $result.kind)
+    if kind == tyFloat64 and result.kind == tyFloat: discard # because of aliasing
+    else:
+      internalError(g.config, "wanted: " & $kind & " got: " & $result.kind)
   if result == nil: internalError(g.config, "type not found: " & $kind)
 
 proc resetSysTypes*(g: ModuleGraph) =
@@ -123,7 +125,7 @@ proc addSonSkipIntLit*(father, son: PType) =
   when not defined(nimNoNilSeqs):
     if isNil(father.sons): father.sons = @[]
   let s = son.skipIntLit
-  add(father.sons, s)
+  father.sons.add(s)
   propagateToOwner(father, s)
 
 proc setIntLitType*(g: ModuleGraph; result: PNode) =
@@ -175,3 +177,28 @@ proc getNimScriptSymbol*(g: ModuleGraph; name: string): PSym =
   strTableGet(g.exposed, getIdent(g.cache, name))
 
 proc resetNimScriptSymbols*(g: ModuleGraph) = initStrTable(g.exposed)
+
+proc getMagicEqSymForType*(g: ModuleGraph; t: PType; info: TLineInfo): PSym =
+  case t.kind
+  of tyInt,  tyInt8, tyInt16, tyInt32, tyInt64,
+     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64: 
+    result = getSysMagic(g, info, "==", mEqI)
+  of tyEnum: 
+    result = getSysMagic(g, info, "==", mEqEnum)
+  of tyBool: 
+    result = getSysMagic(g, info, "==", mEqB)
+  of tyRef, tyPtr, tyPointer: 
+    result = getSysMagic(g, info, "==", mEqRef)
+  of tyString:
+    result = getSysMagic(g, info, "==", mEqStr)
+  of tyChar:
+    result = getSysMagic(g, info, "==", mEqCh)
+  of tySet:
+    result = getSysMagic(g, info, "==", mEqSet)
+  of tyProc:
+    result = getSysMagic(g, info, "==", mEqProc)
+  else:
+    globalError(g.config, info,
+      "can't find magic equals operator for type kind " & $t.kind)
+
+

compiler/main.nim

@@ -13,22 +13,17 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  llstream, strutils, ast, astalgo, lexer, syntaxes, renderer, options, msgs,
-  os, condsyms, times,
-  wordrecg, sem, semdata, idents, passes, extccomp,
+  llstream, strutils, os, ast, lexer, syntaxes, options, msgs,
+  condsyms, times,
+  sem, idents, passes, extccomp,
   cgen, json, nversion,
-  platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
-  parser, modules, ccgutils, sigmatch, ropes,
+  platform, nimconf, passaux, depends, vm, idgen,
+  modules,
   modulegraphs, tables, rod, lineinfos, pathutils
 
 when not defined(leanCompiler):
   import jsgen, docgen, docgen2
 
-from magicsys import resetSysTypes
-
-proc codegenPass(g: ModuleGraph) =
-  registerPass g, cgenPass
-
 proc semanticPasses(g: ModuleGraph) =
   registerPass g, verbosePass
   registerPass g, semPass
@@ -87,6 +82,16 @@ proc commandCompileToC(graph: ModuleGraph) =
   semanticPasses(graph)
   registerPass(graph, cgenPass)
 
+  if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"):
+    let proj = changeFileExt(conf.projectFull, "")
+    if not changeDetectedViaJsonBuildInstructions(conf, proj):
+      # nothing changed
+      graph.config.notes = graph.config.mainPackageNotes
+      return
+
+  if not extccomp.ccHasSaneOverflow(conf):
+    conf.symbols.defineSymbol("nimEmulateOverflowChecks")
+
   compileProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933
@@ -106,6 +111,7 @@ proc commandJsonScript(graph: ModuleGraph) =
 when not defined(leanCompiler):
   proc commandCompileToJS(graph: ModuleGraph) =
     let conf = graph.config
+    conf.exc = excCpp
 
     if conf.outDir.isEmpty:
       conf.outDir = conf.projectPath
@@ -148,14 +154,6 @@ const evalPasses = [verbosePass, semPass, evalPass]
 proc evalNim(graph: ModuleGraph; nodes: PNode, module: PSym) =
   carryPasses(graph, nodes, module, evalPasses)
 
-proc commandEval(graph: ModuleGraph; exp: string) =
-  if graph.systemModule == nil:
-    interactivePasses(graph)
-    compileSystemModule(graph)
-  let echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
-  evalNim(graph, echoExp.parseString(graph.cache, graph.config),
-    makeStdinModule(graph))
-
 proc commandScan(cache: IdentCache, config: ConfigRef) =
   var f = addFileExt(AbsoluteFile mainCommandArg(config), NimExt)
   var stream = llStreamOpen(f, fmRead)
@@ -190,10 +188,12 @@ proc mainCommand*(graph: ModuleGraph) =
   of "c", "cc", "compile", "compiletoc":
     # compile means compileToC currently
     conf.cmd = cmdCompileToC
+    if conf.exc == excNone: conf.exc = excSetjmp
     defineSymbol(graph.config.symbols, "c")
     commandCompileToC(graph)
   of "cpp", "compiletocpp":
     conf.cmd = cmdCompileToCpp
+    if conf.exc == excNone: conf.exc = excCpp
     defineSymbol(graph.config.symbols, "cpp")
     commandCompileToC(graph)
   of "objc", "compiletooc":
@@ -203,7 +203,7 @@ proc mainCommand*(graph: ModuleGraph) =
   of "run":
     conf.cmd = cmdRun
     when hasTinyCBackend:
-      extccomp.setCC("tcc")
+      extccomp.setCC(conf, "tcc", unknownLineInfo)
       commandCompileToC(graph)
     else:
       rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
@@ -228,6 +228,12 @@ proc mainCommand*(graph: ModuleGraph) =
       loadConfigs(DocConfig, cache, conf)
       commandDoc(cache, conf)
   of "doc2", "doc":
+    conf.setNoteDefaults(warnLockLevel, false) # issue #13218
+    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+      # because currently generates lots of false positives due to conflation
+      # of labels links in doc comments, eg for random.rand:
+      #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
+      #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
     when defined(leanCompiler):
       quit "compiler wasn't built with documentation generator"
     else:
@@ -236,6 +242,7 @@ proc mainCommand*(graph: ModuleGraph) =
       defineSymbol(conf.symbols, "nimdoc")
       commandDoc2(graph, false)
   of "rst2html":
+    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
     when defined(leanCompiler):
       quit "compiler wasn't built with documentation generator"
     else:
@@ -296,7 +303,9 @@ proc mainCommand*(graph: ModuleGraph) =
       for s in definedSymbolNames(conf.symbols): definedSymbols.elems.add(%s)
 
       var libpaths = newJArray()
+      var lazyPaths = newJArray()
       for dir in conf.searchPaths: libpaths.elems.add(%dir.string)
+      for dir in conf.lazyPaths: lazyPaths.elems.add(%dir.string)
 
       var hints = newJObject() # consider factoring with `listHints`
       for a in hintMin..hintMax:
@@ -309,9 +318,13 @@ proc mainCommand*(graph: ModuleGraph) =
 
       var dumpdata = %[
         (key: "version", val: %VersionAsString),
+        (key: "nimExe", val: %(getAppFilename())),
+        (key: "prefixdir", val: %conf.getPrefixDir().string),
+        (key: "libpath", val: %conf.libpath.string),
         (key: "project_path", val: %conf.projectFull.string),
         (key: "defined_symbols", val: definedSymbols),
         (key: "lib_paths", val: %libpaths),
+        (key: "lazyPaths", val: %lazyPaths),
         (key: "outdir", val: %conf.outDir.string),
         (key: "out", val: %conf.outFile.string),
         (key: "nimcache", val: %getNimcacheDir(conf).string),
@@ -343,8 +356,11 @@ proc mainCommand*(graph: ModuleGraph) =
     conf.cmd = cmdInteractive
     commandInteractive(graph)
   of "e":
-    incl conf.globalOptions, optWasNimscript
-    commandEval(graph, mainCommandArg(conf))
+    if not fileExists(conf.projectFull):
+      rawMessage(conf, errGenerated, "NimScript file does not exist: " & conf.projectFull.string)
+    elif not conf.projectFull.string.endsWith(".nims"):
+      rawMessage(conf, errGenerated, "not a NimScript file: " & conf.projectFull.string)
+    # main NimScript logic handled in cmdlinehelper.nim.
   of "nop", "help":
     # prevent the "success" message:
     conf.cmd = cmdDump
@@ -356,16 +372,25 @@ proc mainCommand*(graph: ModuleGraph) =
 
   if conf.errorCounter == 0 and
      conf.cmd notin {cmdInterpret, cmdRun, cmdDump}:
-    when declared(system.getMaxMem):
-      let usedMem = formatSize(getMaxMem()) & " peakmem"
-    else:
-      let usedMem = formatSize(getTotalMem())
-    rawMessage(conf, hintSuccessX, [$conf.linesCompiled,
-               formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3),
-               usedMem,
-               if isDefined(conf, "danger"): "Dangerous Release Build"
-               elif isDefined(conf, "release"): "Release Build"
-               else: "Debug Build"])
+    let mem =
+      when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem"
+      else: formatSize(getTotalMem()) & " totmem"
+    let loc = $conf.linesCompiled
+    let build = if isDefined(conf, "danger"): "Dangerous Release"
+                elif isDefined(conf, "release"): "Release"
+                else: "Debug"
+    let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3)
+    let project = if optListFullPaths in conf.globalOptions: $conf.projectFull else: $conf.projectName
+    var output = $conf.absOutFile
+    if optListFullPaths notin conf.globalOptions: output = output.AbsoluteFile.extractFilename
+    rawMessage(conf, hintSuccessX, [
+      "loc", loc,
+      "sec", sec,
+      "mem", mem,
+      "build", build,
+      "project", project,
+      "output", output,
+      ])
 
   when PrintRopeCacheStats:
     echo "rope cache stats: "

compiler/modulegraphs.nim

@@ -29,7 +29,7 @@ import ast, intsets, tables, options, lineinfos, hashes, idents,
   incremental, btrees, md5
 
 type
-  SigHash* = distinct Md5Digest
+  SigHash* = distinct MD5Digest
 
   ModuleGraph* = ref object
     modules*: seq[PSym]  ## indexed by int32 fileIdx
@@ -64,7 +64,7 @@ type
     importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.}
     includeFileCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PNode {.nimcall.}
     recordStmt*: proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.}
-    cacheSeqs*: Table[string, PNode] # state that is shared to suppor the 'macrocache' API
+    cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API
     cacheCounters*: Table[string, BiggestInt]
     cacheTables*: Table[string, BTree[string, PNode]]
     passes*: seq[TPass]
@@ -72,6 +72,10 @@ type
     onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     globalDestructors*: seq[PNode]
+    proofEngine*: proc (graph: ModuleGraph; assumptions: seq[PNode]; toProve: PNode): (bool, string) {.nimcall.}
+    requirementsCheck*: proc (graph: ModuleGraph; assumptions: seq[PNode];
+                             call, requirement: PNode): (bool, string) {.nimcall.}
+    compatibleProps*: proc (graph: ModuleGraph; formal, actual: PType): bool {.nimcall.}
 
   TPassContext* = object of RootObj # the pass's context
   PPassContext* = ref TPassContext
@@ -98,7 +102,7 @@ const
 proc toBase64a(s: cstring, len: int): string =
   ## encodes `s` into base64 representation.
   result = newStringOfCap(((len + 2) div 3) * 4)
-  result.add '_'
+  result.add "__"
   var i = 0
   while i < len - 2:
     let a = ord(s[i])
@@ -163,8 +167,9 @@ proc stopCompile*(g: ModuleGraph): bool {.inline.} =
   result = g.doStopCompile != nil and g.doStopCompile()
 
 proc createMagic*(g: ModuleGraph; name: string, m: TMagic): PSym =
-  result = newSym(skProc, getIdent(g.cache, name), nil, unknownLineInfo(), {})
+  result = newSym(skProc, getIdent(g.cache, name), nil, unknownLineInfo, {})
   result.magic = m
+  result.flags = {sfNeverRaises}
 
 proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result = ModuleGraph()
@@ -215,7 +220,7 @@ proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) =
   addModuleDep(g.incr, g.config, m.info.fileIndex, dep, isIncludeFile = false)
   if g.suggestMode:
     g.deps.incl m.position.dependsOn(dep.int)
-    # we compute the transitive closure later when quering the graph lazily.
+    # we compute the transitive closure later when querying the graph lazily.
     # this improves efficiency quite a lot:
     #invalidTransitiveClosure = true
 

compiler/modulepaths.nim

@@ -8,7 +8,7 @@
 #
 
 import ast, renderer, strutils, msgs, options, idents, os, lineinfos,
-  pathutils, nimblecmd
+  pathutils
 
 when false:
   const
@@ -139,7 +139,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
       result.add modname
   of nkPrefix:
     when false:
-      if n.sons[0].kind == nkIdent and n.sons[0].ident.s == "$":
+      if n[0].kind == nkIdent and n[0].ident.s == "$":
         result = lookupPackage(n[1], nil)
       else:
         discard
@@ -149,7 +149,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
     localError(conf, n.info, warnDeprecated, "using '.' instead of '/' in import paths is deprecated")
     result = renderTree(n, {renderNoComments}).replace(".", "/")
   of nkImportAs:
-    result = getModuleName(conf, n.sons[0])
+    result = getModuleName(conf, n[0])
   else:
     localError(conf, n.info, "invalid module name: '$1'" % n.renderTree)
     result = ""
@@ -162,6 +162,6 @@ proc checkModuleName*(conf: ConfigRef; n: PNode; doLocalError=true): FileIndex =
     if doLocalError:
       let m = if modulename.len > 0: modulename else: $n
       localError(conf, n.info, "cannot open file: " & m)
-    result = InvalidFileIDX
+    result = InvalidFileIdx
   else:
     result = fileInfoIdx(conf, fullPath)

compiler/modules.nim

@@ -10,8 +10,8 @@
 ## Implements the module handling, including the caching of modules.
 
 import
-  ast, astalgo, magicsys, std / sha1, msgs, cgendata, sigmatch, options,
-  idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod,
+  ast, astalgo, magicsys, msgs, options,
+  idents, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod,
   lineinfos, pathutils, tables
 
 proc resetSystemArtifacts*(g: ModuleGraph) =
@@ -39,7 +39,8 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil
         # but starting with version 0.20 we now produce a fake Nimble package instead
         # to resolve the conflicts:
         let pck3 = fakePackageName(graph.config, filename)
-        packSym = newSym(skPackage, getIdent(graph.cache, pck3), nil, result.info)
+        # this makes the new `packSym`'s owner be the original `packSym`
+        packSym = newSym(skPackage, getIdent(graph.cache, pck3), packSym, result.info)
         initStrTable(packSym.tab)
         graph.packageSyms.strTableAdd(packSym)
 
@@ -50,22 +51,19 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil
     setLen(graph.modules, int(fileIdx) + 1)
   graph.modules[result.position] = result
 
-  incl(result.flags, sfUsed)
   initStrTable(result.tab)
   strTableAdd(result.tab, result) # a module knows itself
   strTableAdd(packSym.tab, result)
 
 proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID.
-  new(result)
-  result.id = -1             # for better error checking
-  result.kind = skModule
-  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-  result.name = getIdent(graph.cache, splitFile(filename).name)
+  result = PSym(kind: skModule, id: -1, # for better error checking
+                name: getIdent(graph.cache, splitFile(filename).name),
+                info: newLineInfo(fileIdx, 1, 1))
   if not isNimIdentifier(result.name.s):
     rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
-  result.info = newLineInfo(fileIdx, 1, 1)
   partialInitModule(result, graph, fileIdx, filename)
 
 proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym =
@@ -77,8 +75,6 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
     if result == nil:
       result = newModule(graph, fileIdx)
       result.flags = result.flags + flags
-      if sfMainModule in result.flags:
-        graph.config.mainPackageId = result.owner.id
       result.id = id
       registerModule(graph, result)
     else:
@@ -96,7 +92,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
       if sfMainModule in flags and graph.config.projectIsStdin: stdin.llStreamOpen else: nil)
     graph.markClientsDirty(fileIdx)
 
-proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym {.procvar.} =
+proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
   # this is called by the semantic checking phase
   assert graph.config != nil
   result = compileModule(graph, fileIdx, {})
@@ -111,7 +107,7 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym {.proc
     if s.owner.id == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
     else: graph.config.foreignPackageNotes
 
-proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode {.procvar.} =
+proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode =
   result = syntaxes.parseFile(fileIdx, graph.cache, graph.config)
   graph.addDep(s, fileIdx)
   graph.addIncludeDep(s.position.FileIndex, fileIdx)
@@ -133,12 +129,12 @@ proc wantMainModule*(conf: ConfigRef) =
         "command expects a filename")
   conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt))
 
-proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIDX) =
+proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
   connectCallbacks(graph)
   let conf = graph.config
   wantMainModule(conf)
   let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
-  let projectFile = if projectFileIdx == InvalidFileIDX: conf.projectMainIdx else: projectFileIdx
+  let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
   graph.importStack.add projectFile
   if projectFile == systemFileIdx:
     discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})

compiler/msgs.nim

@@ -8,7 +8,7 @@
 #
 
 import
-  options, strutils, os, tables, ropes, platform, terminal, macros,
+  options, strutils, os, tables, ropes, terminal, macros,
   lineinfos, pathutils
 
 proc toCChar*(c: char; result: var string) =
@@ -26,13 +26,13 @@ proc makeCString*(s: string): Rope =
   const MaxLineLength = 64
   result = nil
   var res = newStringOfCap(int(s.len.toFloat * 1.1) + 1)
-  add(res, "\"")
-  for i in 0 ..< len(s):
+  res.add("\"")
+  for i in 0..<s.len:
     if (i + 1) mod MaxLineLength == 0:
-      add(res, "\"\L\"")
+      res.add("\"\L\"")
     toCChar(s[i], res)
-  add(res, '\"')
-  add(result, rope(res))
+  res.add('\"')
+  result.add(rope(res))
 
 
 proc newFileInfo(fullPath: AbsoluteFile, projPath: RelativeFile): TFileInfo =
@@ -40,9 +40,8 @@ proc newFileInfo(fullPath: AbsoluteFile, projPath: RelativeFile): TFileInfo =
   #shallow(result.fullPath)
   result.projPath = projPath
   #shallow(result.projPath)
-  let fileName = fullPath.extractFilename
-  result.shortName = fileName.changeFileExt("")
-  result.quotedName = fileName.makeCString
+  result.shortName = fullPath.extractFilename
+  result.quotedName = result.shortName.makeCString
   result.quotedFullName = fullPath.string.makeCString
   result.lines = @[]
   when defined(nimpretty):
@@ -82,6 +81,7 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile; isKnownFile: var bool
     pseudoPath = true
 
   if conf.m.filenameToIndexTbl.hasKey(canon.string):
+    isKnownFile = true
     result = conf.m.filenameToIndexTbl[canon.string]
   else:
     isKnownFile = false
@@ -96,14 +96,20 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile): FileIndex =
 
 proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo =
   result.fileIndex = fileInfoIdx
-  result.line = uint16(line)
-  result.col = int16(col)
+  if line < int high(uint16):
+    result.line = uint16(line)
+  else:
+    result.line = high(uint16)
+  if col < int high(int16):
+    result.col = int16(col)
+  else:
+    result.col = -1
 
 proc newLineInfo*(conf: ConfigRef; filename: AbsoluteFile, line, col: int): TLineInfo {.inline.} =
   result = newLineInfo(fileInfoIdx(conf, filename), line, col)
 
 
-proc concat(strings: openarray[string]): string =
+proc concat(strings: openArray[string]): string =
   var totalLen = 0
   for s in strings: totalLen += s.len
   result = newStringOfCap totalLen
@@ -147,26 +153,32 @@ proc pushInfoContext*(conf: ConfigRef; info: TLineInfo; detail: string = "") =
   conf.m.msgContext.add((info, detail))
 
 proc popInfoContext*(conf: ConfigRef) =
-  setLen(conf.m.msgContext, len(conf.m.msgContext) - 1)
+  setLen(conf.m.msgContext, conf.m.msgContext.len - 1)
 
 proc getInfoContext*(conf: ConfigRef; index: int): TLineInfo =
-  let L = conf.m.msgContext.len
-  let i = if index < 0: L + index else: index
-  if i >=% L: result = unknownLineInfo()
+  let i = if index < 0: conf.m.msgContext.len + index else: index
+  if i >=% conf.m.msgContext.len: result = unknownLineInfo
   else: result = conf.m.msgContext[i].info
 
+const
+  commandLineDesc = "command line"
+
 template toFilename*(conf: ConfigRef; fileIdx: FileIndex): string =
   if fileIdx.int32 < 0 or conf == nil:
-    "???"
+    (if fileIdx == commandLineIdx: commandLineDesc else: "???")
   else:
-    if optListFullPaths in conf.globalOptions:
-      conf.m.fileInfos[fileIdx.int32].fullPath.string
-    else:
-      conf.m.fileInfos[fileIdx.int32].projPath.string
+    conf.m.fileInfos[fileIdx.int32].shortName
+
+proc toProjPath*(conf: ConfigRef; fileIdx: FileIndex): string =
+  if fileIdx.int32 < 0 or conf == nil:
+    (if fileIdx == commandLineIdx: commandLineDesc else: "???")
+  else: conf.m.fileInfos[fileIdx.int32].projPath.string
 
 proc toFullPath*(conf: ConfigRef; fileIdx: FileIndex): string =
-  if fileIdx.int32 < 0 or conf == nil: result = "???"
-  else: result = conf.m.fileInfos[fileIdx.int32].fullPath.string
+  if fileIdx.int32 < 0 or conf == nil:
+    result = (if fileIdx == commandLineIdx: commandLineDesc else: "???")
+  else:
+    result = conf.m.fileInfos[fileIdx.int32].fullPath.string
 
 proc setDirtyFile*(conf: ConfigRef; fileIdx: FileIndex; filename: AbsoluteFile) =
   assert fileIdx.int32 >= 0
@@ -183,7 +195,7 @@ proc getHash*(conf: ConfigRef; fileIdx: FileIndex): string =
 
 proc toFullPathConsiderDirty*(conf: ConfigRef; fileIdx: FileIndex): AbsoluteFile =
   if fileIdx.int32 < 0:
-    result = AbsoluteFile"???"
+    result = AbsoluteFile(if fileIdx == commandLineIdx: commandLineDesc else: "???")
   elif not conf.m.fileInfos[fileIdx.int32].dirtyFile.isEmpty:
     result = conf.m.fileInfos[fileIdx.int32].dirtyFile
   else:
@@ -192,22 +204,28 @@ proc toFullPathConsiderDirty*(conf: ConfigRef; fileIdx: FileIndex): AbsoluteFile
 template toFilename*(conf: ConfigRef; info: TLineInfo): string =
   toFilename(conf, info.fileIndex)
 
+template toProjPath*(conf: ConfigRef; info: TLineInfo): string =
+  toProjPath(conf, info.fileIndex)
+
 template toFullPath*(conf: ConfigRef; info: TLineInfo): string =
   toFullPath(conf, info.fileIndex)
 
 template toFullPathConsiderDirty*(conf: ConfigRef; info: TLineInfo): string =
   string toFullPathConsiderDirty(conf, info.fileIndex)
 
-proc toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
-  if info.fileIndex.int32 < 0:
-    result = "???"
-    return
-  let absPath = conf.m.fileInfos[info.fileIndex.int32].fullPath.string
-  if optListFullPaths in conf.globalOptions:
-    result = absPath
-  else:
-    let relPath = conf.m.fileInfos[info.fileIndex.int32].projPath.string
-    result = if relPath.count("..") > 2: absPath else: relPath
+proc toMsgFilename*(conf: ConfigRef; info: FileIndex): string =
+  let
+    absPath = toFullPath(conf, info)
+    relPath = toProjPath(conf, info)
+  result = if (optListFullPaths in conf.globalOptions) or
+              (relPath.len > absPath.len) or
+              (relPath.count("..") > 2):
+             absPath
+           else:
+             relPath
+
+template toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
+  toMsgFilename(conf, info.fileIndex)
 
 proc toLinenumber*(info: TLineInfo): int {.inline.} =
   result = int info.line
@@ -215,12 +233,9 @@ proc toLinenumber*(info: TLineInfo): int {.inline.} =
 proc toColumn*(info: TLineInfo): int {.inline.} =
   result = info.col
 
-proc toFileLine*(conf: ConfigRef; info: TLineInfo): string {.inline.} =
-  result = toFilename(conf, info) & ":" & $info.line
-
 proc toFileLineCol*(conf: ConfigRef; info: TLineInfo): string {.inline.} =
   # consider calling `helpers.lineInfoToString` instead
-  result = toFilename(conf, info) & "(" & $info.line & ", " &
+  result = toMsgFilename(conf, info) & "(" & $info.line & ", " &
     $(info.col + ColOffset) & ")"
 
 proc `$`*(conf: ConfigRef; info: TLineInfo): string = toFileLineCol(conf, info)
@@ -318,14 +333,14 @@ proc getMessageStr(msg: TMsgKind, arg: string): string =
 type
   TErrorHandling = enum doNothing, doAbort, doRaise
 
-proc log*(s: string) {.procvar.} =
+proc log*(s: string) =
   var f: File
   if open(f, getHomeDir() / "nimsuggest.log", fmAppend):
     f.writeLine(s)
     close(f)
 
 proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} =
-  if defined(debug) or msg == errInternal or hintStackTrace in conf.notes:
+  if defined(debug) or msg == errInternal or conf.hasHint(hintStackTrace):
     {.gcsafe.}:
       if stackTraceAvailable() and isNil(conf.writelnHook):
         writeStackTrace()
@@ -359,12 +374,12 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
   const instantiationFrom = "template/generic instantiation from here"
   const instantiationOfFrom = "template/generic instantiation of `$1` from here"
   var info = lastinfo
-  for i in 0 ..< len(conf.m.msgContext):
+  for i in 0..<conf.m.msgContext.len:
     let context = conf.m.msgContext[i]
     if context.info != lastinfo and context.info != info:
       if conf.structuredErrorHook != nil:
         conf.structuredErrorHook(conf, context.info, instantiationFrom,
-                                 Severity.Error)
+                                 Severity.Hint)
       else:
         let message = if context.detail == "":
           instantiationFrom
@@ -390,22 +405,20 @@ proc rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
   case msg
   of errMin..errMax:
     sev = Severity.Error
-    writeContext(conf, unknownLineInfo())
+    writeContext(conf, unknownLineInfo)
     title = ErrorTitle
     color = ErrorColor
   of warnMin..warnMax:
     sev = Severity.Warning
-    if optWarns notin conf.options: return
-    if msg notin conf.notes: return
-    writeContext(conf, unknownLineInfo())
+    if not conf.hasWarn(msg): return
+    writeContext(conf, unknownLineInfo)
     title = WarningTitle
     color = WarningColor
     kind = WarningsToStr[ord(msg) - ord(warnMin)]
     inc(conf.warnCounter)
   of hintMin..hintMax:
     sev = Severity.Hint
-    if optHints notin conf.options: return
-    if msg notin conf.notes: return
+    if not conf.hasHint(msg): return
     title = HintTitle
     color = HintColor
     if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
@@ -413,7 +426,7 @@ proc rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
   let s = msgKindToString(msg) % args
 
   if conf.structuredErrorHook != nil:
-    conf.structuredErrorHook(conf, unknownLineInfo(),
+    conf.structuredErrorHook(conf, unknownLineInfo,
       s & (if kind.len > 0: KindFormat % kind else: ""), sev)
 
   if not ignoreMsgBecauseOfIdeTools(conf, msg):
@@ -452,7 +465,8 @@ proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
 proc writeSurroundingSrc(conf: ConfigRef; info: TLineInfo) =
   const indent = "  "
   msgWriteln(conf, indent & $sourceLine(conf, info))
-  msgWriteln(conf, indent & spaces(info.col) & '^')
+  if info.col >= 0:
+    msgWriteln(conf, indent & spaces(info.col) & '^')
 
 proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): string =
   let title = case msg
@@ -484,7 +498,7 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
     conf.m.lastError = info
   of warnMin..warnMax:
     sev = Severity.Warning
-    ignoreMsg = optWarns notin conf.options or msg notin conf.notes
+    ignoreMsg = not conf.hasWarn(msg)
     if not ignoreMsg: writeContext(conf, info)
     title = WarningTitle
     color = WarningColor
@@ -492,7 +506,7 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
     inc(conf.warnCounter)
   of hintMin..hintMax:
     sev = Severity.Hint
-    ignoreMsg = optHints notin conf.options or msg notin conf.notes
+    ignoreMsg = not conf.hasHint(msg)
     title = HintTitle
     color = HintColor
     if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
@@ -510,7 +524,7 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
                          KindColor, `%`(KindFormat, kind))
       else:
         styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s)
-      if hintSource in conf.notes:
+      if conf.hasHint(hintSource):
         conf.writeSurroundingSrc(info)
   handleError(conf, msg, eh, s)
 
@@ -532,7 +546,7 @@ proc localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
 proc localError*(conf: ConfigRef; info: TLineInfo, arg: string) =
   liMessage(conf, info, errGenerated, arg, doNothing)
 
-proc localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openarray[string]) =
+proc localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openArray[string]) =
   localError(conf, info, format % params)
 
 proc message*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
@@ -545,7 +559,7 @@ proc internalError*(conf: ConfigRef; info: TLineInfo, errMsg: string) =
 
 proc internalError*(conf: ConfigRef; errMsg: string) =
   if conf.cmd == cmdIdeTools and conf.structuredErrorHook.isNil: return
-  writeContext(conf, unknownLineInfo())
+  writeContext(conf, unknownLineInfo)
   rawMessage(conf, errInternal, errMsg)
 
 template assertNotNil*(conf: ConfigRef; e): untyped =
@@ -556,8 +570,9 @@ template internalAssert*(conf: ConfigRef, e: bool) =
   if not e: internalError(conf, $instantiationInfo())
 
 proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
-  assert i.fileIndex.int32 >= 0
-  if optExcessiveStackTrace in conf.globalOptions:
+  if i.fileIndex.int32 < 0:
+    result = makeCString "???"
+  elif optExcessiveStackTrace in conf.globalOptions:
     result = conf.m.fileInfos[i.fileIndex.int32].quotedFullName
   else:
     result = conf.m.fileInfos[i.fileIndex.int32].quotedName
@@ -577,3 +592,10 @@ proc listHints*(conf: ConfigRef) =
       if hint in conf.notes: "x" else: " ",
       lineinfos.HintsToStr[ord(hint) - ord(hintMin)]
     ])
+
+proc lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string) =
+  let m = "'$2' should be: '$1'" % [beau, got]
+  if optStyleError in conf.globalOptions:
+    localError(conf, info, m)
+  else:
+    message(conf, info, hintName, m)

compiler/ndi.nim

@@ -17,6 +17,8 @@ type
     enabled: bool
     f: File
     buf: string
+    filename: AbsoluteFile
+    syms: seq[PSym]
 
 proc doWrite(f: var NdiFile; s: PSym; conf: ConfigRef) =
   f.buf.setLen 0
@@ -28,13 +30,20 @@ proc doWrite(f: var NdiFile; s: PSym; conf: ConfigRef) =
   f.f.writeLine("\t", toFullPath(conf, s.info), "\t", f.buf)
 
 template writeMangledName*(f: NdiFile; s: PSym; conf: ConfigRef) =
-  if f.enabled: doWrite(f, s, conf)
+  if f.enabled: f.syms.add s
 
 proc open*(f: var NdiFile; filename: AbsoluteFile; conf: ConfigRef) =
   f.enabled = not filename.isEmpty
   if f.enabled:
-    f.f = open(filename.string, fmWrite, 8000)
+    f.filename = filename
     f.buf = newStringOfCap(20)
 
-proc close*(f: var NdiFile) =
-  if f.enabled: close(f.f)
+proc close*(f: var NdiFile, conf: ConfigRef) =
+  if f.enabled:
+    f.f = open(f.filename.string, fmWrite, 8000)
+    doAssert f.f != nil, f.filename.string
+    for s in f.syms:
+      doWrite(f, s, conf)
+    close(f.f)
+    f.syms.reset
+    f.filename.reset

compiler/nim.cfg

@@ -7,16 +7,18 @@ define:nimcore
 #define:nimIncremental
 #import:"$projectpath/testability"
 
+#define:staticSqlite
+
 @if windows:
   cincludes: "$lib/wrappers/libffi/common"
 @end
 
 define:useStdoutAsStdmsg
-define:nimOldCaseObjects
+
+@if nimHasStyleChecks:
+  styleCheck:error
+@end
+
 
 #define:useNodeIds
 #gc:markAndSweep
-
-@if nimHasWarningCaseTransition:
-warning[CaseTransition]:off
-@end

compiler/nim.nim

@@ -19,9 +19,9 @@ when defined(i386) and defined(windows) and defined(vcc):
   {.link: "../icons/nim-i386-windows-vcc.res".}
 
 import
-  commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes,
-  extccomp, strutils, os, osproc, platform, main, parseopt,
-  scriptconfig, idents, modulegraphs, lineinfos, cmdlinehelper,
+  commands, options, msgs,
+  extccomp, strutils, os, main, parseopt,
+  idents, lineinfos, cmdlinehelper,
   pathutils
 
 include nodejs
@@ -40,20 +40,40 @@ proc prependCurDir(f: AbsoluteFile): AbsoluteFile =
   else:
     result = f
 
+proc addCmdPrefix*(result: var string, kind: CmdLineKind) =
+  # consider moving this to std/parseopt
+  case kind
+  of cmdLongOption: result.add "--"
+  of cmdShortOption: result.add "-"
+  of cmdArgument, cmdEnd: discard
+
 proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
   var p = parseopt.initOptParser(cmd)
   var argsCount = 0
+
+  config.commandLine.setLen 0
+    # bugfix: otherwise, config.commandLine ends up duplicated
+
   while true:
     parseopt.next(p)
     case p.kind
     of cmdEnd: break
-    of cmdLongoption, cmdShortOption:
+    of cmdLongOption, cmdShortOption:
+      config.commandLine.add " "
+      config.commandLine.addCmdPrefix p.kind
+      config.commandLine.add p.key.quoteShell # quoteShell to be future proof
+      if p.val.len > 0:
+        config.commandLine.add ':'
+        config.commandLine.add p.val.quoteShell
+
       if p.key == " ":
         p.key = "-"
         if processArgument(pass, p, argsCount, config): break
       else:
         processSwitch(pass, p, config)
     of cmdArgument:
+      config.commandLine.add " "
+      config.commandLine.add p.key.quoteShell
       if processArgument(pass, p, argsCount, config): break
   if pass == passCmd2:
     if {optRun, optWasNimscript} * config.globalOptions == {} and
@@ -73,12 +93,12 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
 
   self.processCmdLineAndProjectPath(conf)
   if not self.loadConfigsAndRunMainCommand(cache, conf): return
-  if optHints in conf.options and hintGCStats in conf.notes: echo(GC_getStatistics())
+  if conf.hasHint(hintGCStats): echo(GC_getStatistics())
   #echo(GC_getStatistics())
   if conf.errorCounter != 0: return
   when hasTinyCBackend:
     if conf.cmd == cmdRun:
-      tccgen.run(conf.arguments)
+      tccgen.run(conf, conf.arguments)
   if optRun in conf.globalOptions:
     var ex = quoteShell conf.absOutFile
     if conf.cmd == cmdCompileToJS:

compiler/nimblecmd.nim

@@ -48,7 +48,7 @@ proc `<`*(ver: Version, ver2: Version): bool =
   # Handling for normal versions such as "0.1.0" or "1.0".
   var sVer = string(ver).split('.')
   var sVer2 = string(ver2).split('.')
-  for i in 0..max(sVer.len, sVer2.len)-1:
+  for i in 0..<max(sVer.len, sVer2.len):
     var sVerI = 0
     if i < sVer.len:
       discard parseInt(sVer[i], sVerI)
@@ -69,9 +69,10 @@ proc getPathVersion*(p: string): tuple[name, version: string] =
   result.version = ""
 
   const specialSeparator = "-#"
-  var sepIdx = p.find(specialSeparator)
+  let last = p.rfind(p.lastPathPart) # the index where the last path part begins
+  var sepIdx = p.find(specialSeparator, start = last)
   if sepIdx == -1:
-    sepIdx = p.rfind('-')
+    sepIdx = p.rfind('-', start = last)
 
   if sepIdx == -1:
     result.name = p
@@ -82,7 +83,7 @@ proc getPathVersion*(p: string): tuple[name, version: string] =
       result.name = p
       return
 
-  result.name = p[0 .. sepIdx - 1]
+  result.name = p[0..sepIdx - 1]
   result.version = p.substr(sepIdx + 1)
 
 proc addPackage(conf: ConfigRef; packages: StringTableRef, p: string; info: TLineInfo) =
@@ -129,6 +130,10 @@ proc addPathRec(conf: ConfigRef; dir: string, info: TLineInfo) =
 proc nimblePath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
   addPathRec(conf, path.string, info)
   addNimblePath(conf, path.string, info)
+  let i = conf.nimblePaths.find(path)
+  if i != -1:
+    conf.nimblePaths.delete(i)
+  conf.nimblePaths.insert(path, 0)
 
 when isMainModule:
   proc v(s: string): Version = s.newVersion
@@ -142,16 +147,16 @@ when isMainModule:
 
   let conf = newConfigRef()
   var rr = newStringTable()
-  addPackage conf, rr, "irc-#a111", unknownLineInfo()
-  addPackage conf, rr, "irc-#head", unknownLineInfo()
-  addPackage conf, rr, "irc-0.1.0", unknownLineInfo()
-  #addPackage conf, rr, "irc", unknownLineInfo()
-  #addPackage conf, rr, "another", unknownLineInfo()
-  addPackage conf, rr, "another-0.1", unknownLineInfo()
+  addPackage conf, rr, "irc-#a111", unknownLineInfo
+  addPackage conf, rr, "irc-#head", unknownLineInfo
+  addPackage conf, rr, "irc-0.1.0", unknownLineInfo
+  #addPackage conf, rr, "irc", unknownLineInfo
+  #addPackage conf, rr, "another", unknownLineInfo
+  addPackage conf, rr, "another-0.1", unknownLineInfo
 
-  addPackage conf, rr, "ab-0.1.3", unknownLineInfo()
-  addPackage conf, rr, "ab-0.1", unknownLineInfo()
-  addPackage conf, rr, "justone-1.0", unknownLineInfo()
+  addPackage conf, rr, "ab-0.1.3", unknownLineInfo
+  addPackage conf, rr, "ab-0.1", unknownLineInfo
+  addPackage conf, rr, "justone-1.0", unknownLineInfo
 
   doAssert toSeq(rr.chosen) ==
     @["irc-#head", "another-0.1", "ab-0.1.3", "justone-1.0"]

compiler/nimconf.nim

@@ -10,8 +10,8 @@
 # This module handles the reading of the config file.
 
 import
-  llstream, nversion, commands, os, strutils, msgs, platform, condsyms, lexer,
-  options, idents, wordrecg, strtabs, lineinfos, pathutils
+  llstream, commands, os, strutils, msgs, lexer,
+  options, idents, wordrecg, strtabs, lineinfos, pathutils, scriptconfig
 
 # ---------------- configuration file parser -----------------------------
 # we use Nim's scanner here to save space and work
@@ -114,7 +114,7 @@ proc parseDirective(L: var TLexer, tok: var TToken; config: ConfigRef; condStack
   ppGetTok(L, tok)            # skip @
   case whichKeyword(tok.ident)
   of wIf:
-    setLen(condStack, len(condStack) + 1)
+    setLen(condStack, condStack.len + 1)
     let res = evalppIf(L, tok, config)
     condStack[high(condStack)] = res
     if not res: jumpToDirective(L, tok, jdElseEndif, config, condStack)
@@ -123,31 +123,31 @@ proc parseDirective(L: var TLexer, tok: var TToken; config: ConfigRef; condStack
   of wEnd: doEnd(L, tok, condStack)
   of wWrite:
     ppGetTok(L, tok)
-    msgs.msgWriteln(config, strtabs.`%`(tokToStr(tok), config.configVars,
+    msgs.msgWriteln(config, strtabs.`%`($tok, config.configVars,
                                 {useEnvironment, useKey}))
     ppGetTok(L, tok)
   else:
     case tok.ident.s.normalize
     of "putenv":
       ppGetTok(L, tok)
-      var key = tokToStr(tok)
+      var key = $tok
       ppGetTok(L, tok)
-      os.putEnv(key, tokToStr(tok))
+      os.putEnv(key, $tok)
       ppGetTok(L, tok)
     of "prependenv":
       ppGetTok(L, tok)
-      var key = tokToStr(tok)
+      var key = $tok
       ppGetTok(L, tok)
-      os.putEnv(key, tokToStr(tok) & os.getEnv(key))
+      os.putEnv(key, $tok & os.getEnv(key))
       ppGetTok(L, tok)
     of "appendenv":
       ppGetTok(L, tok)
-      var key = tokToStr(tok)
+      var key = $tok
       ppGetTok(L, tok)
-      os.putEnv(key, os.getEnv(key) & tokToStr(tok))
+      os.putEnv(key, os.getEnv(key) & $tok)
       ppGetTok(L, tok)
     else:
-      lexMessage(L, errGenerated, "invalid directive: '$1'" % tokToStr(tok))
+      lexMessage(L, errGenerated, "invalid directive: '$1'" % $tok)
 
 proc confTok(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var seq[bool]) =
   ppGetTok(L, tok)
@@ -156,44 +156,51 @@ proc confTok(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var s
 
 proc checkSymbol(L: TLexer, tok: TToken) =
   if tok.tokType notin {tkSymbol..tkInt64Lit, tkStrLit..tkTripleStrLit}:
-    lexMessage(L, errGenerated, "expected identifier, but got: " & tokToStr(tok))
+    lexMessage(L, errGenerated, "expected identifier, but got: " & $tok)
 
 proc parseAssignment(L: var TLexer, tok: var TToken;
                      config: ConfigRef; condStack: var seq[bool]) =
-  if tok.ident.s == "-" or tok.ident.s == "--":
-    confTok(L, tok, config, condStack)           # skip unnecessary prefix
+  if tok.ident != nil:
+    if tok.ident.s == "-" or tok.ident.s == "--":
+      confTok(L, tok, config, condStack)           # skip unnecessary prefix
   var info = getLineInfo(L, tok) # save for later in case of an error
   checkSymbol(L, tok)
-  var s = tokToStr(tok)
+  var s = $tok
   confTok(L, tok, config, condStack)             # skip symbol
   var val = ""
   while tok.tokType == tkDot:
-    add(s, '.')
+    s.add('.')
     confTok(L, tok, config, condStack)
     checkSymbol(L, tok)
-    add(s, tokToStr(tok))
+    s.add($tok)
     confTok(L, tok, config, condStack)
   if tok.tokType == tkBracketLe:
     # BUGFIX: val, not s!
     confTok(L, tok, config, condStack)
     checkSymbol(L, tok)
-    add(val, '[')
-    add(val, tokToStr(tok))
+    val.add('[')
+    val.add($tok)
     confTok(L, tok, config, condStack)
     if tok.tokType == tkBracketRi: confTok(L, tok, config, condStack)
     else: lexMessage(L, errGenerated, "expected closing ']'")
-    add(val, ']')
+    val.add(']')
   let percent = tok.ident != nil and tok.ident.s == "%="
   if tok.tokType in {tkColon, tkEquals} or percent:
-    if len(val) > 0: add(val, ':')
+    if val.len > 0: val.add(':')
     confTok(L, tok, config, condStack)           # skip ':' or '=' or '%'
     checkSymbol(L, tok)
-    add(val, tokToStr(tok))
+    val.add($tok)
     confTok(L, tok, config, condStack)           # skip symbol
+    if tok.tokType in {tkColon, tkEquals}:
+      val.add($tok) # add the :
+      confTok(L, tok, config, condStack)           # skip symbol
+      checkSymbol(L, tok)
+      val.add($tok) # add the token after it
+      confTok(L, tok, config, condStack)           # skip symbol
     while tok.ident != nil and tok.ident.s == "&":
       confTok(L, tok, config, condStack)
       checkSymbol(L, tok)
-      add(val, tokToStr(tok))
+      val.add($tok)
       confTok(L, tok, config, condStack)
   if percent:
     processSwitch(s, strtabs.`%`(val, config.configVars,
@@ -201,7 +208,7 @@ proc parseAssignment(L: var TLexer, tok: var TToken;
   else:
     processSwitch(s, val, passPP, info, config)
 
-proc readConfigFile(filename: AbsoluteFile; cache: IdentCache;
+proc readConfigFile*(filename: AbsoluteFile; cache: IdentCache;
                     config: ConfigRef): bool =
   var
     L: TLexer
@@ -215,7 +222,7 @@ proc readConfigFile(filename: AbsoluteFile; cache: IdentCache;
     var condStack: seq[bool] = @[]
     confTok(L, tok, config, condStack)           # read in the first token
     while tok.tokType != tkEof: parseAssignment(L, tok, config, condStack)
-    if len(condStack) > 0: lexMessage(L, errGenerated, "expected @end")
+    if condStack.len > 0: lexMessage(L, errGenerated, "expected @end")
     closeLexer(L)
     return true
 
@@ -239,19 +246,34 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef) =
   template readConfigFile(path) =
     let configPath = path
     if readConfigFile(configPath, cache, conf):
-      add(configFiles, configPath)
+      configFiles.add(configPath)
+
+  template runNimScriptIfExists(path: AbsoluteFile) =
+    let p = path # eval once
+    if fileExists(p):
+      configFiles.add(p)
+      runNimScript(cache, p, freshDefines = false, conf)
 
   if optSkipSystemConfigFile notin conf.globalOptions:
     readConfigFile(getSystemConfigPath(conf, cfg))
 
+    if cfg == DefaultConfig:
+      runNimScriptIfExists(getSystemConfigPath(conf, DefaultConfigNims))
+
   if optSkipUserConfigFile notin conf.globalOptions:
     readConfigFile(getUserConfigPath(cfg))
 
+    if cfg == DefaultConfig:
+      runNimScriptIfExists(getUserConfigPath(DefaultConfigNims))
+
   let pd = if not conf.projectPath.isEmpty: conf.projectPath else: AbsoluteDir(getCurrentDir())
   if optSkipParentConfigFiles notin conf.globalOptions:
     for dir in parentDirs(pd.string, fromRoot=true, inclusive=false):
       readConfigFile(AbsoluteDir(dir) / cfg)
 
+      if cfg == DefaultConfig:
+        runNimScriptIfExists(AbsoluteDir(dir) / DefaultConfigNims)
+
   if optSkipProjConfigFile notin conf.globalOptions:
     readConfigFile(pd / cfg)
 
@@ -262,6 +284,20 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef) =
         projectConfig = changeFileExt(conf.projectFull, "nim.cfg")
       readConfigFile(projectConfig)
 
+    if cfg == DefaultConfig:
+      runNimScriptIfExists(pd / DefaultConfigNims)
+
   for filename in configFiles:
     # delayed to here so that `hintConf` is honored
     rawMessage(conf, hintConf, filename.string)
+
+  block:
+    let scriptFile = conf.projectFull.changeFileExt("nims")
+    if conf.command != "nimsuggest":
+      runNimScriptIfExists(scriptFile)
+    else:
+      if scriptFile != conf.projectFull:
+        runNimScriptIfExists(scriptFile)
+      else:
+        # 'nimsuggest foo.nims' means to just auto-complete the NimScript file
+        discard

compiler/nimeval.nim

@@ -46,7 +46,7 @@ proc selectUniqueSymbol*(i: Interpreter; name: string;
     s = nextIdentIter(it, i.mainModule.tab)
 
 proc selectRoutine*(i: Interpreter; name: string): PSym =
-  ## Selects a declared rountine (proc/func/etc) from the main module.
+  ## Selects a declared routine (proc/func/etc) from the main module.
   ## The routine needs to have the export marker ``*``. The only matching
   ## routine is returned and ``nil`` if it is overloaded.
   result = selectUniqueSymbol(i, name, {skTemplate, skMacro, skFunc,

compiler/nimfix/nimfix.nim

@@ -98,7 +98,7 @@ proc handleCmdLine(config: ConfigRef) =
       gProjectPath = getCurrentDir()
     loadConfigs(DefaultConfig, config) # load all config files
     # now process command line arguments again, because some options in the
-    # command line can overwite the config file's settings
+    # command line can overwrite the config file's settings
     extccomp.initVars()
     processCmdLine(passCmd2, "", config)
     mainCommand()

compiler/nimfix/prettybase.nim

@@ -8,9 +8,7 @@
 #
 
 import strutils except Letters
-import lexbase, streams
 import ".." / [ast, msgs, lineinfos, idents, options, linter]
-from os import splitFile
 
 proc replaceDeprecated*(conf: ConfigRef; info: TLineInfo; oldSym, newSym: PIdent) =
   let line = sourceLine(conf, info)

compiler/nimlexbase.nim

@@ -157,7 +157,7 @@ proc getCurrentLine(L: TBaseLexer, marker: bool = true): string =
   result = ""
   var i = L.lineStart
   while not (L.buf[i] in {CR, LF, EndOfFile}):
-    add(result, L.buf[i])
+    result.add(L.buf[i])
     inc(i)
   result.add("\n")
   if marker: