bump Nim version to 1.2.0

finalize() should run in insert()

.builds/freebsd.yml

@@ -0,0 +1,37 @@
+# see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd
+image: freebsd/latest
+
+# packages:
+# - databases/sqlite3
+# - devel/boehm-gc-threaded
+# - devel/pcre
+# - devel/sdl20
+# - devel/sfml
+# - www/node
+# - devel/gmake
+# - devel/git
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    # workaround https://github.com/timotheecour/Nim/issues/76
+    sudo pkg update -q -f
+    sudo pkg install -y -q databases/sqlite3 devel/boehm-gc-threaded devel/pcre \
+      devel/sdl20 devel/sfml www/node devel/gmake devel/git
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    gmake -C csources -j $(sysctl -n hw.ncpu)
+    bin/nim c --skipUserCfg --skipParentCfg koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: 'araq' # Replace with a single Patreon username
+open_collective: 'nim' # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: 'https://nim-lang.org/donate.html' # Replace with a single custom sponsorship URL

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,40 @@
+---
+name: Bug report
+about: Have you found an unexpected behavior? Use this template.
+title: Think about the title, twice
+labels: ''
+assignees: ''
+
+---
+
+Function `echo` outputs the wrong string.
+
+### Example
+```nim
+echo "Hello World!"
+```
+
+### Current Output
+```
+Hola mundo!
+```
+
+### Expected Output
+```
+Hello World!
+```
+
+### Possible Solution
+
+* In file xyz there is a call that might be the cause of it.
+
+### Additional Information
+
+* It was working in version a.b.c
+* Issue #abc is related, but different because of ...
+* This issue is blocking my project xyz
+
+```
+$ nim -v
+Nim Compiler Version 0.1.2
+```

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,35 @@
+---
+name: Feature request
+about: Do you want to suggest a new feature? Use this template.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+<!---
+Notice there is now a separate repository for the detailed RFCs and proposals:
+https://github.com/nim-lang/RFCs
+
+If you have a simple feature request, you can post it here using this template,
+but bear in mind that adding new features to the language is currently a low priority.
+-->
+
+
+### Summary
+<!--- Short summary of your proposed feature -->
+
+
+### Description
+<!--- Describe your solution, what problem does it fix? -->
+
+
+### Alternatives
+<!--- Are there any alternatives you've considered? -->
+
+
+### Additional Information
+<!--- For Example:
+* A link to a project where the issue is relevant.
+* A link to a related issue or discussion.
+-->

.github/workflows/ci.yml.disabled

@@ -0,0 +1,122 @@
+name: Continous Integration
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64, i386]
+        cpp: ['false', 'true']
+        pkg: ['false', 'true']
+        exclude:
+          - os: ubuntu-18.04
+            cpp: 'true'
+          - os: ubuntu-18.04
+            pkg: 'true'
+          - os: macos-10.15
+            cpu: i386
+          - os: macos-10.15
+            pkg: 'true'
+          - os: windows-2019
+            cpu: i386
+          - os: windows-2019
+            cpp: 'true'
+    name: '${{ matrix.os }} (${{ matrix.cpu }}, cpp: ${{ matrix.cpp }}, pkg: ${{ matrix.pkg }})'
+    runs-on: ${{ matrix.os }}
+    env:
+      NIM_COMPILE_TO_CPP: ${{ matrix.cpp }}
+      NIM_TEST_PACKAGES: ${{ matrix.pkg }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install node.js 8.x'
+        uses: actions/setup-node@v1
+        with:
+          node-version: '8.x'
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -yq \
+              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
+              valgrind libc6-dbg
+      - name: 'Install dependencies (Linux i386)'
+        if: runner.os == 'Linux' && matrix.cpu == 'i386'
+        run: |
+          sudo dpkg --add-architecture i386
+
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+              g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
+              libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 \
+              libffi-dev:i386
+
+          cat << EOF > bin/gcc
+          #!/bin/bash
+
+          exec $(which gcc) -m32 "\$@"
+          EOF
+          cat << EOF > bin/g++
+          #!/bin/bash
+
+          exec $(which g++) -m32 "\$@"
+          EOF
+
+          chmod 755 bin/gcc
+          chmod 755 bin/g++
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install boehmgc make sfml
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "::add-path::${{ github.workspace }}/bin"
+
+      - name: 'Build csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+      - name: 'Run CI'
+        shell: bash
+        run: ./koch runCI
+
+      - name: 'Show failed tests'
+        if: failure()
+        shell: bash
+        run: nim c -r tools/ci_testresults.nim

.github/workflows/ci_docs.yml

@@ -0,0 +1,133 @@
+name: Nim Docs CI
+on:
+  push:
+    # Run only on changes on these files
+    paths:
+      - 'lib/**.nim'
+      - 'doc/**.rst'
+      - 'doc/nimdoc.css'
+      - '.github/workflows/ci_docs.yml'
+
+  pull_request:
+    # Run only on changes on these files
+    paths:
+      - 'lib/**.nim'
+      - 'doc/**.rst'
+      - '.github/workflows/ci_docs.yml'
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [linux, windows, osx]
+        include:
+          - target: linux
+            os: ubuntu-18.04
+          - target: windows
+            os: windows-2019
+          - target: osx
+            os: macos-10.15
+
+    name: ${{ matrix.target }}
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+
+      - name: 'Install build dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install make
+
+      - name: 'Install build dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "::add-path::${{ github.workspace }}/bin"
+
+      - name: 'Get current csources version'
+        id: csources-version
+        shell: bash
+        run: |
+          sha=$(git ls-remote https://github.com/nim-lang/csources master | cut -f 1)
+          echo "::set-output name=sha::$sha"
+
+      - name: 'Get prebuilt csources from cache'
+        id: csources-cache
+        uses: actions/cache@v1
+        with:
+          path: bin
+          key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}'
+
+      - name: 'Checkout csources'
+        if: steps.csources-cache.outputs.cache-hit != 'true'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Build 1-stage compiler from csources'
+        shell: bash
+        run: |
+          ext=
+          [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe
+          if [[ ! -x bin/nim-csources$ext ]]; then
+            ncpu=
+            case '${{ runner.os }}' in
+            'Linux')
+              ncpu=$(nproc)
+              ;;
+            'macOS')
+              ncpu=$(sysctl -n hw.ncpu)
+              ;;
+            'Windows')
+              ncpu=$NUMBER_OF_PROCESSORS
+              ;;
+            esac
+            [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+            make -C csources -j $ncpu CC=gcc
+            cp bin/nim{,-csources}$ext
+          else
+            echo 'Cache hit, using prebuilt csources'
+            cp bin/nim{-csources,}$ext
+          fi
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build the real compiler'
+        shell: bash
+        run: ./koch boot -d:release
+
+      - name: 'Build documentation'
+        shell: bash
+        run: ./koch doc --git.commit:devel
+
+      - name: 'Prepare documentation for deployment'
+        if: |
+          github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
+          matrix.target == 'linux'
+        shell: bash
+        run: cp -f doc/html/{overview,index}.html
+
+      - name: 'Publish documentation to Github Pages'
+        if: |
+          github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
+          matrix.target == 'linux'
+        uses: crazy-max/ghaction-github-pages@v1
+        with:
+          build_dir: doc/html
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci_ssl.yml

@@ -0,0 +1,91 @@
+name: Nim SSL CI
+on:
+  pull_request:
+    # Run only on changes on related files
+    paths:
+      - 'lib/pure/httpclient.nim'
+      - 'lib/pure/net.nim'
+      - 'lib/pure/ssl_certs.nim'
+      - 'lib/wrappers/openssl.nim'
+      - 'tests/stdlib/thttpclient_ssl*'
+      - 'tests/untestable/thttpclient_ssl*'
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64]
+    name: '${{ matrix.os }} (${{ matrix.cpu }})'
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -y libssl1.1
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install make
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "::add-path::${{ github.workspace }}/bin"
+
+      - name: 'Build 1-stage compiler from csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build the real compiler'
+        shell: bash
+        run: ./koch boot
+
+      - name: 'Run SSL nimDisableCertificateValidation integration tests'
+        shell: bash
+        run: nim c -d:nimDisableCertificateValidation -d:ssl -r -p:. tests/untestable/thttpclient_ssl_disabled.nim
+
+      - name: 'Run SSL certificate check integration tests'
+        # Not supported on Windows due to old openssl version
+        if: runner.os != 'Windows'
+        shell: bash
+        run: nim c -d:ssl -p:. --threads:on -r tests/untestable/thttpclient_ssl.nim

.gitignore

@@ -11,7 +11,9 @@ dnimcache/
 !/icons/*.o
 *.obj
 *.ilk
+*.exp
 *.pdb
+*.lib
 *.dll
 *.exe
 *.so
@@ -19,6 +21,7 @@ dnimcache/
 *.zip
 *.iss
 *.log
+*.pdb
 
 mapping.txt
 tags
@@ -30,7 +33,7 @@ doc/*.html
 doc/*.pdf
 doc/*.idx
 /web/upload
-build/*
+/build/*
 bin/*
 
 # iOS specific wildcards.
@@ -40,6 +43,7 @@ bin/*
 *.perspectivev3
 *.swp
 .DS_Store
+.tags
 project.xcworkspace/
 xcuserdata/
 
@@ -48,14 +52,20 @@ xcuserdata/
 /compiler/nim.dot
 /reject.json
 /run.json
+/tools/dochack/dochack.js
+*.json
+/pkgstemp/**/*
 # for `nim doc foo.nim`
 /*.html
+lib/**/*.html
 #/testresults.html #covered by /*.html
 
 /testresults.json
 testament.db
+/tests/**/*.json
+/tests/**/*.js
 /csources
-dist/
+/dist/
 
 # Private directories and files (IDEs)
 .*/
@@ -66,3 +76,23 @@ testresults/
 test.txt
 /test.ini
 
+tweeter.db
+tweeter_test.db
+megatest.nim
+
+/outputExpected.txt
+/outputGotten.txt
+
+/lib/pure/*.js
+
+!/.builds/
+
+!/.github
+
+# ignore debug dirs generated by dsymutil on OSX
+*.dSYM
+
+nimdoc.out.css
+
+# for `nim c -r nimdoc/tester` etc; this can be in multiple places
+htmldocs

.gitlab-ci.yml

@@ -53,9 +53,8 @@ test-windows:
   <<: *win_set_path_def
   script:
     - call ci\deps.bat
-    - nim c --taintMode:on tests\testament\tester
-    - tests\testament\tester.exe --pedantic all
+    - nim c testament\tester
+    - testament\tester.exe --pedantic all
   tags:
     - windows
     - fast
-

.travis.yml

@@ -1,60 +1,46 @@
 sudo: false
 language: c
-os:
-  - linux
-  - osx
-dist: trusty
 
-#matrix:
-#  allow_failures:
-#    - os: osx
+dist: xenial
+
+matrix:
+  include:
+    - os: linux
+      env:
+        - NIM_COMPILE_TO_CPP=false
+        - CPU=amd64
 
 addons:
   apt:
+    # update the list above if more deps are introduced
     packages:
     - libcurl4-openssl-dev
     - libsdl1.2-dev
     - libgc-dev
     - libsfml-dev
-
-before_install:
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install boehmgc; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install sfml; fi
+    - libc6-dbg
+    - valgrind
 
 before_script:
-  - set -e
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then unset -f cd; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then shell_session_update() { :; }; fi
   - git clone --depth 1 https://github.com/nim-lang/csources.git
-  - cd csources
-  - sh build.sh
-  - cd ..
-  - export PATH=$(pwd)/bin${PATH:+:$PATH}
+  - export PATH="$PWD/bin${PATH:+:$PATH}"
+  - make -C csources -j 2 LD=$CC ucpu=$CPU
+
 script:
+  - echo "travis_fold:start:nim_c_koch"
   - nim c koch
+  - echo "travis_fold:end:nim_c_koch"
+  - echo "travis_fold:start:koch_boot"
   - ./koch boot
-  - ./koch boot -d:release
-  - ./koch nimble
-  - nim e tests/test_nimscript.nims
-  #- nimble install zip -y
-  #- nimble install opengl
-  #- nimble install sdl1
-  #- nimble install jester@#head -y
-  #- nimble install niminst
-  - nim c --taintMode:on -d:nimCoroutines tests/testament/tester
-  - tests/testament/tester --pedantic all -d:nimCoroutines
-  - nim c -o:bin/nimpretty nimpretty/nimpretty.nim
-  - nim c -r nimpretty/tester.nim
-  - ./koch docs
-  - ./koch csource
-  - ./koch nimsuggest
-  - nim c -r nimsuggest/tester
-  - nim c -r nimdoc/tester
+  - echo "travis_fold:end:koch_boot"
+  - echo "travis_fold:start:koch_doc"
+  - ./koch doc
+  - echo "travis_fold:end:koch_doc"
 
 before_deploy:
   # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html
   - cp -f ./doc/html/overview.html ./doc/html/index.html
+
 deploy:                         # https://nim-lang.github.io/Nim
   provider: pages
   # local-dir *has* to be a relative path from the repo root.

appveyor.yml

@@ -1,65 +1,37 @@
 version: '{build}'
 
-cache:
-- x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z
-- sqlite-dll-win64-x64-3160200.zip
-# - i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
-
-matrix:
-  fast_finish: true
-
 environment:
+  DLLS_URL: https://nim-lang.org/download/dlls.zip
+  DLLS_ARCHIVE: dlls.zip
+  MINGW_DIR: mingw64
+  MINGW_URL: https://nim-lang.org/download/mingw64.7z
+  MINGW_ARCHIVE: mingw64.7z
+
   matrix:
-    - MINGW_DIR: mingw64
-      MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Personal%20Builds/mingw-builds/4.9.2/threads-win32/seh/x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z/download
-      MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z
-      SQLITE_URL: http://www.sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
-      SQLITE_ARCHIVE: sqlite-dll-win64-x64-3160200.zip
-      platform: x64
-    # - MINGW_DIR: mingw32
-    #   MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/4.9.2/threads-win32/dwarf/i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z/download
-    #   MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
-    #   SQLITE_URL: http://www.sqlite.org/2017/sqlite-dll-win32-x86-3160200.zip
-    #   SQLITE_ARCHIVE: sqlite-dll-win32-x86-3160200.zip
-    #   platform: x86
+    - NIM_TEST_PACKAGES: false
+    - NIM_TEST_PACKAGES: true
+
+cache:
+    - '%MINGW_ARCHIVE%'
+    - '%DLLS_ARCHIVE%'
+
 
 install:
   - ps: Install-Product node 8 # node 8 or later is required to test js async stuff
-  - MKDIR %CD%\DIST
-  - MKDIR %CD%\DIST\PCRE
-  - nuget install pcre -Verbosity quiet -Version 8.33.0.1 -OutputDirectory %CD%\DIST\PCRE
-  - IF not exist "%SQLITE_ARCHIVE%" appveyor DownloadFile "%SQLITE_URL%" -FileName "%SQLITE_ARCHIVE%"
-  - 7z x -y "%SQLITE_ARCHIVE%" -o"%CD%\DIST"> nul
+  - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%"
+  - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\BIN"> nul
   - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%"
   - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul
   - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH%
-  - IF "%PLATFORM%" == "x64" ( copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay64.dll & copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl64.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl32.dll )
-    ELSE ( copy C:\OpenSSL-Win32\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win32\libssl32.dll %CD%\BIN\libssl32.dll )
-  - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\sqlite3.dll %CD%\BIN\sqlite3_64.dll ) ELSE ( copy %CD%\DIST\sqlite3.dll %CD%\BIN\sqlite3_32.dll )
-  - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\x64\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre64.dll ) ELSE ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\Win32\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre32.dll )
   - git clone --depth 1 https://github.com/nim-lang/csources
   - cd csources
-  - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat )
+  - build64.bat
   - cd ..
 
 build_script:
+  - openssl version
+  - openssl version -d
   - bin\nim c koch
-  - koch boot -d:release
-  - koch nimble
-  - nim e tests/test_nimscript.nims
-  - nim c -o:bin/nimpretty.exe nimpretty/nimpretty.nim
-  - nim c -r nimpretty/tester.nim
-#  - nimble install zip -y
-#  - nimble install opengl -y
-#  - nimble install sdl1 -y
-#  - nimble install jester@#head -y
-  - nim c --taintMode:on -d:nimCoroutines --os:genode -d:posix --compileOnly tests/testament/tester
-  - nim c --taintMode:on -d:nimCoroutines tests/testament/tester
-
-test_script:
-  - tests\testament\tester --pedantic all -d:nimCoroutines
-  - nim c -r nimdoc\tester
-#  - koch csource
-#  - koch zip
+  - koch runCI
 
 deploy: off

azure-pipelines.yml

@@ -0,0 +1,164 @@
+trigger:
+  branches:
+    include:
+    - '*'
+pr:
+  branches:
+    include:
+    - '*'
+
+jobs:
+- job: packages
+
+  timeoutInMinutes: 90 # default `60` led to lots of cancelled jobs; use 0 for unlimited (may be undesirable)
+
+  strategy:
+    matrix:
+      Linux_amd64:
+        vmImage: 'ubuntu-16.04'
+        CPU: amd64
+      Linux_i386:
+        vmImage: 'ubuntu-16.04'
+        CPU: i386
+      OSX_amd64:
+        vmImage: 'macOS-10.15'
+        CPU: amd64
+      OSX_amd64_cpp:
+        vmImage: 'macOS-10.15'
+        CPU: amd64
+        NIM_COMPILE_TO_CPP: true
+      Windows_amd64:
+        vmImage: 'windows-2019'
+        CPU: amd64
+      Windows_amd64_pkg:
+        vmImage: 'windows-2019'
+        CPU: amd64
+        NIM_TEST_PACKAGES: true
+
+  pool:
+    vmImage: $(vmImage)
+
+  workspace:
+    clean: all
+
+  steps:
+    - bash: git config --global core.autocrlf false
+      displayName: 'Disable auto conversion to CRLF by git (Windows-only)'
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+    - checkout: self
+
+    - bash: git clone --depth 1 https://github.com/nim-lang/csources.git
+      displayName: 'Checkout csources'
+
+    - task: NodeTool@0
+      inputs:
+        versionSpec: '8.x'
+      displayName: 'Install node.js 8.x'
+
+    - bash: |
+        sudo apt-fast update -qq
+        DEBIAN_FRONTEND='noninteractive' \
+          sudo apt-fast install --no-install-recommends -yq \
+            libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg
+      displayName: 'Install dependencies (amd64 Linux)'
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
+
+    - bash: |
+        sudo dpkg --add-architecture i386
+
+        # Downgrade llvm, libgcc and libstdc++:
+        # - llvm has to be downgraded to have 32bit version installed for sfml.
+        # - libgcc and libstdc++ have to be downgraded as an optimization to
+        #   prevent the use of the toolchain ppa, which has a terrible download
+        #   speed.
+        cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
+        Package: libllvm6.0 libgcc1 libstdc++6
+        Pin: origin "azure.archive.ubuntu.com"
+        Pin-Priority: 1001
+
+        Package: *
+        Pin: release o=LP-PPA-ubuntu-toolchain-r-test
+        Pin-Priority: 100
+        EOF
+
+        sudo apt-fast update -qq
+        # `:i386` (e.g. in `libffi-dev:i386`) is needed otherwise you may get:
+        # `could not load: libffi.so` during dynamic loading.
+        DEBIAN_FRONTEND='noninteractive' \
+          sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+            g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
+            libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 libffi-dev:i386
+
+        cat << EOF > bin/gcc
+        #!/bin/bash
+
+        exec $(which gcc) -m32 "\$@"
+        EOF
+        cat << EOF > bin/g++
+        #!/bin/bash
+
+        exec $(which g++) -m32 "\$@"
+        EOF
+
+        chmod 755 bin/gcc
+        chmod 755 bin/g++
+
+      displayName: 'Install dependencies (i386 Linux)'
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
+
+    - bash: brew install boehmgc make sfml
+      displayName: 'Install dependencies (OSX)'
+      condition: eq(variables['Agent.OS'], 'Darwin')
+
+    - bash: |
+        mkdir dist
+        curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+        curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+        7z x dist/mingw64.7z -odist
+        7z x dist/dlls.zip -obin
+        echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
+
+      displayName: 'Install dependencies (Windows)'
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+    - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin'
+      displayName: 'Add build binaries to PATH'
+
+    - bash: |
+        echo 'PATH:' "$PATH"
+        echo '##[section]gcc version'
+        gcc -v
+        echo '##[section]nodejs version'
+        node -v
+        echo '##[section]make version'
+        make -v
+      displayName: 'System information'
+
+    - bash: |
+        ncpu=
+        case '$(Agent.OS)' in
+        'Linux')
+          ncpu=$(nproc)
+          ;;
+        'Darwin')
+          ncpu=$(sysctl -n hw.ncpu)
+          ;;
+        'Windows_NT')
+          ncpu=$NUMBER_OF_PROCESSORS
+          ;;
+        esac
+        [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+        make -C csources -j $ncpu CC=gcc ucpu=$(CPU)
+      displayName: 'Build csources'
+
+    - bash: nim c koch
+      displayName: 'Build koch'
+
+      # set result to omit the "bash exited with error code '1'" message
+    - bash: |
+        ./koch runCI || echo '##vso[task.complete result=Failed]'
+      displayName: 'Run CI'
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)

bin/nim-gdb

@@ -3,8 +3,19 @@
 # Exit if anything fails
 set -e
 
+which nim > /dev/null || (echo "nim not in PATH"; exit 1)
+which gdb > /dev/null || (echo "gdb not in PATH"; exit 1)
+
+if [[ "$(uname -s)" == "Darwin" ]]; then
+  which greadlink > /dev/null || (echo "readlink not in PATH. Please install coreutils from homebrew."; exit 1)
+  READLINK=greadlink
+else
+  which readlink > /dev/null || (echo "readlink not in PATH."; exit 1)
+  READLINK=readlink
+fi
+
 # Find out where the pretty printer Python module is
-NIM_SYSROOT=$(dirname $(dirname $(readlink -e $(which nim))))
+NIM_SYSROOT=$(dirname $(dirname $($READLINK -e $(which nim))))
 GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"
 
 # Run GDB with the additional arguments that load the pretty printers
@@ -13,6 +24,4 @@ GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"
 NIM_GDB="${NIM_GDB:-gdb}"
 # exec replaces the new process of bash with gdb. It is always good to
 # have fewer processes.
-exec ${NIM_GDB} \
-  -eval-command "source $GDB_PYTHON_MODULE_PATH" \
-  "$@"
+exec "${NIM_GDB}" -eval-command="source $GDB_PYTHON_MODULE_PATH" "$@"

bin/nim-gdb.bash

@@ -0,0 +1 @@
+nim-gdb

bin/nim-gdb.bat

@@ -0,0 +1,14 @@
+@echo off
+for %%i in (nim.exe) do (set NIM_BIN=%%~dp$PATH:i)
+
+for %%i in ("%NIM_BIN%\..\") do (set NIM_ROOT=%%~fi)
+
+set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\nim-gdb.py
+set @NIM_GDB=gdb.exe
+
+@echo source %@GDB_PYTHON_MODULE_PATH%> wingdbcommand.txt
+%@NIM_GDB% --command="wingdbcommand.txt" %*
+del wingdbcommand.txt /f /q
+
+EXIT /B %ERRORLEVEL%
+@echo on

build_all.bat

@@ -0,0 +1,17 @@
+@echo off
+rem build development version of the compiler; can be rerun safely
+if not exist csources (
+  git clone --depth 1 https://github.com/nim-lang/csources.git
+)
+if not exist bin\nim.exe (
+  cd csources
+  if PROCESSOR_ARCHITECTURE == AMD64 (
+    SET ARCH=64
+  )
+  CALL build.bat
+  cd ..
+)
+bin\nim.exe c --skipUserCfg --skipParentCfg koch
+koch.exe boot -d:release --skipUserCfg --skipParentCfg
+koch.exe tools --skipUserCfg --skipParentCfg
+

build_all.sh

@@ -1,31 +1,52 @@
 #! /bin/sh
 
-# build development version of the compiler; can be rerun safely
+# build development version of the compiler; can be rerun safely.
+# arguments can be passed, eg `--os freebsd`
 
 set -u # error on undefined variables
 set -e # exit on first error
 
 echo_run(){
-  printf "\n$*\n"
+  echo "$*"
   "$@"
 }
 
-[ -d csources ] || echo_run git clone --depth 1 https://github.com/nim-lang/csources.git
+[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources.git
 
 nim_csources=bin/nim_csources
+
+build_nim_csources_via_script(){
+  echo_run cd csources
+  echo_run sh build.sh "$@"
+}
+
 build_nim_csources(){
-  ## avoid changing dir in case of failure
+  # avoid changing dir in case of failure
   (
-    echo_run cd csources
-    echo_run sh build.sh
+    if [ $# -ne 0 ]; then
+      # some args were passed (eg: `--cpu i386`), need to call build.sh
+      build_nim_csources_via_script "$@"
+    else
+      # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s)
+      makeX=make
+      unamestr=$(uname)
+      if [ "$unamestr" = 'FreeBSD' ]; then
+        makeX=gmake
+      fi
+      nCPU=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
+      which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script
+    fi
   )
   # keep $nim_csources in case needed to investigate bootstrap issues
   # without having to rebuild from csources
   echo_run cp bin/nim $nim_csources
 }
 
-[ -f $nim_csources ] || echo_run build_nim_csources
+[ -f $nim_csources ] || echo_run build_nim_csources $@
+
+# Note: if fails, may need to `cd csources && git pull`
+echo_run bin/nim c --skipUserCfg --skipParentCfg koch
+
+echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg
+echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools.
 
-echo_run bin/nim c koch
-echo_run ./koch boot -d:release
-echo_run ./koch tools # Compile Nimble and other tools.

changelog.md

@@ -1,253 +1,15 @@
-## v0.19.X - XX/XX/2018
-
-### Changes affecting backwards compatibility
-
-- The stdlib module ``future`` has been renamed to ``sugar``.
-- ``macros.callsite`` is now deprecated. Since the introduction of ``varargs``
-  parameters this became unnecessary.
-- Anonymous tuples with a single element can now be written as ``(1,)`` with a
-  trailing comma. The underlying AST is ``nnkTupleConstr(newLit 1)`` for this
-  example. ``nnkTupleConstr`` is a new node kind your macros need to be able
-  to deal with!
-- Indexing into a ``cstring`` for the JS target is now mapped
-  to ``charCodeAt``.
-- Assignments that would "slice" an object into its supertype are now prevented
-  at runtime. Use ``ref object`` with inheritance rather than ``object`` with
-  inheritance to prevent this issue.
-- The ``not nil`` type annotation now has to be enabled explicitly
-  via ``{.experimental: "notnil"}`` as we are still not pleased with how this
-  feature works with Nim's containers.
-- The parser now warns about inconsistent spacing around binary operators as
-  these can easily be confused with unary operators. This warning will likely
-  become an error in the future.
-- The ``'c`` and ``'C'`` suffix for octal literals is now deprecated to
-  bring the language in line with the standard library (e.g. ``parseOct``).
-- The dot style for import paths (e.g ``import path.to.module`` instead of
-  ``import path/to/module``) has been deprecated.
-
-#### Breaking changes in the standard library
-
-- ``re.split`` for empty regular expressions now yields every character in
-  the string which is what other programming languages chose to do.
-- The returned tuple of ``system.instantiationInfo`` now has a third field
-  containing the column of the instantiation.
-
-- ``cookies.setCookie`` no longer assumes UTC for the expiration date.
-- ``strutils.formatEng`` does not distinguish between ``nil`` and ``""``
-  strings anymore for its ``unit`` parameter. Instead the space is controlled
-  by a new parameter ``useUnitSpace``.
-
-- The ``times.parse`` and ``times.format`` procs have been rewritten.
-  The proc signatures are the same so it should generally not break anything.
-  However, the new implementation is a bit stricter, which is a breaking change.
-  For example ``parse("2017-01-01 foo", "yyyy-MM-dd")`` will now raise an error.
-
-- ``proc `-`*(a, b: Time): int64`` in the ``times`` module has changed return type
-  to ``times.Duration`` in order to support higher time resolutions.
-  The proc is no longer deprecated.
-
-- The ``times.Timezone`` is now an immutable ref-type that must be initialized
-  with an explicit constructor (``newTimezone``).
-
-- ``posix.Timeval.tv_sec`` has changed type to ``posix.Time``.
-
-- ``math.`mod` `` for floats now behaves the same as ``mod`` for integers
-  (previously it used floor division like Python). Use ``math.floorMod`` for the old behavior.
-
-- For string inputs, ``unicode.isUpper`` and ``unicode.isLower`` now require a
-  second mandatory parameter ``skipNonAlpha``.
-
-- For string inputs, ``strutils.isUpperAscii`` and ``strutils.isLowerAscii`` now
-  require a second mandatory parameter ``skipNonAlpha``.
-
-- ``osLastError`` is now marked with ``sideEffect``
-- The procs ``parseHexInt`` and ``parseOctInt`` now fail on empty strings
-  and strings containing only valid prefixes, e.g. "0x" for hex integers.
-
-- ``terminal.setCursorPos`` and ``terminal.setCursorXPos`` now work correctly
-  with 0-based coordinates on POSIX (previously, you needed to use
-  1-based coordinates on POSIX for correct behaviour; the Windows behaviour
-  was always correct).
-
-- ``lineInfoObj`` now returns absolute path instead of project path.
-  It's used by ``lineInfo``, ``check``, ``expect``, ``require``, etc.
-
-- ``net.sendTo`` no longer returns an int and now raises an ``OSError``.
-- `threadpool`'s `await` and derivatives have been renamed to `blockUntil`
-  to avoid confusions with `await` from the `async` macro.
+# v1.4.0 - yyyy-mm-dd
 
 
-#### Breaking changes in the compiler
 
-- The undocumented ``#? braces`` parsing mode was removed.
-- The undocumented PHP backend was removed.
-- The default location of ``nimcache`` for the native code targets was
-  changed. Read [the compiler user guide](https://nim-lang.org/docs/nimc.html#generated-c-code-directory)
-  for more information.
-
-### Library additions
-
-- ``re.split`` now also supports the ``maxsplit`` parameter for consistency
-  with ``strutils.split``.
-- Added ``system.toOpenArray`` in order to support zero-copy slicing
-  operations. This is currently not yet available for the JavaScript target.
-- Added ``getCurrentDir``, ``findExe``, ``cpDir`` and  ``mvDir`` procs to
-  ``nimscript``.
-- The ``times`` module now supports up to nanosecond time resolution when available.
-- Added the type ``times.Duration`` for representing fixed durations of time.
-- Added the proc ``times.convert`` for converting between different time units,
-  e.g days to seconds.
-- Added the proc ``algorithm.binarySearch[T, K]`` with the ```cmp``` parameter.
-- Added the proc ``algorithm.upperBound``.
-- Added inverse hyperbolic functions, ``math.arcsinh``, ``math.arccosh`` and ``math.arctanh`` procs.
-- Added cotangent, secant and cosecant procs ``math.cot``, ``math.sec`` and ``math.csc``; and their hyperbolic, inverse and inverse hyperbolic functions, ``math.coth``, ``math.sech``, ``math.csch``, ``math.arccot``, ``math.arcsec``, ``math.arccsc``, ``math.arccoth``, ``math.arcsech`` and ``math.arccsch`` procs.
-- Added the procs ``math.floorMod`` and ``math.floorDiv`` for floor based integer division.
-- Added the procs ``rationals.`div```, ``rationals.`mod```, ``rationals.floorDiv`` and ``rationals.floorMod`` for rationals.
-- Added the proc ``math.prod`` for product of elements in openArray.
-- Added the proc ``parseBinInt`` to parse a binary integer from a string, which returns the value.
-- ``parseOct`` and ``parseBin`` in parseutils now also support the ``maxLen`` argument similar to ``parseHexInt``.
-- Added the proc ``flush`` for memory mapped files.
-- Added the ``MemMapFileStream``.
-- Added a simple interpreting event parser template ``eventParser`` to the ``pegs`` module.
-- Added ``macros.copyLineInfo`` to copy lineInfo from other node.
-- Added ``system.ashr`` an arithmetic right shift for integers.
-
-### Library changes
-
-- ``macros.astGenRepr``, ``macros.lispRepr`` and ``macros.treeRepr``
-  now escapes the content of string literals consistently.
-- ``macros.NimSym`` and ``macros.NimIdent`` is now deprecated in favor
-  of the more general ``NimNode``.
-- ``macros.getImpl`` now includes the pragmas of types, instead of omitting them.
-- ``macros.hasCustomPragma`` and ``macros.getCustomPragmaVal`` now
-  also support ``ref`` and ``ptr`` types, pragmas on types and variant
-  fields.
-- ``system.SomeReal`` is now called ``SomeFloat`` for consistency and
-  correctness.
-- ``algorithm.smartBinarySearch`` and ``algorithm.binarySearch`` is
-  now joined in ``binarySearch``. ``smartbinarySearch`` is now
-  deprecated.
-- The `terminal` module now exports additional procs for generating ANSI color
-  codes as strings.
-- Added the parameter ``val`` for the ``CritBitTree[int].inc`` proc.
-- An exception raised from a ``test`` block of ``unittest`` now shows its type in
-  error message.
-- The ``compiler/nimeval`` API was rewritten to simplify the "compiler as an
-  API". Using the Nim compiler and its VM as a scripting engine has never been
-  easier. See ``tests/compilerapi/tcompilerapi.nim`` for an example of how to
-  use the Nim VM in a native Nim application.
-- Added the parameter ``val`` for the ``CritBitTree[T].incl`` proc.
-- The proc ``tgamma`` was renamed to ``gamma``. ``tgamma`` is deprecated.
-- The ``pegs`` module now exports getters for the fields of its ``Peg`` and ``NonTerminal``
-  object types. ``Peg``s with child nodes now have the standard ``items`` and ``pairs``
-  iterators.
-- The ``accept`` socket procedure defined in the ``net`` module can now accept
-  a nil socket.
-
-### Language additions
-
-- Dot calls combined with explicit generic instantiations can now be written
-  as ``x.y[:z]`` which is transformed into ``y[z](x)`` by the parser.
-- ``func`` is now an alias for ``proc {.noSideEffect.}``.
-- In order to make ``for`` loops and iterators more flexible to use Nim now
-  supports so called "for-loop macros". See
-  the [manual](manual.html#macros-for-loop-macros) for more details.
-  This feature enables a Python-like generic ``enumerate`` implementation.
-
-- Case statements can now be rewritten via macros. See the [manual](manual.html#macros-case-statement-macros) for more information.
-  This feature enables custom pattern matchers.
+## Standard library additions and changes
 
 
-- the `typedesc` special type has been renamed to just `type`.
-- `static` and `type` are now also modifiers similar to `ref` and `ptr`.
-  They denote the special types `static[T]` and `type[T]`.
-- Forcing compile-time evaluation with `static` now supports specifying
-  the desired target type (as a concrete type or as a type class)
-- The `type` operator now supports checking that the supplied expression
-  matches an expected type constraint.
-
-### Language changes
-
-- The `importcpp` pragma now allows importing the listed fields of generic
-  C++ types. Support for numeric parameters have also been added through
-  the use of `static[T]` types.
-  (#6415)
-
-- Native C++ exceptions can now be imported with `importcpp` pragma.
-  Imported exceptions can be raised and caught just like Nim exceptions.
-  More details in language manual.
-
-- ``nil`` for strings/seqs is finally gone. Instead the default value for
-  these is ``"" / @[]``. Use ``--nilseqs:on`` for a transition period.
-
-- Accessing the binary zero terminator in Nim's native strings
-  is now invalid. Internally a Nim string still has the trailing zero for
-  zero-copy interoperability with ``cstring``. Compile your code with the
-  new switch ``--laxStrings:on`` if you need a transition period.
-
-- The command syntax now supports keyword arguments after the first comma.
-
-- Thread-local variables can now be declared inside procs. This implies all
-  the effects of the ``global`` pragma.
-
-- Nim now supports the ``except`` clause in the export statement.
-
-- Range float types, example ``range[0.0 .. Inf]``. More details in language manual.
-- The ``{.this.}`` pragma has been deprecated. It never worked within generics and
-  we found the resulting code harder to read than the more explicit ``obj.field``
-  syntax.
-- "Memory regions" for pointer types have been deprecated, they were hardly used
-  anywhere. Note that this has **nothing** to do with the ``--gc:regions`` switch
-  of managing memory.
-
-- The exception hierarchy was slightly reworked, ``SystemError`` was renamed to
-  ``CatchableError`` and is the new base class for any exception that is guaranteed to
-  be catchable. This change should have minimal impact on most existing Nim code.
+## Language changes
 
 
-### Tool changes
+## Compiler changes
 
-- ``jsondoc2`` has been renamed ``jsondoc``, similar to how ``doc2`` was renamed
-  ``doc``. The old ``jsondoc`` can still be invoked with ``jsondoc0``.
 
-### Compiler changes
+## Tool changes
 
-- The VM's instruction count limit was raised to 3 million instructions in
-  order to support more complex computations at compile-time.
-
-- Support for hot code reloading has been implemented for the JavaScript
-  target. To use it, compile your code with `--hotCodeReloading:on` and use a
-  helper library such as LiveReload or BrowserSync.
-
-- A new compiler option `--cppCompileToNamespace` puts the generated C++ code
-  into the namespace "Nim" in order to avoid naming conflicts with existing
-  C++ code. This is done for all Nim code - internal and exported.
-
-- Added ``macros.getProjectPath`` and ``ospaths.putEnv`` procs to Nim's virtual
-  machine.
-
-- The ``deadCodeElim`` option is now always turned on and the switch has no
-  effect anymore, but is recognized for backwards compatibility.
-
-- ``experimental`` is now a pragma / command line switch that can enable specific
-  language extensions, it is not an all-or-nothing switch anymore.
-
-- Nintendo Switch was added as a new platform target. See [the compiler user guide](https://nim-lang.org/docs/nimc.html)
-  for more info.
-
-- macros.bindSym now capable to accepts not only literal string or string constant expression.
-  bindSym enhancement make it also can accepts computed string or ident node inside macros /
-  compile time functions / static blocks. Only in templates / regular code it retains it's old behavior.
-  This new feature can be accessed via {.experimental: "dynamicBindSym".} pragma/switch.
-
-- On Posix systems the global system wide configuration is now put under ``/etc/nim/nim.cfg``,
-  it used to be ``/etc/nim.cfg``. Usually it does not exist, however.
-
-- On Posix systems the user configuration is now looked under ``$XDG_CONFIG_HOME/nim/nim.cfg``
-  (if ``XDG_CONFIG_HOME`` is not defined, then under ``~/.config/nim/nim.cfg``). It used to be
-  ``$XDG_CONFIG_DIR/nim.cfg`` (and ``~/.config/nim.cfg``).
-
-  Similarly, on Windows, the user configuration is now looked under ``%APPDATA%/nim/nim.cfg``.
-  This used to be ``%APPDATA%/nim.cfg``.
-
-### Bugfixes

changelogs/changelog_0_18_1.md

@@ -0,0 +1,663 @@
+## v0.18.0 - 01/03/2018
+
+### Changes affecting backwards compatibility
+
+#### Breaking changes in the standard library
+
+- The ``[]`` proc for strings now raises an ``IndexError`` exception when
+  the specified slice is out of bounds. See issue
+  [#6223](https://github.com/nim-lang/Nim/issues/6223) for more details.
+  You can use ``substr(str, start, finish)`` to get the old behaviour back,
+  see [this commit](https://github.com/nim-lang/nimbot/commit/98cc031a27ea89947daa7f0bb536bcf86462941f) for an example.
+
+- ``strutils.split`` and ``strutils.rsplit`` with an empty string and a
+  separator now returns that empty string.
+  See issue [#4377](https://github.com/nim-lang/Nim/issues/4377).
+
+- Arrays of char cannot be converted to ``cstring`` anymore, pointers to
+  arrays of char can! This means ``$`` for arrays can finally exist
+  in ``system.nim`` and do the right thing. This means ``$myArrayOfChar`` changed
+  its behaviour! Compile with ``-d:nimNoArrayToString`` to see where to fix your
+  code.
+
+- `reExtended` is no longer default for the `re` constructor in the `re`
+  module.
+
+- The behavior of ``$`` has been changed for all standard library collections. The
+  collection-to-string implementations now perform proper quoting and escaping of
+  strings and chars.
+
+- `newAsyncSocket` taking an `AsyncFD` now runs `setBlocking(false)` on the
+  fd.
+
+- ``mod`` and bitwise ``and`` do not produce ``range`` subtypes anymore. This
+  turned out to be more harmful than helpful and the language is simpler
+  without this special typing rule.
+
+- ``formatFloat``/``formatBiggestFloat`` now support formatting floats with zero
+  precision digits. The previous ``precision = 0`` behavior (default formatting)
+  is now available via ``precision = -1``.
+
+- Moved from stdlib into Nimble packages:
+  - [``basic2d``](https://github.com/nim-lang/basic2d)
+    _deprecated: use ``glm``, ``arraymancer``, ``neo``, or another package instead_
+  - [``basic3d``](https://github.com/nim-lang/basic3d)
+    _deprecated: use ``glm``, ``arraymancer``, ``neo``, or another package instead_
+  - [``gentabs``](https://github.com/lcrees/gentabs)
+  - [``libuv``](https://github.com/lcrees/libuv)
+  - [``numeric``](https://github.com/lcrees/polynumeric)
+  - [``poly``](https://github.com/lcrees/polynumeric)
+  - [``pdcurses``](https://github.com/lcrees/pdcurses)
+  - [``romans``](https://github.com/lcrees/romans)
+  - [``libsvm``](https://github.com/nim-lang/libsvm_legacy)
+  - [``joyent_http_parser``](https://github.com/nim-lang/joyent_http_parser)
+
+- Proc [toCountTable](https://nim-lang.org/docs/tables.html#toCountTable,openArray[A])
+  now produces a `CountTable` with values correspoding to the number of occurrences
+  of the key in the input. It used to produce a table with all values set to `1`.
+
+  Counting occurrences in a sequence used to be:
+
+  ```nim
+  let mySeq = @[1, 2, 1, 3, 1, 4]
+  var myCounter = initCountTable[int]()
+
+  for item in mySeq:
+    myCounter.inc item
+  ```
+
+  Now, you can simply do:
+
+  ```nim
+  let
+    mySeq = @[1, 2, 1, 3, 1, 4]
+    myCounter = mySeq.toCountTable()
+  ```
+
+- If you use ``--dynlibOverride:ssl`` with OpenSSL 1.0.x, you now have to
+  define ``openssl10`` symbol (``-d:openssl10``). By default OpenSSL 1.1.x is
+  assumed.
+
+- ``newNativeSocket`` is now named ``createNativeSocket``.
+
+- ``newAsyncNativeSocket`` is now named ``createAsyncNativeSocket``
+  and it no longer raises an OS error but returns an ``osInvalidSocket`` when
+  creation fails.
+
+- The ``securehash`` module is now deprecated. Instead import ``std / sha1``.
+
+- The ``readPasswordFromStdin`` proc has been moved from the ``rdstdin``
+  to the ``terminal`` module, thus it does not depend on linenoise anymore.
+
+#### Breaking changes in the compiler
+
+- ``\n`` is now only the single line feed character like in most
+  other programming languages. The new platform specific newline escape sequence is
+  written as ``\p``. This change only affects the Windows platform.
+
+- The overloading rules changed slightly so that constrained generics are
+  preferred over unconstrained generics. (Bug #6526)
+
+- We changed how array accesses "from backwards" like ``a[^1]`` or ``a[0..^1]`` are
+  implemented. These are now implemented purely in ``system.nim`` without compiler
+  support. There is a new "heterogeneous" slice type ``system.HSlice`` that takes 2
+  generic parameters which can be ``BackwardsIndex`` indices. ``BackwardsIndex`` is
+  produced by ``system.^``.
+  This means if you overload ``[]`` or ``[]=`` you need to ensure they also work
+  with ``system.BackwardsIndex`` (if applicable for the accessors).
+
+- The parsing rules of ``if`` expressions were changed so that multiple
+  statements are allowed in the branches. We found few code examples that
+  now fail because of this change, but here is one:
+
+```nim
+t[ti] = if exp_negative: '-' else: '+'; inc(ti)
+```
+
+This now needs to be written as:
+
+```nim
+t[ti] = (if exp_negative: '-' else: '+'); inc(ti)
+```
+
+- The experimental overloading of the dot ``.`` operators now take
+  an ``untyped``` parameter as the field name, it used to be
+  a ``static[string]``. You can use ``when defined(nimNewDot)`` to make
+  your code work with both old and new Nim versions.
+  See [special-operators](https://nim-lang.org/docs/manual.html#special-operators)
+  for more information.
+
+- ``yield`` (or ``await`` which is mapped to ``yield``) never worked reliably
+  in an array, seq or object constructor and is now prevented at compile-time.
+
+### Library additions
+
+- **Added ``sequtils.mapLiterals`` for easier construction of array and tuple literals.**
+
+- Added ``system.runnableExamples`` to make examples in Nim's documentation easier
+  to write and test. The examples are tested as the last step of
+  ``nim doc``.
+
+- Implemented ``getIoHandler`` proc in the ``asyncdispatch`` module that allows
+  you to retrieve the underlying IO Completion Port or ``Selector[AsyncData]``
+  object in the specified dispatcher.
+
+- For string formatting / interpolation a new module
+  called [strformat](https://nim-lang.org/docs/strformat.html) has been added
+  to the stdlib.
+
+- The `ReadyKey` type in the selectors module now contains an ``errorCode``
+  field to help distinguish between ``Event.Error`` events.
+
+- Implemented an `accept` proc that works on a `SocketHandle` in
+  ``nativesockets``.
+
+- Added ``algorithm.rotateLeft``.
+
+- Added ``typetraits.$`` as an alias for ``typetraits.name``.
+
+- Added ``system.getStackTraceEntries`` that allows you to access the stack
+  trace in a structured manner without string parsing.
+
+- Added ``parseutils.parseSaturatedNatural``.
+
+- Added ``macros.unpackVarargs``.
+
+- Added support for asynchronous programming for the JavaScript backend using
+  the `asyncjs` module.
+
+- Added true color support for some terminals. Example:
+```nim
+import colors, terminal
+
+const Nim = "Efficient and expressive programming."
+
+var
+  fg = colYellow
+  bg = colBlue
+  int = 1.0
+
+enableTrueColors()
+
+for i in 1..15:
+  styledEcho bgColor, bg, fgColor, fg, Nim, resetStyle
+  int -= 0.01
+  fg = intensity(fg, int)
+
+setForegroundColor colRed
+setBackgroundColor colGreen
+styledEcho "Red on Green.", resetStyle
+```
+
+### Library changes
+
+- ``echo`` now works with strings that contain ``\0`` (the binary zero is not
+  shown) and ``nil`` strings are equal to empty strings.
+
+- JSON: Deprecated `getBVal`, `getFNum`, and `getNum` in favour of
+  `getBool`, `getFloat`, `getBiggestInt`. A new `getInt` procedure was also
+  added.
+
+- ``rationals.toRational`` now uses an algorithm based on continued fractions.
+  This means its results are more precise and it can't run into an infinite loop
+  anymore.
+
+- ``os.getEnv`` now takes an optional ``default`` parameter that tells ``getEnv``
+  what to return if the environment variable does not exist.
+
+- The ``random`` procs in ``random.nim`` have all been deprecated. Instead use
+  the new ``rand`` procs. The module now exports the state of the random
+  number generator as type ``Rand`` so multiple threads can easily use their
+  own random number generators that do not require locking. For more information
+  about this rename see issue [#6934](https://github.com/nim-lang/Nim/issues/6934)
+
+- ``writeStackTrace`` is now proclaimed to have no IO effect (even though it does)
+  so that it is more useful for debugging purposes.
+
+- ``db_mysql`` module: ``DbConn`` is now a ``distinct`` type that doesn't expose the
+  details of the underlying ``PMySQL`` type.
+
+- ``parseopt2`` is now deprecated, use ``parseopt`` instead.
+
+### Language additions
+
+- It is now possible to forward declare object types so that mutually
+  recursive types can be created across module boundaries. See
+  [package level objects](https://nim-lang.org/docs/manual.html#package-level-objects)
+  for more information.
+
+- Added support for casting between integers of same bitsize in VM (compile time and nimscript).
+  This allows to, among other things, reinterpret signed integers as unsigned.
+
+- Custom pragmas are now supported using pragma ``pragma``, please see language
+  manual for details.
+
+- Standard library modules can now also be imported via the ``std`` pseudo-directory.
+  This is useful in order to distinguish between standard library and nimble package
+  imports:
+
+  ```nim
+  import std / [strutils, os, osproc]
+  import someNimblePackage / [strutils, os]
+  ```
+
+### Language changes
+
+- The **unary** ``<`` is now deprecated, for ``.. <`` use ``..<`` for other usages
+  use the ``pred`` proc.
+
+- Bodies of ``for`` loops now get their own scope:
+
+```nim
+# now compiles:
+for i in 0..4:
+  let i = i + 1
+  echo i
+```
+
+- To make Nim even more robust the system iterators ``..`` and ``countup``
+  now only accept a single generic type ``T``. This means the following code
+  doesn't die with an "out of range" error anymore:
+
+```nim
+var b = 5.Natural
+var a = -5
+for i in a..b:
+  echo i
+```
+
+- ``atomic`` and ``generic`` are no longer keywords in Nim. ``generic`` used to be
+  an alias for ``concept``, ``atomic`` was not used for anything.
+
+- The memory manager now uses a variant of the TLSF algorithm that has much
+  better memory fragmentation behaviour. According
+  to [http://www.gii.upv.es/tlsf/](http://www.gii.upv.es/tlsf/) the maximum
+  fragmentation measured is lower than 25%. As a nice bonus ``alloc`` and
+  ``dealloc`` became O(1) operations.
+
+- The compiler is now more consistent in its treatment of ambiguous symbols:
+  Types that shadow procs and vice versa are marked as ambiguous (bug #6693).
+
+- codegenDecl pragma now works for the JavaScript backend. It returns an empty
+  string for function return type placeholders.
+
+- Extra semantic checks for procs with noreturn pragma: return type is not allowed,
+  statements after call to noreturn procs are no longer allowed.
+
+- Noreturn proc calls and raising exceptions branches are now skipped during common type
+  deduction in ``if`` and ``case`` expressions. The following code snippets now compile:
+  ```nim
+  import strutils
+  let str = "Y"
+  let a = case str:
+    of "Y": true
+    of "N": false
+    else: raise newException(ValueError, "Invalid boolean")
+  let b = case str:
+    of nil, "": raise newException(ValueError, "Invalid boolean")
+    elif str.startsWith("Y"): true
+    elif str.startsWith("N"): false
+    else: false
+  let c = if str == "Y": true
+    elif str == "N": false
+    else:
+      echo "invalid bool"
+      quit("this is the end")
+  ```
+
+- Pragmas now support call syntax, for example: ``{.exportc"myname".}`` and
+  ``{.exportc("myname").}``
+
+- The ``deprecated`` pragma now supports a user-definable warning message for procs.
+
+  ```nim
+  proc bar {.deprecated: "use foo instead".} =
+    return
+
+  bar()
+  ```
+
+### Tool changes
+
+- The ``nim doc`` command is now an alias for ``nim doc2``, the second version of
+  the documentation generator. The old version 1 can still be accessed
+  via the new ``nim doc0`` command.
+
+- Nim's ``rst2html`` command now supports the testing of code snippets via an RST
+  extension that we called ``:test:``::
+
+  ```rst
+  .. code-block:: nim
+      :test:
+    # shows how the 'if' statement works
+    if true: echo "yes"
+  ```
+
+### Compiler changes
+
+### Bugfixes
+
+- Fixed "ReraiseError when using try/except within finally block"
+  ([#5871](https://github.com/nim-lang/Nim/issues/5871))
+- Fixed "Range type inference leads to counter-intuitive behvaiour"
+  ([#5854](https://github.com/nim-lang/Nim/issues/5854))
+- Fixed "JSON % operator can fail in extern procs with dynamic types"
+  ([#6385](https://github.com/nim-lang/Nim/issues/6385))
+- Fixed ""intVal is not accessible" in VM"
+  ([#6083](https://github.com/nim-lang/Nim/issues/6083))
+- Fixed "Add excl for OrderedSet"
+  ([#2467](https://github.com/nim-lang/Nim/issues/2467))
+- Fixed "newSeqOfCap actually doesn't reserve memory"
+  ([#6403](https://github.com/nim-lang/Nim/issues/6403))
+- Fixed "[Regression] Nim segfaults"
+  ([#6435](https://github.com/nim-lang/Nim/issues/6435))
+- Fixed "Seq assignment is slower than expected"
+  ([#6433](https://github.com/nim-lang/Nim/issues/6433))
+- Fixed "json module issues with empty dicts and lists"
+  ([#6438](https://github.com/nim-lang/Nim/issues/6438))
+- Fixed "mingw installed via finish.exe fails to link if Nim located in path with whitespace"
+  ([#6452](https://github.com/nim-lang/Nim/issues/6452))
+- Fixed "unittest.check does not perform short-circuit evaluation"
+  ([#5784](https://github.com/nim-lang/Nim/issues/5784))
+- Fixed "Error while concatenating an array of chars."
+  ([#5861](https://github.com/nim-lang/Nim/issues/5861))
+- Fixed "range initialization: [ProveInit] hint: Cannot prove that"
+  ([#6474](https://github.com/nim-lang/Nim/issues/6474))
+- Fixed "scanf can call procs with side-effects multiple times"
+  ([#6487](https://github.com/nim-lang/Nim/issues/6487))
+- Fixed "gcsafe detection problem"
+  ([#5620](https://github.com/nim-lang/Nim/issues/5620))
+- Fixed "C++ codegen: `mitems` generates invalid code."
+  ([#4910](https://github.com/nim-lang/Nim/issues/4910))
+- Fixed "strange runtime behavior on macOS"
+  ([#6496](https://github.com/nim-lang/Nim/issues/6496))
+- Fixed "stdtmpl: invalid indentation after a line ending in question mark"
+  ([#5070](https://github.com/nim-lang/Nim/issues/5070))
+- Fixed "Windows: NAN troubles on c backend"
+  ([#6511](https://github.com/nim-lang/Nim/issues/6511))
+- Fixed "lib/nim/system/cellsets.nim(33, 31) Error: type mismatch while attempting to compile for 16bit CPUs"
+  ([#3558](https://github.com/nim-lang/Nim/issues/3558))
+- Fixed "Can't compile dynlib with ``-d:useNimRtl`` and ``--threads:on``"
+  ([#5143](https://github.com/nim-lang/Nim/issues/5143))
+- Fixed "var s = @[0,1,2,...] can generate thousand of single assignments in C code"
+  ([#5007](https://github.com/nim-lang/Nim/issues/5007))
+- Fixed "`echo` discards everything after a null character"
+  ([#1137](https://github.com/nim-lang/Nim/issues/1137))
+- Fixed "Turn off reExtended by default"
+  ([#5627](https://github.com/nim-lang/Nim/issues/5627))
+- Fixed "Bad Links in docs/backends.html"
+  ([#5914](https://github.com/nim-lang/Nim/issues/5914))
+- Fixed "Index out of bounds error in db_postgres when executing non parameter-substituted queries containing "?""
+  ([#6571](https://github.com/nim-lang/Nim/issues/6571))
+- Fixed "Please add pipe2 support to posix stdlib"
+  ([#6553](https://github.com/nim-lang/Nim/issues/6553))
+- Fixed "Return semantics vary depending on return style"
+  ([#6422](https://github.com/nim-lang/Nim/issues/6422))
+- Fixed "parsecsv.open reports SIGSEGV when calling 'open' on missing file"
+  ([#6148](https://github.com/nim-lang/Nim/issues/6148))
+- Fixed "VCC: Nim generates non-compilable code for system.nim"
+  ([#6606](https://github.com/nim-lang/Nim/issues/6606))
+- Fixed "Generic subtype matches worse than a generic"
+  ([#6526](https://github.com/nim-lang/Nim/issues/6526))
+- Fixed "formatFloat inconsistent scientific notation"
+  ([#6589](https://github.com/nim-lang/Nim/issues/6589))
+- Fixed "Generated c code calls function twice"
+  ([#6292](https://github.com/nim-lang/Nim/issues/6292))
+- Fixed "Range type inference leads to counter-intuitive behvaiour"
+  ([#5854](https://github.com/nim-lang/Nim/issues/5854))
+- Fixed "New backward indexing is too limited"
+  ([#6631](https://github.com/nim-lang/Nim/issues/6631))
+- Fixed "Table usage in a macro (SIGSEGV: Illegal storage access.)"
+  ([#1860](https://github.com/nim-lang/Nim/issues/1860))
+- Fixed "Incorrect deprecation error"
+  ([#6634](https://github.com/nim-lang/Nim/issues/6634))
+- Fixed "Wrong indices in arrays not starting with 0"
+  ([#6675](https://github.com/nim-lang/Nim/issues/6675))
+- Fixed "if expressions"
+  ([#6609](https://github.com/nim-lang/Nim/issues/6609))
+- Fixed "BackwardsIndex: converter + `[]` + unrelated type[^1]: lib/system.nim(3536, 3) Error"
+  ([#6692](https://github.com/nim-lang/Nim/issues/6692))
+- Fixed "BackwardsIndex: converter + `[]` + unrelated type[^1]: lib/system.nim(3536, 3) Error"
+  ([#6692](https://github.com/nim-lang/Nim/issues/6692))
+- Fixed "js backend 0.17.3: array bounds check for non zero based arrays is buggy"
+  ([#6532](https://github.com/nim-lang/Nim/issues/6532))
+- Fixed "HttpClient's new API doesn't work through a proxy for https URLs"
+  ([#6685](https://github.com/nim-lang/Nim/issues/6685))
+- Fixed "isServing isn't declared and isn't compiling"
+  ([#6707](https://github.com/nim-lang/Nim/issues/6707))
+- Fixed "[Regression] value out of range"
+  ([#6710](https://github.com/nim-lang/Nim/issues/6710))
+
+- Fixed "Error when using `multisync` macro"
+  ([#6708](https://github.com/nim-lang/Nim/issues/6708))
+
+- Fixed "formatFloat inconsistent scientific notation"
+  ([#6589](https://github.com/nim-lang/Nim/issues/6589))
+- Fixed "Using : (constructor arguments) for passing values to functions with default arguments causes a compiler crash."
+  ([#6765](https://github.com/nim-lang/Nim/issues/6765))
+- Fixed "In-place object initialization leads to vcc incompatible code"
+  ([#6757](https://github.com/nim-lang/Nim/issues/6757))
+- Fixed "Improve parseCookies doc"
+  ([#5721](https://github.com/nim-lang/Nim/issues/5721))
+- Fixed "Parser regression with nested do notation inside conditional"
+  ([#6166](https://github.com/nim-lang/Nim/issues/6166))
+- Fixed "Request for better error message"
+  ([#6776](https://github.com/nim-lang/Nim/issues/6776))
+- Fixed "Testament tester does not execute test with `exitcode` only"
+  ([#6775](https://github.com/nim-lang/Nim/issues/6775))
+- Fixed "JS integer division off by one"
+  ([#6753](https://github.com/nim-lang/Nim/issues/6753))
+- Fixed "Regression: cannot prove not nil"
+  ([#5781](https://github.com/nim-lang/Nim/issues/5781))
+- Fixed "SIGSEGV: Illegal storage access. (Attempt to read from nil?) in generic proc"
+  ([#6073](https://github.com/nim-lang/Nim/issues/6073))
+- Fixed "Request for better error message"
+  ([#6776](https://github.com/nim-lang/Nim/issues/6776))
+- Fixed "Nim #head: sorting via reference hangs compiler"
+  ([#6724](https://github.com/nim-lang/Nim/issues/6724))
+- Fixed "Cannot cast pointer to char in cpp"
+  ([#5979](https://github.com/nim-lang/Nim/issues/5979))
+- Fixed "asynchttpserver replies with several errors on single request"
+  ([#6386](https://github.com/nim-lang/Nim/issues/6386))
+- Fixed "object variants superclass trigger bad codegen"
+  ([#5521](https://github.com/nim-lang/Nim/issues/5521))
+- Fixed "JS integer division off by one"
+  ([#6753](https://github.com/nim-lang/Nim/issues/6753))
+- Fixed "js backend compiler crash with tables indexed by certain types"
+  ([#6568](https://github.com/nim-lang/Nim/issues/6568))
+- Fixed "Jsgen bug with is"
+  ([#6445](https://github.com/nim-lang/Nim/issues/6445))
+- Fixed "Subrange definition with ..<"
+  ([#6788](https://github.com/nim-lang/Nim/issues/6788))
+- Fixed "fields not initialized: array with enum index type as object field."
+  ([#6682](https://github.com/nim-lang/Nim/issues/6682))
+- Fixed "Can not delete data in table when table's data type is kind of "not nil""
+  ([#6555](https://github.com/nim-lang/Nim/issues/6555))
+- Fixed "tables.nim: Cannot prove that 'n' is initialized"
+  ([#6121](https://github.com/nim-lang/Nim/issues/6121))
+- Fixed "issues with 'not nil' applied to a closure proc"
+  ([#6489](https://github.com/nim-lang/Nim/issues/6489))
+- Fixed "`not nil` not working in some cases"
+  ([#4686](https://github.com/nim-lang/Nim/issues/4686))
+- Fixed "Cannot prove '@[v]' is not nil"
+  ([#3993](https://github.com/nim-lang/Nim/issues/3993))
+
+- Fixed "Feature: support TCP_NODELAY in net.sockets"
+  ([#6795](https://github.com/nim-lang/Nim/issues/6795))
+- Fixed "Code that makes the compiler throw an error message and then hangs"
+  ([#6820](https://github.com/nim-lang/Nim/issues/6820))
+- Fixed "Code that makes the compiler throw an error message and then hangs"
+  ([#6820](https://github.com/nim-lang/Nim/issues/6820))
+- Fixed "Inconsistent behavior with sequence and string slicing"
+  ([#6223](https://github.com/nim-lang/Nim/issues/6223))
+- Fixed "Wrong behavior of "split" (proc and iterator)"
+  ([#4377](https://github.com/nim-lang/Nim/issues/4377))
+- Fixed "[Documentation] Invalid module name: [foo, bar]"
+  ([#6831](https://github.com/nim-lang/Nim/issues/6831))
+- Fixed "The destructor is not called for temporary objects"
+  ([#4214](https://github.com/nim-lang/Nim/issues/4214))
+- Fixed "Destructors does not work with implicit items iterator in for loop"
+  ([#985](https://github.com/nim-lang/Nim/issues/985))
+- Fixed "Error in template when using the type of the parameter inside it"
+  ([#6756](https://github.com/nim-lang/Nim/issues/6756))
+- Fixed "should json.to() respect parent attributes?"
+  ([#5856](https://github.com/nim-lang/Nim/issues/5856))
+- Fixed "json 'to' macro can not marshalize into tuples"
+  ([#6095](https://github.com/nim-lang/Nim/issues/6095))
+- Fixed "json.to fails with seq[T]"
+  ([#6604](https://github.com/nim-lang/Nim/issues/6604))
+- Fixed "json.to() is not worth using compared to marshal.to[T]"
+  ([#5848](https://github.com/nim-lang/Nim/issues/5848))
+- Fixed "Memory not being released in time, running out of memory"
+  ([#6031](https://github.com/nim-lang/Nim/issues/6031))
+- Fixed "[Regression] Bad C codegen for generic code"
+  ([#6889](https://github.com/nim-lang/Nim/issues/6889))
+- Fixed "rollingFileLogger deletes file on every start."
+  ([#6264](https://github.com/nim-lang/Nim/issues/6264))
+- Fixed "Remove/deprecate securehash module."
+  ([#6033](https://github.com/nim-lang/Nim/issues/6033))
+- Fixed "[bug or not] object construction for seq[T] failed without space after colon"
+  ([#5999](https://github.com/nim-lang/Nim/issues/5999))
+- Fixed "issues with the random module"
+  ([#4726](https://github.com/nim-lang/Nim/issues/4726))
+- Fixed "Reassigning local var to seq of objects results in nil element in Object's seq field"
+  ([#668](https://github.com/nim-lang/Nim/issues/668))
+- Fixed "Compilation error with "newseq[string]""
+  ([#6726](https://github.com/nim-lang/Nim/issues/6726))
+- Fixed "await inside array/dict literal produces invalid code - Part 2"
+  ([#6626](https://github.com/nim-lang/Nim/issues/6626))
+- Fixed "terminal.eraseline() gives OverflowError on Windows"
+  ([#6931](https://github.com/nim-lang/Nim/issues/6931))
+- Fixed "[Regression] `sequtils.any` conflicts with `system.any`"
+  ([#6932](https://github.com/nim-lang/Nim/issues/6932))
+- Fixed "C++ codegen: `mitems` generates invalid code."
+  ([#4910](https://github.com/nim-lang/Nim/issues/4910))
+- Fixed "seq.mitems produces invalid cpp codegen"
+  ([#6892](https://github.com/nim-lang/Nim/issues/6892))
+- Fixed "Concepts regression"
+  ([#6108](https://github.com/nim-lang/Nim/issues/6108))
+- Fixed "Generic iterable concept with array crashes compiler"
+  ([#6277](https://github.com/nim-lang/Nim/issues/6277))
+- Fixed "C code generation "‘a’ is a pointer; did you mean to use ‘->’?""
+  ([#6462](https://github.com/nim-lang/Nim/issues/6462))
+- Fixed "`--NimblePath` fails if a `-` in path which is not followed by a number"
+  ([#6949](https://github.com/nim-lang/Nim/issues/6949))
+- Fixed ""not registered in the selector" in asyncfile.close() for something that clearly was registered"
+  ([#6906](https://github.com/nim-lang/Nim/issues/6906))
+- Fixed "strange frexp behavior"
+  ([#6353](https://github.com/nim-lang/Nim/issues/6353))
+
+- Fixed "noreturn branches of case statements shouldn't contribute to type"
+  ([#6885](https://github.com/nim-lang/Nim/issues/6885))
+- Fixed "Type inference for 'if' statements changed"
+  ([#6980](https://github.com/nim-lang/Nim/issues/6980))
+- Fixed "newest asyncdispatch recursion"
+  ([#6100](https://github.com/nim-lang/Nim/issues/6100))
+- Fixed "Ambiguous identifier between set type and proc"
+  ([#6965](https://github.com/nim-lang/Nim/issues/6965))
+
+- Fixed "Inconsistent behavior with sequence and string slicing"
+  ([#6223](https://github.com/nim-lang/Nim/issues/6223))
+
+- Fixed "Unsupported OpenSSL library imported dynamically"
+  ([#5000](https://github.com/nim-lang/Nim/issues/5000))
+- Fixed "`nim check` segfaults"
+  ([#6972](https://github.com/nim-lang/Nim/issues/6972))
+- Fixed "GC deadlock"
+  ([#6988](https://github.com/nim-lang/Nim/issues/6988))
+- Fixed "Create a seq without memory initialization"
+  ([#6401](https://github.com/nim-lang/Nim/issues/6401))
+- Fixed "Fix bug for getch on Windows while using the arrow keys"
+  ([#6966](https://github.com/nim-lang/Nim/issues/6966))
+- Fixed "runnableExamples doesn't work in templates"
+  ([#7018](https://github.com/nim-lang/Nim/issues/7018))
+- Fixed "runnableExamples doesn't work with untyped statement blocks"
+  ([#7019](https://github.com/nim-lang/Nim/issues/7019))
+
+- Fixed "Critical bug in parseBiggestFloat"
+  ([#7060](https://github.com/nim-lang/Nim/issues/7060))
+- Fixed "[RFC] strformat.% should be gone"
+  ([#7078](https://github.com/nim-lang/Nim/issues/7078))
+- Fixed "compiler crash on simple macro"
+  ([#7093](https://github.com/nim-lang/Nim/issues/7093))
+- Fixed "Make newlines sane again"
+  ([#7089](https://github.com/nim-lang/Nim/issues/7089))
+- Fixed "JS - Unicode enum string representation issue"
+  ([#6741](https://github.com/nim-lang/Nim/issues/6741))
+- Fixed "Strange behaviour of 0.17.3 (working ok in 0.17.2)"
+  ([#6989](https://github.com/nim-lang/Nim/issues/6989))
+- Fixed "Strange behaviour of 0.17.3 (working ok in 0.17.2)"
+  ([#6989](https://github.com/nim-lang/Nim/issues/6989))
+- Fixed "Compiler crash: try expression with infix as"
+  ([#7116](https://github.com/nim-lang/Nim/issues/7116))
+- Fixed "nimsuggest crash"
+  ([#7140](https://github.com/nim-lang/Nim/issues/7140))
+- Fixed "[RFC] Reintroduce readChar"
+  ([#7072](https://github.com/nim-lang/Nim/issues/7072))
+- Fixed "Copyright line needs updating"
+  ([#7129](https://github.com/nim-lang/Nim/issues/7129))
+- Fixed "-0.0 doesn't result in negative zero in VM"
+  ([#7079](https://github.com/nim-lang/Nim/issues/7079))
+- Fixed "Windows large filesize"
+  ([#7121](https://github.com/nim-lang/Nim/issues/7121))
+- Fixed "Securehash is not parsimonious with MD5 and other hash modules"
+  ([#6961](https://github.com/nim-lang/Nim/issues/6961))
+- Fixed "os.findExe() shouldn't look in current directory on posix, unless exe has a /"
+  ([#6939](https://github.com/nim-lang/Nim/issues/6939))
+- Fixed "`compiles(...)` with `fatal` pragma causes compiler to exit early"
+  ([#7080](https://github.com/nim-lang/Nim/issues/7080))
+- Fixed "NPE when compile macro that returns concrete value"
+  ([#5450](https://github.com/nim-lang/Nim/issues/5450))
+- Fixed "Using a variable of type `int | float` causes internal compiler error"
+  ([#6946](https://github.com/nim-lang/Nim/issues/6946))
+- Fixed "Unsigned integers could not be used as array indexes."
+  ([#7153](https://github.com/nim-lang/Nim/issues/7153))
+- Fixed "countdown with uint causes underflow"
+  ([#4220](https://github.com/nim-lang/Nim/issues/4220))
+- Fixed "Inconsistent method call syntax"
+  ([#7200](https://github.com/nim-lang/Nim/issues/7200))
+- Fixed "Impossible to create an empty const array"
+  ([#6853](https://github.com/nim-lang/Nim/issues/6853))
+- Fixed "Strange UINT handling"
+  ([#3985](https://github.com/nim-lang/Nim/issues/3985))
+- Fixed "Bad codegen when passing arg that is part of return value destination"
+  ([#6960](https://github.com/nim-lang/Nim/issues/6960))
+- Fixed "No info about gcsafety in error message when global var is accessed in async proc"
+  ([#6186](https://github.com/nim-lang/Nim/issues/6186))
+- Fixed "Resolving package vs. local import ambiguities"
+  ([#2819](https://github.com/nim-lang/Nim/issues/2819))
+- Fixed "Internal error with type() operator"
+  ([#3711](https://github.com/nim-lang/Nim/issues/3711))
+- Fixed "newAsyncSocket should raise an OS error plus other inconsistencies"
+  ([#4995](https://github.com/nim-lang/Nim/issues/4995))
+- Fixed "mapLiterals fails with negative values"
+  ([#7215](https://github.com/nim-lang/Nim/issues/7215))
+- Fixed "fmWrite doesn't truncate file with openAsync, unlike open()"
+  ([#5531](https://github.com/nim-lang/Nim/issues/5531))
+- Fixed "Move libsvm to an external nimble module"
+  ([#5786](https://github.com/nim-lang/Nim/issues/5786))
+- Fixed "Prevent acceptAddr gotcha with newSocket"
+  ([#7227](https://github.com/nim-lang/Nim/issues/7227))
+- Fixed "strtabs.getOrDefault is inconsistent with tables.getOrDefault"
+  ([#4265](https://github.com/nim-lang/Nim/issues/4265))
+
+- Fixed "Code falling through into exception handler when no exception thrown."
+  ([#7232](https://github.com/nim-lang/Nim/issues/7232))
+- Fixed "the new generic inference rules are broken"
+  ([#7247](https://github.com/nim-lang/Nim/issues/7247))
+- Fixed "Odd `..<` regression"
+  ([#6992](https://github.com/nim-lang/Nim/issues/6992))
+- Fixed "Different proc type inferred from default parameter"
+  ([#4659](https://github.com/nim-lang/Nim/issues/4659))
+- Fixed "Different proc type inferred from default parameter"
+  ([#4659](https://github.com/nim-lang/Nim/issues/4659))
+- Fixed "Testament sometimes ignores test failures"
+  ([#7236](https://github.com/nim-lang/Nim/issues/7236))
+- Fixed "New Allocator Fails On >=4GB Requests"
+  ([#7120](https://github.com/nim-lang/Nim/issues/7120))
+- Fixed "User pragmas hide effect specifications from sempass2"
+  ([#7216](https://github.com/nim-lang/Nim/issues/7216))
+- Fixed "C++: SIGABRT instead of IndexError for out-of-bounds"
+  ([#6512](https://github.com/nim-lang/Nim/issues/6512))
+- Fixed "An uncaught exception in cpp mode doesn't show the exception name/msg"
+  ([#6431](https://github.com/nim-lang/Nim/issues/6431))

changelogs/changelog_0_19_0.md

@@ -0,0 +1,253 @@
+## v0.19.X - XX/XX/2018
+
+### Changes affecting backwards compatibility
+
+- The stdlib module ``future`` has been renamed to ``sugar``.
+- ``macros.callsite`` is now deprecated. Since the introduction of ``varargs``
+  parameters this became unnecessary.
+- Anonymous tuples with a single element can now be written as ``(1,)`` with a
+  trailing comma. The underlying AST is ``nnkTupleConstr(newLit 1)`` for this
+  example. ``nnkTupleConstr`` is a new node kind your macros need to be able
+  to deal with!
+- Indexing into a ``cstring`` for the JS target is now mapped
+  to ``charCodeAt``.
+- Assignments that would "slice" an object into its supertype are now prevented
+  at runtime. Use ``ref object`` with inheritance rather than ``object`` with
+  inheritance to prevent this issue.
+- The ``not nil`` type annotation now has to be enabled explicitly
+  via ``{.experimental: "notnil"}`` as we are still not pleased with how this
+  feature works with Nim's containers.
+- The parser now warns about inconsistent spacing around binary operators as
+  these can easily be confused with unary operators. This warning will likely
+  become an error in the future.
+- The ``'c`` and ``'C'`` suffix for octal literals is now deprecated to
+  bring the language in line with the standard library (e.g. ``parseOct``).
+- The dot style for import paths (e.g ``import path.to.module`` instead of
+  ``import path/to/module``) has been deprecated.
+
+#### Breaking changes in the standard library
+
+- ``re.split`` for empty regular expressions now yields every character in
+  the string which is what other programming languages chose to do.
+- The returned tuple of ``system.instantiationInfo`` now has a third field
+  containing the column of the instantiation.
+
+- ``cookies.setCookie`` no longer assumes UTC for the expiration date.
+- ``strutils.formatEng`` does not distinguish between ``nil`` and ``""``
+  strings anymore for its ``unit`` parameter. Instead the space is controlled
+  by a new parameter ``useUnitSpace``.
+
+- The ``times.parse`` and ``times.format`` procs have been rewritten.
+  The proc signatures are the same so it should generally not break anything.
+  However, the new implementation is a bit stricter, which is a breaking change.
+  For example ``parse("2017-01-01 foo", "yyyy-MM-dd")`` will now raise an error.
+
+- ``proc `-`*(a, b: Time): int64`` in the ``times`` module has changed return type
+  to ``times.Duration`` in order to support higher time resolutions.
+  The proc is no longer deprecated.
+
+- The ``times.Timezone`` is now an immutable ref-type that must be initialized
+  with an explicit constructor (``newTimezone``).
+
+- ``posix.Timeval.tv_sec`` has changed type to ``posix.Time``.
+
+- ``math.`mod` `` for floats now behaves the same as ``mod`` for integers
+  (previously it used floor division like Python). Use ``math.floorMod`` for the old behavior.
+
+- For string inputs, ``unicode.isUpper`` and ``unicode.isLower`` now require a
+  second mandatory parameter ``skipNonAlpha``.
+
+- For string inputs, ``strutils.isUpperAscii`` and ``strutils.isLowerAscii`` now
+  require a second mandatory parameter ``skipNonAlpha``.
+
+- ``osLastError`` is now marked with ``sideEffect``
+- The procs ``parseHexInt`` and ``parseOctInt`` now fail on empty strings
+  and strings containing only valid prefixes, e.g. "0x" for hex integers.
+
+- ``terminal.setCursorPos`` and ``terminal.setCursorXPos`` now work correctly
+  with 0-based coordinates on POSIX (previously, you needed to use
+  1-based coordinates on POSIX for correct behaviour; the Windows behaviour
+  was always correct).
+
+- ``lineInfoObj`` now returns absolute path instead of project path.
+  It's used by ``lineInfo``, ``check``, ``expect``, ``require``, etc.
+
+- ``net.sendTo`` no longer returns an int and now raises an ``OSError``.
+- `threadpool`'s `await` and derivatives have been renamed to `blockUntil`
+  to avoid confusions with `await` from the `async` macro.
+
+
+#### Breaking changes in the compiler
+
+- The undocumented ``#? braces`` parsing mode was removed.
+- The undocumented PHP backend was removed.
+- The default location of ``nimcache`` for the native code targets was
+  changed. Read [the compiler user guide](https://nim-lang.org/docs/nimc.html#generated-c-code-directory)
+  for more information.
+
+### Library additions
+
+- ``re.split`` now also supports the ``maxsplit`` parameter for consistency
+  with ``strutils.split``.
+- Added ``system.toOpenArray`` in order to support zero-copy slicing
+  operations. This is currently not yet available for the JavaScript target.
+- Added ``getCurrentDir``, ``findExe``, ``cpDir`` and  ``mvDir`` procs to
+  ``nimscript``.
+- The ``times`` module now supports up to nanosecond time resolution when available.
+- Added the type ``times.Duration`` for representing fixed durations of time.
+- Added the proc ``times.convert`` for converting between different time units,
+  e.g days to seconds.
+- Added the proc ``algorithm.binarySearch[T, K]`` with the ```cmp``` parameter.
+- Added the proc ``algorithm.upperBound``.
+- Added inverse hyperbolic functions, ``math.arcsinh``, ``math.arccosh`` and ``math.arctanh`` procs.
+- Added cotangent, secant and cosecant procs ``math.cot``, ``math.sec`` and ``math.csc``; and their hyperbolic, inverse and inverse hyperbolic functions, ``math.coth``, ``math.sech``, ``math.csch``, ``math.arccot``, ``math.arcsec``, ``math.arccsc``, ``math.arccoth``, ``math.arcsech`` and ``math.arccsch`` procs.
+- Added the procs ``math.floorMod`` and ``math.floorDiv`` for floor based integer division.
+- Added the procs ``rationals.`div```, ``rationals.`mod```, ``rationals.floorDiv`` and ``rationals.floorMod`` for rationals.
+- Added the proc ``math.prod`` for product of elements in openArray.
+- Added the proc ``parseBinInt`` to parse a binary integer from a string, which returns the value.
+- ``parseOct`` and ``parseBin`` in parseutils now also support the ``maxLen`` argument similar to ``parseHexInt``.
+- Added the proc ``flush`` for memory mapped files.
+- Added the ``MemMapFileStream``.
+- Added a simple interpreting event parser template ``eventParser`` to the ``pegs`` module.
+- Added ``macros.copyLineInfo`` to copy lineInfo from other node.
+- Added ``system.ashr`` an arithmetic right shift for integers.
+
+### Library changes
+
+- ``macros.astGenRepr``, ``macros.lispRepr`` and ``macros.treeRepr``
+  now escapes the content of string literals consistently.
+- ``macros.NimSym`` and ``macros.NimIdent`` is now deprecated in favor
+  of the more general ``NimNode``.
+- ``macros.getImpl`` now includes the pragmas of types, instead of omitting them.
+- ``macros.hasCustomPragma`` and ``macros.getCustomPragmaVal`` now
+  also support ``ref`` and ``ptr`` types, pragmas on types and variant
+  fields.
+- ``system.SomeReal`` is now called ``SomeFloat`` for consistency and
+  correctness.
+- ``algorithm.smartBinarySearch`` and ``algorithm.binarySearch`` is
+  now joined in ``binarySearch``. ``smartbinarySearch`` is now
+  deprecated.
+- The `terminal` module now exports additional procs for generating ANSI color
+  codes as strings.
+- Added the parameter ``val`` for the ``CritBitTree[int].inc`` proc.
+- An exception raised from a ``test`` block of ``unittest`` now shows its type in
+  error message.
+- The ``compiler/nimeval`` API was rewritten to simplify the "compiler as an
+  API". Using the Nim compiler and its VM as a scripting engine has never been
+  easier. See ``tests/compilerapi/tcompilerapi.nim`` for an example of how to
+  use the Nim VM in a native Nim application.
+- Added the parameter ``val`` for the ``CritBitTree[T].incl`` proc.
+- The proc ``tgamma`` was renamed to ``gamma``. ``tgamma`` is deprecated.
+- The ``pegs`` module now exports getters for the fields of its ``Peg`` and ``NonTerminal``
+  object types. ``Peg``s with child nodes now have the standard ``items`` and ``pairs``
+  iterators.
+- The ``accept`` socket procedure defined in the ``net`` module can now accept
+  a nil socket.
+
+### Language additions
+
+- Dot calls combined with explicit generic instantiations can now be written
+  as ``x.y[:z]`` which is transformed into ``y[z](x)`` by the parser.
+- ``func`` is now an alias for ``proc {.noSideEffect.}``.
+- In order to make ``for`` loops and iterators more flexible to use Nim now
+  supports so called "for-loop macros". See
+  the [manual](manual.html#macros-for-loop-macros) for more details.
+  This feature enables a Python-like generic ``enumerate`` implementation.
+
+- Case statements can now be rewritten via macros. See the [manual](manual.html#macros-case-statement-macros) for more information.
+  This feature enables custom pattern matchers.
+
+
+- the `typedesc` special type has been renamed to just `type`.
+- `static` and `type` are now also modifiers similar to `ref` and `ptr`.
+  They denote the special types `static[T]` and `type[T]`.
+- Forcing compile-time evaluation with `static` now supports specifying
+  the desired target type (as a concrete type or as a type class)
+- The `type` operator now supports checking that the supplied expression
+  matches an expected type constraint.
+
+### Language changes
+
+- The `importcpp` pragma now allows importing the listed fields of generic
+  C++ types. Support for numeric parameters have also been added through
+  the use of `static[T]` types.
+  (#6415)
+
+- Native C++ exceptions can now be imported with `importcpp` pragma.
+  Imported exceptions can be raised and caught just like Nim exceptions.
+  More details in language manual.
+
+- ``nil`` for strings/seqs is finally gone. Instead the default value for
+  these is ``"" / @[]``. Use ``--nilseqs:on`` for a transition period.
+
+- Accessing the binary zero terminator in Nim's native strings
+  is now invalid. Internally a Nim string still has the trailing zero for
+  zero-copy interoperability with ``cstring``. Compile your code with the
+  new switch ``--laxStrings:on`` if you need a transition period.
+
+- The command syntax now supports keyword arguments after the first comma.
+
+- Thread-local variables can now be declared inside procs. This implies all
+  the effects of the ``global`` pragma.
+
+- Nim now supports the ``except`` clause in the export statement.
+
+- Range float types, example ``range[0.0 .. Inf]``. More details in language manual.
+- The ``{.this.}`` pragma has been deprecated. It never worked within generics and
+  we found the resulting code harder to read than the more explicit ``obj.field``
+  syntax.
+- "Memory regions" for pointer types have been deprecated, they were hardly used
+  anywhere. Note that this has **nothing** to do with the ``--gc:regions`` switch
+  of managing memory.
+
+- The exception hierarchy was slightly reworked, ``SystemError`` was renamed to
+  ``CatchableError`` and is the new base class for any exception that is guaranteed to
+  be catchable. This change should have minimal impact on most existing Nim code.
+
+
+### Tool changes
+
+- ``jsondoc2`` has been renamed ``jsondoc``, similar to how ``doc2`` was renamed
+  ``doc``. The old ``jsondoc`` can still be invoked with ``jsondoc0``.
+
+### Compiler changes
+
+- The VM's instruction count limit was raised to 3 million instructions in
+  order to support more complex computations at compile-time.
+
+- Support for hot code reloading has been implemented for the JavaScript
+  target. To use it, compile your code with `--hotCodeReloading:on` and use a
+  helper library such as LiveReload or BrowserSync.
+
+- A new compiler option `--cppCompileToNamespace` puts the generated C++ code
+  into the namespace "Nim" in order to avoid naming conflicts with existing
+  C++ code. This is done for all Nim code - internal and exported.
+
+- Added ``macros.getProjectPath`` and ``os.putEnv`` procs to Nim's virtual
+  machine.
+
+- The ``deadCodeElim`` option is now always turned on and the switch has no
+  effect anymore, but is recognized for backwards compatibility.
+
+- ``experimental`` is now a pragma / command line switch that can enable specific
+  language extensions, it is not an all-or-nothing switch anymore.
+
+- Nintendo Switch was added as a new platform target. See [the compiler user guide](https://nim-lang.org/docs/nimc.html)
+  for more info.
+
+- macros.bindSym now capable to accepts not only literal string or string constant expression.
+  bindSym enhancement make it also can accepts computed string or ident node inside macros /
+  compile time functions / static blocks. Only in templates / regular code it retains it's old behavior.
+  This new feature can be accessed via {.experimental: "dynamicBindSym".} pragma/switch.
+
+- On Posix systems the global system wide configuration is now put under ``/etc/nim/nim.cfg``,
+  it used to be ``/etc/nim.cfg``. Usually it does not exist, however.
+
+- On Posix systems the user configuration is now looked under ``$XDG_CONFIG_HOME/nim/nim.cfg``
+  (if ``XDG_CONFIG_HOME`` is not defined, then under ``~/.config/nim/nim.cfg``). It used to be
+  ``$XDG_CONFIG_DIR/nim.cfg`` (and ``~/.config/nim.cfg``).
+
+  Similarly, on Windows, the user configuration is now looked under ``%APPDATA%/nim/nim.cfg``.
+  This used to be ``%APPDATA%/nim.cfg``.
+
+### Bugfixes

changelogs/changelog_0_20_0.md

@@ -0,0 +1,294 @@
+## v0.20.0 - 2019-06-06
+
+
+### Changes affecting backwards compatibility
+
+- `shr` is now sign preserving. Use `-d:nimOldShiftRight` to enable
+  the old behavior globally.
+
+- The ``isLower``, ``isUpper`` family of procs in strutils/unicode
+  operating on **strings** have been
+  deprecated since it was unclear what these do. Note that the much more
+  useful procs that operate on ``char`` or ``Rune`` are not affected.
+
+- `strutils.editDistance` has been deprecated,
+  use `editdistance.editDistance` or `editdistance.editDistanceAscii`
+  instead.
+
+- The OpenMP parallel iterator \``||`\` now supports any `#pragma omp directive`
+  and not just `#pragma omp parallel for`. See
+  [OpenMP documentation](https://www.openmp.org/wp-content/uploads/OpenMP-4.5-1115-CPP-web.pdf).
+
+  The default annotation is `parallel for`, if you used OpenMP without annotation
+  the change is transparent, if you used annotations you will have to prefix
+  your previous annotations with `parallel for`.
+
+  Furthermore, an overload with positive stepping is available.
+
+- The `unchecked` pragma was removed, instead use `system.UncheckedArray`.
+
+- The undocumented ``#? strongSpaces`` parsing mode has been removed.
+
+- The `not` operator is now always a unary operator, this means that code like
+  ``assert not isFalse(3)`` compiles.
+
+- `getImpl` on a `var` or `let` symbol will now return the full `IdentDefs`
+  tree from the symbol declaration instead of just the initializer portion.
+
+- Methods are now ordinary "single" methods, only the first parameter is
+  used to select the variant at runtime. For backwards compatibility
+  use the new `--multimethods:on` switch.
+
+- Generic methods are now deprecated; they never worked well.
+
+- Compile time checks for integer and float conversions are now stricter.
+  For example, `const x = uint32(-1)` now gives a compile time error instead
+  of being equivalent to `const x = 0xFFFFFFFF'u32`.
+
+- Using `typed` as the result type in templates/macros now means
+  "expression with a type". The old meaning of `typed` is preserved
+  as `void` or no result type at all.
+
+- A bug allowed `macro foo(): int = 123` to compile even though a
+  macro has to return a `NimNode`. This has been fixed.
+
+- With the exception of `uint` and `uint64`, conversion to unsigned types
+  are now range checked during runtime.
+
+- Macro arguments of type `typedesc` are now passed to the macro as
+  `NimNode` like every other type except `static`. Use `typed` for a
+  behavior that is identical in new and old
+  Nim. See the RFC [Pass typedesc as NimNode to macros](https://github.com/nim-lang/RFCs/issues/148)
+  for more details.
+
+
+#### Breaking changes in the standard library
+
+- `osproc.execProcess` now also takes a `workingDir` parameter.
+
+- `std/sha1.secureHash` now accepts `openArray[char]`, not `string`. (Former
+   successful matches should keep working, though former failures will not.)
+
+- `options.UnpackError` is no longer a ref type and inherits from `system.Defect`
+  instead of `system.ValueError`.
+
+- `system.ValueError` now inherits from `system.CatchableError` instead of `system.Defect`.
+
+- The procs `parseutils.parseBiggestInt`, `parseutils.parseInt`,
+  `parseutils.parseBiggestUInt` and `parseutils.parseUInt` now raise a
+  `ValueError` when the parsed integer is outside of the valid range.
+  Previously they sometimes raised an `OverflowError` and sometimes they
+  returned `0`.
+
+- The procs `parseutils.parseBin`, `parseutils.parseOct` and `parseutils.parseHex`
+  were not clearing their `var` parameter `number` and used to push its value to
+  the left when storing the parsed string into it. Now they always set the value
+  of the parameter to `0` before storing the result of the parsing, unless the
+  string to parse is not valid (then the value of `number` is not changed).
+
+- `streams.StreamObject` now restricts its fields to only raise `system.Defect`,
+  `system.IOError` and `system.OSError`.
+  This change only affects custom stream implementations.
+
+- nre's `RegexMatch.{captureBounds,captures}[]`  no longer return `Option` or
+  `nil`/`""`, respectively. Use the newly added `n in p.captures` method to
+  check if a group is captured, otherwise you'll receive an exception.
+
+- nre's `RegexMatch.{captureBounds,captures}.toTable` no longer accept a
+  default parameter. Instead uncaptured entries are left empty. Use
+  `Table.getOrDefault()` if you need defaults.
+
+- nre's `RegexMatch.captures.{items,toSeq}` now returns an `Option[string]`
+  instead of a `string`. With the removal of `nil` strings, this is the only
+  way to indicate a missing match. Inside your loops, instead
+  of `capture == ""` or `capture == nil`, use `capture.isSome` to check if a capture is
+  present, and `capture.get` to get its value.
+
+- nre's `replace()` no longer throws `ValueError` when the replacement string
+  has missing captures. It instead throws `KeyError` for named captures, and
+  `IndexError` for unnamed captures. This is consistent with
+  `RegexMatch.{captureBounds,captures}[]`.
+
+- `splitFile` now correctly handles edge cases, see #10047.
+
+- `isNil` is no longer false for undefined in the JavaScript backend:
+  now it's true for both nil and undefined.
+  Use `isNull` or `isUndefined` if you need exact equality:
+  `isNil` is consistent with `===`, `isNull` and `isUndefined` with `==`.
+
+- several deprecated modules were removed: `ssl`, `matchers`, `httpserver`,
+  `unsigned`, `actors`, `parseurl`
+
+- two poorly documented and not used modules (`subexes`, `scgi`) were moved to
+  graveyard (they are available as Nimble packages)
+
+- procs `string.add(int)` and `string.add(float)` which implicitly convert
+  ints and floats to string have been deprecated.
+  Use `string.addInt(int)` and `string.addFloat(float)` instead.
+
+- ``case object`` branch transitions via ``system.reset`` are deprecated.
+  Compile your code with ``-d:nimOldCaseObjects`` for a transition period.
+
+- base64 module: The default parameter `newLine` for the `encode` procs
+  was changed from `"\13\10"` to the empty string `""`.
+
+
+#### Breaking changes in the compiler
+
+- The compiler now implements the "generic symbol prepass" for `when` statements
+  in generics, see bug #8603. This means that code like this does not compile
+  anymore:
+
+```nim
+proc enumToString*(enums: openArray[enum]): string =
+  # typo: 'e' instead 'enums'
+  when e.low.ord >= 0 and e.high.ord < 256:
+    result = newString(enums.len)
+  else:
+    result = newString(enums.len * 2)
+```
+
+- ``discard x`` is now illegal when `x` is a function symbol.
+
+- Implicit imports via ``--import: module`` in a config file are now restricted
+  to the main package.
+
+
+### Library additions
+
+- There is a new stdlib module `std/editdistance` as a replacement for the
+  deprecated `strutils.editDistance`.
+
+- There is a new stdlib module `std/wordwrap` as a replacement for the
+  deprecated `strutils.wordwrap`.
+
+- Added `split`, `splitWhitespace`, `size`, `alignLeft`, `align`,
+  `strip`, `repeat` procs and iterators to `unicode.nim`.
+
+- Added `or` for `NimNode` in `macros`.
+
+- Added `system.typeof` for more control over how `type` expressions
+  can be deduced.
+
+- Added `macros.isInstantiationOf` for checking if the proc symbol
+  is instantiation of generic proc symbol.
+
+- Added the parameter ``isSorted`` for the ``sequtils.deduplicate`` proc.
+
+- There is a new stdlib module `std/diff` to compute the famous "diff"
+  of two texts by line.
+
+- Added `os.relativePath`.
+
+- Added `parseopt.remainingArgs`.
+
+- Added `os.getCurrentCompilerExe` (implemented as `getAppFilename` at CT),
+  can be used to retrieve the currently executing compiler.
+
+- Added `xmltree.toXmlAttributes`.
+
+- Added ``std/sums`` module for fast summation functions.
+
+- Added `Rusage`, `getrusage`, `wait4` to the posix interface.
+
+- Added the `posix_utils` module.
+
+- Added `system.default`.
+
+- Added `sequtils.items` for closure iterators, allows closure iterators
+  to be used by the `mapIt`, `filterIt`, `allIt`, `anyIt`, etc.
+
+
+### Library changes
+
+- The string output of `macros.lispRepr` proc has been tweaked
+  slightly. The `dumpLisp` macro in this module now outputs an
+  indented proper Lisp, devoid of commas.
+
+- Added `macros.signatureHash` that returns a stable identifier
+  derived from the signature of a symbol.
+
+- In `strutils` empty strings now no longer matched as substrings
+  anymore.
+
+- The `Complex` type is now a generic object and not a tuple anymore.
+
+- The `ospaths` module is now deprecated, use `os` instead. Note that
+  `os` is available in a NimScript environment but unsupported
+  operations produce a compile-time error.
+
+- The `parseopt` module now supports a new flag `allowWhitespaceAfterColon`
+  (default value: true) that can be set to `false` for better Posix
+  interoperability. (Bug #9619.)
+
+- `os.joinPath` and `os.normalizePath` handle edge cases like ``"a/b/../../.."``
+  differently.
+
+- `securehash` was moved to `lib/deprecated`.
+
+- The switch ``-d:useWinAnsi`` is not supported anymore.
+
+- In `times` module, procs `format` and `parse` accept a new optional
+  `DateTimeLocale` argument for formatting/parsing dates in other languages.
+
+
+### Language additions
+
+- Vm support for float32<->int32 and float64<->int64 casts was added.
+- There is a new pragma block `noSideEffect` that works like
+  the `gcsafe` pragma block.
+- added `os.getCurrentProcessId`.
+- User defined pragmas are now allowed in the pragma blocks.
+- Pragma blocks are no longer eliminated from the typed AST tree to preserve
+  pragmas for further analysis by macros.
+- Custom pragmas are now supported for `var` and `let` symbols.
+- Tuple unpacking is now supported for constants and for loop variables.
+- Case object branches can be initialized with a runtime discriminator if
+  possible discriminator values are constrained within a case statement.
+
+### Language changes
+
+- The standard extension for SCF (source code filters) files was changed from
+  ``.tmpl`` to ``.nimf``,
+  it's more recognizable and allows tools like Github to recognize it as Nim,
+  see [#9647](https://github.com/nim-lang/Nim/issues/9647).
+  The previous extension will continue to work.
+
+- Pragma syntax is now consistent. Previous syntax where type pragmas did not
+  follow the type name is now deprecated. Also pragma before generic parameter
+  list is deprecated to be consistent with how pragmas are used with a proc. See
+  [#8514](https://github.com/nim-lang/Nim/issues/8514) and
+  [#1872](https://github.com/nim-lang/Nim/issues/1872) for further details.
+
+- Hash sets and tables are initialized by default. The explicit `initHashSet`,
+  `initTable`, etc. are not needed anymore.
+
+
+### Tool changes
+
+- `jsondoc` now includes a `moduleDescription` field with the module
+  description. `jsondoc0` shows comments as its own objects as shown in the
+  documentation.
+- `nimpretty`: --backup now defaults to `off` instead of `on` and the flag was
+  undocumented; use `git` instead of relying on backup files.
+- `koch` now defaults to build the latest *stable* Nimble version unless you
+  explicitly ask for the latest master version via `--latest`.
+
+
+### Compiler changes
+
+- The deprecated `fmod` proc is now unavailable on the VM.
+- A new `--outdir` option was added.
+- The compiled JavaScript file for the project produced by executing `nim js`
+  will no longer be placed in the nimcache directory.
+- The `--hotCodeReloading` has been implemented for the native targets.
+  The compiler also provides a new more flexible API for handling the
+  hot code reloading events in the code.
+- The compiler now supports a ``--expandMacro:macroNameHere`` switch
+  for easy introspection into what a macro expands into.
+- The `-d:release` switch now does not disable runtime checks anymore.
+  For a release build that also disables runtime checks
+  use `-d:release -d:danger` or simply `-d:danger`.
+
+
+### Bugfixes

changelogs/changelog_0_20_2.md

@@ -0,0 +1,55 @@
+# v0.20.2 - 2019-07-17
+
+
+## Changes affecting backwards compatibility
+
+- All `strutils.rfind` procs now take `start` and `last` like `strutils.find`
+  with the same data slice/index meaning. This is backwards compatible for
+  calls *not* changing the `rfind` `start` parameter from its default. (#11487)
+
+  In the unlikely case that you were using `rfind X, start=N`, or `rfind X, N`,
+  then you need to change that to `rfind X, last=N` or `rfind X, 0, N`. (This
+  should minimize gotchas porting code from other languages like Python or C++.)
+
+- On Windows stderr/stdout/stdin are not opened as binary files anymore. Use the switch
+  `-d:nimBinaryStdFiles` for a transition period.
+
+### Breaking changes in the standard library
+
+- Mac OS X / BSD: TSa_Family is now the ``uint8`` type, so type
+  conversions like ``x.sin_family = uint16 toInt(nativesockets.AF_INET)``
+  need to be changed into ``x.sin_family = TSa_Family toInt(nativesockets.AF_INET)``.
+
+
+### Breaking changes in the compiler
+
+
+## Library additions
+
+- `toOpenArray` is now available for the JS target.
+
+## Library changes
+
+- Fix async IO operations stalling even after socket is closed. (#11232)
+
+- More informative error message for `streams.openFileStream`. (#11438)
+
+
+## Language additions
+
+
+## Language changes
+
+
+### Tool changes
+
+
+### Compiler changes
+
+- Better error message for IndexError for empty containers. (#11476)
+
+- Fix regression in semfold for old right shift. (#11477)
+
+- Fix for passing tuples as static params to macros. (#11423)
+
+## Bugfixes

changelogs/changelog_1_0_0.md

@@ -0,0 +1,138 @@
+# v1.0 - 2019-09-23
+
+
+## Changes affecting backwards compatibility
+
+- The switch ``-d:nimBinaryStdFiles`` does not exist anymore. Instead
+  stdin/stdout/stderr are binary files again. This change only affects
+  Windows.
+
+- On Windows console applications the code-page is set at program startup
+  to UTF-8. Use the new switch `-d:nimDontSetUtf8CodePage` to disable this
+  feature.
+
+- The language definition and compiler are now stricter about ``gensym``'ed
+  symbols in hygienic templates. See the section in the
+  [manual](https://nim-lang.org/docs/manual.html#templates-hygiene-in-templates)
+  for further details. Use the compiler switch `--oldgensym:on` for a
+  transition period.
+
+
+### Breaking changes in the standard library
+
+- We removed `unicode.Rune16` without any deprecation period as the name
+  was wrong (see the [RFC](https://github.com/nim-lang/RFCs/issues/151) for details)
+  and we didn't find any usages of it in the wild. If you still need it, add this
+  piece of code to your project:
+```nim
+type
+  Rune16* = distinct int16
+```
+
+- `exportc` now uses C instead of C++ mangling with `nim cpp`, matching behavior
+  of `importc`, see #10578.
+  Use the new `exportcpp` to mangle as C++ when using `nim cpp`.
+
+
+### Breaking changes in the compiler
+
+- A bug allowing `int` to be implicitly converted to range types of smaller size
+  (e.g `range[0'i8..10'i8]`) has been fixed.
+
+
+
+## Library additions
+
+- `encodings.getCurrentEncoding` now distinguishes between the console's
+  encoding and the OS's encoding. This distinction is only meaningful on
+  Windows.
+- Added `system.getOsFileHandle` which is usually more useful
+  than `system.getFileHandle`. This distinction is only meaningful on
+  Windows.
+- Added a `json.parseJsonFragments` iterator that can be used to speedup
+  JSON processing substantially when there are JSON fragments separated
+  by whitespace.
+
+
+
+## Library changes
+
+- Added `os.delEnv` and `nimscript.delEnv`. (#11466)
+
+- Enabled Oid usage in hashtables. (#11472)
+
+- Added `unsafeColumnAt` procs, that return unsafe cstring from InstantRow. (#11647)
+
+- Make public `Sha1Digest` and `Sha1State` types and `newSha1State`,
+  `update` and `finalize` procedures from `sha1` module. (#11694)
+
+- Added the `std/monotimes` module which implements monotonic timestamps.
+
+- Consistent error handling of two `exec` overloads. (#10967)
+
+- Officially the following modules now have an unstable API:
+  - std/varints
+  - core/allocators
+  - core/hotcodereloading
+  - asyncstreams
+  - base64
+  - browsers
+  - collections/rtarrays
+  - collections/sharedlist
+  - collections/sharedtable
+  - concurrency/atomics
+  - concurrency/cpuload
+  - concurrency/threadpool
+  - coro
+  - endians
+  - httpcore
+  - parsesql
+  - pathnorm
+  - reservedmem
+  - typetraits
+
+  Every other stdlib module is API stable with respect to version 1.
+
+
+
+## Language additions
+
+- Inline iterators returning `lent T` types are now supported, similarly to
+  iterators returning `var T`:
+```nim
+iterator myitems[T](x: openarray[T]): lent T
+iterator mypairs[T](x: openarray[T]): tuple[idx: int, val: lent T]
+```
+
+- Added an `importjs` pragma that can now be used instead of `importcpp`
+  and `importc` to import symbols from JavaScript. `importjs` for routines always
+  takes a "pattern" for maximum flexibility.
+
+
+
+## Language changes
+
+- `uint64` is now finally a regular ordinal type. This means `high(uint64)` compiles
+  and yields the correct value.
+
+
+### Tool changes
+
+- The Nim compiler now does not recompile the Nim project via ``nim c -r`` if
+  no dependent Nim file changed. This feature can be overridden by
+  the ``--forceBuild`` command line option.
+- The Nim compiler now warns about unused module imports. You can use a
+  top level ``{.used.}`` pragma in the module that you want to be importable
+  without producing this warning.
+- The "testament" testing tool's name was changed
+  from `tester` to `testament` and is generally available as a tool to run Nim
+  tests automatically.
+
+
+### Compiler changes
+
+- VM can now cast integer type arbitrarily. (#11459)
+
+
+
+## Bugfixes

changelogs/changelog_1_0_2.md

@@ -0,0 +1,56 @@
+# v1.0.2 - 2019-10-23
+
+
+## Bugfixes
+
+* fixes the --verbosity:2 regression
+* Fixed "Fail to compile a file twice under Windows (v1.0 bug)." [#12242](https://github.com/nim-lang/Nim/issues/12242)
+* fix nimpretty removing space before pragma
+* JS: gensym is stricter for 'this'
+* Fixed "VM Assertion Error with newruntime" [#12294](https://github.com/nim-lang/Nim/issues/12294)
+* Fixed "Assertion error when running `nim check` on compiler/nim.nim" [#12281](https://github.com/nim-lang/Nim/issues/12281)
+* Fixed "Compiler crash with empty array and generic instantiation with int as parameter" [#12264](https://github.com/nim-lang/Nim/issues/12264)
+* Fixed "Regression in JS backend codegen "Error: request to generate code for .compileTime proc"" [#12240](https://github.com/nim-lang/Nim/issues/12240)
+* Fix how `relativePath` handle case sensitiviy
+* Fixed "SIGSEGV in compiler when using generic types and seqs" [#12336](https://github.com/nim-lang/Nim/issues/12336)
+* Fixed "[1.0.0] weird interaction between `import os` and casting integer to char on macosx trigger bad codegen" [#12291](https://github.com/nim-lang/Nim/issues/12291)
+* VM: no special casing for big endian machines
+* Fixed "`internal error: environment misses` with a simple template inside one of Jester macros" [#12323](https://github.com/nim-lang/Nim/issues/12323)
+* nimsuggest: fix tcp socket leak
+* nimsuggest: fix tcp socket leak for epc backend
+* Fixed "`writeFile` and `write(f, str)` skip null bytes on Windows" [#12315](https://github.com/nim-lang/Nim/issues/12315)
+* Fixed "Crash in intsets symmetric_difference" [#12366](https://github.com/nim-lang/Nim/issues/12366)
+* Fixed "[regression] VM crash when dealing with var param of a proc result" [#12244](https://github.com/nim-lang/Nim/issues/12244)
+* fixes a koch regression that made 'koch boot --listcmd' not work anymore
+* Fixed "[regression] inconsistent signed int `mod` operator between runtime, compiletime, and semfold" [#12332](https://github.com/nim-lang/Nim/issues/12332)
+* Fixed "Boehm disables interior pointer checking" [#12286](https://github.com/nim-lang/Nim/issues/12286)
+* Fixes semCustomPragma when nkSym
+* Fixed yield in nkCheckedFieldExpr
+* Fixed "`randomize()` from `random` not working on JS" [#12418](https://github.com/nim-lang/Nim/issues/12418)
+* Fixed "Compiler crash with invalid object variant" [#12379](https://github.com/nim-lang/Nim/issues/12379)
+* fix type's case in random.nim
+* Fixed "Update docs with a better way to signal unimplemented methods" [#10804](https://github.com/nim-lang/Nim/issues/10804)
+* Fixed "Nim language manual, push pragma is not explained well" [#10824](https://github.com/nim-lang/Nim/issues/10824)
+* Fixed "[regression] Importing more than one module with same name from different packages produce bad codegen" [#12420](https://github.com/nim-lang/Nim/issues/12420)
+* Namespace unittest enums to avoid name conflicts
+* Fixed "VM checks unsigned integers for overflow." [#12310](https://github.com/nim-lang/Nim/issues/12310)
+* Fixed "line directive is not generated for first line of function definition" [#12426](https://github.com/nim-lang/Nim/issues/12426)
+
+
+
+## Documentation improvements
+
+* threadpool: fix link in docs (#12258)
+* Fix spellings (#12277)
+* fix #12278, don't expose internal PCRE documentation
+* Fixed "Documentation of quitprocs is wrong" [#12279(https://github.com/nim-lang/Nim/issues/12279)
+* Fix typo in docs
+* Fix reference to parseSpec proc in readme
+* [doc/tut1] removed discard discussion in comments
+* Documentation improvements around the db interface 
+* Easier build instructions for windows - just run `build_all.bat`.
+* fix a few dead links and a missing sentence in documentation
+* Macro docs additions
+* Updated the code example in the os module to use better grammar.
+* Mention "lambdas" and `=>` in the manual
+* Better documentation on Garbage Collector

changelogs/changelog_1_2_0.md

@@ -0,0 +1,532 @@
+# v1.2.0 - 2020-04-02
+
+
+## Standard library additions and changes
+
+- Added overloaded `strformat.fmt` macro that use specified characters as
+  delimiter instead of '{' and '}'.
+- Added new procs in `tables.nim`: `OrderedTable.pop`, `CountTable.del`,
+  `CountTable.pop`, `Table.pop`.
+- Added `strtabs.clear` overload that reuses the existing mode.
+- Added `browsers.osOpen` const alias for the operating system specific *"open"* command.
+- Added `sugar.dup` for turning in-place algorithms like `sort` and `shuffle`
+  into operations that work on a copy of the data and return the mutated copy,
+  like the existing `sorted` does.
+- Added `sugar.collect` that does comprehension for seq/set/table collections.
+- Added `sugar.capture` for capturing some local loop variables when creating a
+  closure. This is an enhanced version of `closureScope`.
+- Added `typetraits.tupleLen` to get number of elements of a tuple/type tuple,
+  and `typetraits.get` to get the ith element of a type tuple.
+- Added `typetraits.genericParams` to return a tuple of generic params from a
+  generic instantiation.
+- `options` now treats `proc` like other pointer types, meaning `nil` proc variables
+  are converted to `None`.
+- Added `os.normalizePathEnd` for additional path sanitization.
+- Added `times.fromUnixFloat,toUnixFloat`, sub-second resolution versions of
+  `fromUnix`,`toUnixFloat`.
+- Added `wrapnils` module for chains of field-access and indexing where the LHS
+  can be nil. This simplifies code by reducing need for if-else branches around
+  intermediate maybe nil values. E.g. `echo ?.n.typ.kind`.
+- Added `minIndex`, `maxIndex` and `unzip` to the `sequtils` module.
+- Added `os.isRelativeTo` to tell whether a path is relative to another.
+- Added `resetOutputFormatters` to `unittest`.
+- Added `expectIdent` to the `macros` module.
+- Added `os.isValidFilename` that returns `true` if `filename` argument is valid
+  for cross-platform use.
+- Added `times.isLeapDay`
+- `base64` adds URL-Safe Base64, implements RFC-4648 Section-7.
+- Added a new module, `std / compilesettings` for querying the compiler about
+  diverse configuration settings.
+- Added `net.getPeerCertificates` and `asyncnet.getPeerCertificates` for
+  retrieving the verified certificate chain of the peer we are connected to
+  through an SSL-wrapped `Socket`/`AsyncSocket`.
+- Added `browsers.openDefaultBrowser` without URL, implements IETF RFC-6694 Section-3.
+- Added `jsconsole.trace`, `jsconsole.table`, `jsconsole.exception` for JavaScript target.
+- Added `distinctBase` overload for values: `assert 12.MyInt.distinctBase == 12`
+- Added new module `std/stackframes`, in particular `setFrameMsg`, which enables
+  custom runtime annotation of stackframes, see #13351 for examples.
+  Turn on/off via `--stackTraceMsgs:on/off`.
+- Added `sequtils.countIt`, allowing for counting items using a predicate.
+- Added a `with` macro for easy function chaining that's available everywhere,
+  there is no need to concern your APIs with returning the first argument
+  to enable "chaining", instead use the dedicated macro `with` that
+  was designed for it. For example:
+
+```nim
+type
+  Foo = object
+    col, pos: string
+
+proc setColor(f: var Foo; r, g, b: int) = f.col = $(r, g, b)
+proc setPosition(f: var Foo; x, y: float) = f.pos = $(x, y)
+
+var f: Foo
+with(f, setColor(2, 3, 4), setPosition(0.0, 1.0))
+echo f
+```
+
+- `macros.newLit` now works for ref object types.
+- `macro pragmas` can now be used in type sections.
+- 5 new pragmas were added to Nim in order to make the upcoming tooling more
+  convenient to use. Nim compiler checks these pragmas for syntax but otherwise
+  ignores them. The pragmas are `requires`, `ensures`, `assume`, `assert`, `invariant`.
+- `system.writeFile` has been overloaded to also support `openarray[byte]`.
+- `asyncdispatch.drain` now properly takes into account `selector.hasPendingOperations`
+  and only returns once all pending async operations are guaranteed to have completed.
+- `sequtils.zip` now returns a sequence of anonymous tuples i.e. those tuples
+  now do not have fields named "a" and "b".
+- `distinctBase` has been moved from `sugar` to `typetraits` and now it is
+  implemented as compiler type trait instead of macro. `distinctBase` in sugar
+  module is now deprecated.
+- `CountTable.mget` has been removed from `tables.nim`. It didn't work, and it
+  was an oversight to be included in v1.0.
+- `tables.merge(CountTable, CountTable): CountTable` has been removed.
+  It didn't work well together with the existing inplace version of the same proc
+  (`tables.merge(var CountTable, CountTable)`).
+  It was an oversight to be included in v1.0.
+- `asyncdispatch.drain` now consistently uses the passed timeout value for all
+  iterations of the event loop, and not just the first iteration.
+  This is more consistent with the other asyncdispatch APIs, and allows
+  `asyncdispatch.drain` to be more efficient.
+- `base64.encode` and `base64.decode` were made faster by about 50%.
+- `htmlgen` adds [MathML](https://wikipedia.org/wiki/MathML) support
+  (ISO 40314).
+- `macros.eqIdent` is now invariant to export markers and backtick quotes.
+- `htmlgen.html` allows `lang` in the `<html>` tag and common valid attributes.
+- `macros.basename` and `basename=` got support for `PragmaExpr`,
+  so that an expression like `MyEnum {.pure.}` is handled correctly.
+- `httpclient.maxredirects` changed from `int` to `Natural`, because negative values
+  serve no purpose whatsoever.
+- `httpclient.newHttpClient` and `httpclient.newAsyncHttpClient` added `headers`
+  argument to set initial HTTP Headers, instead of a hardcoded empty `newHttpHeader()`.
+- `parseutils.parseUntil` has now a different behaviour if the `until` parameter is
+  empty. This was required for intuitive behaviour of the strscans module
+  (see bug #13605).
+- `strutils.formatFloat` with `precision = 0` has the same behavior in all
+  backends, and it is compatible with Python's behavior,
+  e.g. `formatFloat(3.14159, precision = 0)` is now `3`, not `3.`.
+- `times.parse` now only uses input to compute its result, and not `now`:
+  `parse("2020", "YYYY", utc())` is now `2020-01-01T00:00:00Z` instead of
+  `2020-03-02T00:00:00Z` if run on 03-02; it also doesn't crash anymore when
+  used on 29th, 30th, 31st of each month.
+- `httpcore.==(string, HttpCode)` is now deprecated due to lack of practical
+  usage. The `$` operator can be used to obtain the string form of `HttpCode`
+  for comparison if desired.
+- `std/oswalkdir` was buggy, it's now deprecated and reuses `std/os` procs.
+- `net.newContext` now performs SSL Certificate checking on Linux and OSX.
+  Define `nimDisableCertificateValidation` to disable it globally.
+- `os.walkDir` and `os.walkDirRec` now have new flag, `checkDir` (default: false).
+  If it is set to true, it will throw if input dir is invalid instead of a noop
+  (which is the default behaviour, as it was before this change),
+  `os.walkDirRec` only throws if top-level dir is invalid, but ignores errors for
+  subdirs, otherwise it would be impossible to resume iteration.
+- The `FD` variant of `selector.unregister` for `ioselector_epoll` and
+  `ioselector_select` now properly handle the `Event.User` select event type.
+- `joinPath` path normalization when `/` is the first argument works correctly:
+  `assert "/" / "/a" == "/a"`. Fixed the edge case: `assert "" / "" == ""`.
+- `xmltree` now adds indentation consistently to child nodes for any number
+  of children nodes.
+- `os.splitPath()` behavior synchronized with `os.splitFile()` to return "/"
+  as the dir component of `/root_sub_dir` instead of the empty string.
+- The deprecated `lc` macro has been removed from `sugar`. It is now replaced with the
+  more powerful `collect` macro.
+- `os.relativePath("foo", "foo")` is now `"."`, not `""`, as `""` means invalid
+  path and shouldn't be conflated with `"."`; use `-d:nimOldRelativePathBehavior`
+  to restore the old behavior.
+- `os.joinPath(a, b)` now honors trailing slashes in `b` (or `a` if `b` = "").
+- `base64.encode` no longer supports `lineLen` and `newLine`.
+  Use `base64.encodeMime` instead.
+
+
+## Language changes
+
+- An `align` pragma can now be used for variables and object fields, similar
+  to the `alignas` declaration modifier in C/C++.
+- `=sink` type bound operator is now optional. Compiler can now use combination
+  of `=destroy` and `copyMem` to move objects efficiently.
+- Unsigned integer operators have been fixed to allow promotion of the first operand.
+- Conversions to unsigned integers are unchecked at runtime, imitating earlier Nim
+  versions. The documentation was improved to acknowledge this special case.
+  See https://github.com/nim-lang/RFCs/issues/175 for more details.
+- New syntax for lvalue references: `var b {.byaddr.} = expr` enabled by
+  `import std/decls`.
+- `var a {.foo.}: MyType = expr` now lowers to `foo(a, MyType, expr)` for
+  non-builtin pragmas, enabling things like lvalue references (see `decls.byaddr`).
+
+
+## Compiler changes
+
+- Generated JS code uses spaces, instead of mixed spaces and tabs.
+- The Nim compiler now supports the ``--asm`` command option for easier
+  inspection of the produced assembler code.
+- The Nim compiler now supports a new pragma called ``.localPassc`` to
+  pass specific compiler options to the C(++) backend for the C(++) file
+  that was produced from the current Nim module.
+- The compiler now inferes "sink parameters". To disable this for a specific routine,
+  annotate it with `.nosinks`. To disable it for a section of code, use
+  `{.push sinkInference: off.}`...`{.pop.}`.
+- The compiler now supports a new switch `--panics:on` that turns runtime
+  errors like `IndexError` or `OverflowError` into fatal errors that **cannot**
+  be caught via Nim's `try` statement. `--panics:on` can improve the
+  runtime efficiency and code size of your program significantly.
+- The compiler now warns about inheriting directly from `system.Exception` as
+  this is **very bad** style. You should inherit from `ValueError`, `IOError`,
+  `OSError` or from a different specific exception type that inherits from
+  `CatchableError` and cannot be confused with a `Defect`.
+- The error reporting for Nim's effect system has been improved.
+- Implicit conversions for `const` behave correctly now, meaning that code like
+  `const SOMECONST = 0.int; procThatTakesInt32(SOMECONST)` will be illegal now.
+  Simply write `const SOMECONST = 0` instead.
+- The `{.dynlib.}` pragma is now required for exporting symbols when making
+  shared objects on POSIX and macOS, which make it consistent with the behavior
+  on Windows.
+- The compiler is now more strict about type conversions concerning proc
+  types: Type conversions cannot be used to hide `.raise` effects or side
+  effects, instead a `cast` must be used. With the flag `--useVersion:1.0` the
+  old behaviour is emulated.
+- The Nim compiler now implements a faster way to detect overflows based
+  on GCC's `__builtin_sadd_overflow` family of functions. (Clang also
+  supports these). Some versions of GCC lack this feature and unfortunately
+  we cannot detect this case reliably. So if you get compilation errors like
+  "undefined reference to `__builtin_saddll_overflow`" compile your programs
+  with `-d:nimEmulateOverflowChecks`.
+
+
+## Tool changes
+
+- Nimpretty doesn't accept negative indentation argument anymore, because it was
+  breaking files.
+
+
+## Bugfixes
+
+- Fixed "`nimgrep --nocolor` is ignored on posix; should be instead: `--nimgrep --color=[auto]|true|false`"
+  ([#7591](https://github.com/nim-lang/Nim/issues/7591))
+- Fixed "Runtime index on const array (of converted obj) causes C-compiler error"
+  ([#10514](https://github.com/nim-lang/Nim/issues/10514))
+- Fixed "windows x86 with vcc compile error with "asmNoStackFrame""
+  ([#12298](https://github.com/nim-lang/Nim/issues/12298))
+- Fixed "[TODO] regression: Error: Locks requires --threads:on option"
+  ([#12330](https://github.com/nim-lang/Nim/issues/12330))
+- Fixed "Add --cc option to --help or --fullhelp output"
+  ([#12010](https://github.com/nim-lang/Nim/issues/12010))
+- Fixed "questionable `csize` definition in `system.nim`"
+  ([#12187](https://github.com/nim-lang/Nim/issues/12187))
+- Fixed "os.getAppFilename() returns incorrect results on OpenBSD"
+  ([#12389](https://github.com/nim-lang/Nim/issues/12389))
+- Fixed "HashSet[uint64] slow insertion depending on values"
+  ([#11764](https://github.com/nim-lang/Nim/issues/11764))
+- Fixed "Differences between compiling 'classic call syntax' vs 'method call syntax' ."
+  ([#12453](https://github.com/nim-lang/Nim/issues/12453))
+- Fixed "c -d:nodejs --> SIGSEGV: Illegal storage access"
+  ([#12502](https://github.com/nim-lang/Nim/issues/12502))
+- Fixed "Closure iterator crashes on --newruntime due to "dangling references""
+  ([#12443](https://github.com/nim-lang/Nim/issues/12443))
+- Fixed "No `=destroy` for elements of closure environments other than for latest devel --gc:destructors"
+  ([#12577](https://github.com/nim-lang/Nim/issues/12577))
+- Fixed "strutils:formatBiggestFloat() gives different results in JS mode"
+  ([#8242](https://github.com/nim-lang/Nim/issues/8242))
+- Fixed "Regression (devel): the new `csize_t` definition isn't consistently used, nor tested thoroughly..."
+  ([#12597](https://github.com/nim-lang/Nim/issues/12597))
+- Fixed "tables.take() is defined only for `Table` and missed for other table containers"
+  ([#12519](https://github.com/nim-lang/Nim/issues/12519))
+- Fixed "`pthread_key_t` errors on OpenBSD"
+  ([#12135](https://github.com/nim-lang/Nim/issues/12135))
+- Fixed "newruntime: simple seq pop at ct results in compile error"
+  ([#12644](https://github.com/nim-lang/Nim/issues/12644))
+- Fixed "[Windows] finish.exe C:\Users\<USERNAME>\.nimble\bin is not in your PATH environment variable."
+  ([#12319](https://github.com/nim-lang/Nim/issues/12319))
+- Fixed "Error with strformat + asyncdispatch + const"
+  ([#12612](https://github.com/nim-lang/Nim/issues/12612))
+- Fixed "MultipartData needs $"
+  ([#11863](https://github.com/nim-lang/Nim/issues/11863))
+- Fixed "Nim stdlib style issues with --styleCheck:error"
+  ([#12687](https://github.com/nim-lang/Nim/issues/12687))
+- Fixed "new $nimbleDir path substitution yields unexpected search paths"
+  ([#12767](https://github.com/nim-lang/Nim/issues/12767))
+- Fixed "Regression: inlined procs now get multiple rounds of destructor injection"
+  ([#12766](https://github.com/nim-lang/Nim/issues/12766))
+- Fixed "newruntime: compiler generates defective code"
+  ([#12669](https://github.com/nim-lang/Nim/issues/12669))
+- Fixed "broken windows modules path handling because of 'os.relativePath' breaking changes"
+  ([#12734](https://github.com/nim-lang/Nim/issues/12734))
+- Fixed "for loop tuple syntax not rendered correctly"
+  ([#12740](https://github.com/nim-lang/Nim/issues/12740))
+- Fixed "Crash when trying to use `type.name[0]`"
+  ([#12804](https://github.com/nim-lang/Nim/issues/12804))
+- Fixed "Enums should be considered Trivial types in Atomics"
+  ([#12812](https://github.com/nim-lang/Nim/issues/12812))
+- Fixed "Produce static/const initializations for variables when possible"
+  ([#12216](https://github.com/nim-lang/Nim/issues/12216))
+- Fixed "Assigning descriminator field leads to internal assert with --gc:destructors"
+  ([#12821](https://github.com/nim-lang/Nim/issues/12821))
+- Fixed "nimsuggest `use` command does not return all instances of symbol"
+  ([#12832](https://github.com/nim-lang/Nim/issues/12832))
+- Fixed "@[] is a problem for --gc:destructors"
+  ([#12820](https://github.com/nim-lang/Nim/issues/12820))
+- Fixed "Codegen ICE in allPathsAsgnResult"
+  ([#12827](https://github.com/nim-lang/Nim/issues/12827))
+- Fixed "seq[Object with ref and destructor type] doesn't work in old runtime"
+  ([#12882](https://github.com/nim-lang/Nim/issues/12882))
+- Fixed "Destructor not invoked because it is instantiated too late, old runtime"
+  ([#12883](https://github.com/nim-lang/Nim/issues/12883))
+- Fixed "The collect macro does not handle if/case correctly"
+  ([#12874](https://github.com/nim-lang/Nim/issues/12874))
+- Fixed "allow typed/untyped params in magic procs (even if not in stdlib)"
+  ([#12911](https://github.com/nim-lang/Nim/issues/12911))
+- Fixed "ARC/newruntime memory corruption"
+  ([#12899](https://github.com/nim-lang/Nim/issues/12899))
+- Fixed "tasyncclosestall.nim still flaky test: Address already in use"
+  ([#12919](https://github.com/nim-lang/Nim/issues/12919))
+- Fixed "newruntime and computed goto: variables inside the loop are in generated code uninitialised"
+  ([#12785](https://github.com/nim-lang/Nim/issues/12785))
+- Fixed "osx: dsymutil needs to be called for debug builds to keep debug info"
+  ([#12735](https://github.com/nim-lang/Nim/issues/12735))
+- Fixed "codegen ICE with ref objects, gc:destructors"
+  ([#12826](https://github.com/nim-lang/Nim/issues/12826))
+- Fixed "mutable iterator cannot yield named tuples"
+  ([#12945](https://github.com/nim-lang/Nim/issues/12945))
+- Fixed "parsecfg stores "\r\n" line breaks just as "\n""
+  ([#12970](https://github.com/nim-lang/Nim/issues/12970))
+- Fixed "db_postgres.getValue issues warning when no rows found"
+  ([#12973](https://github.com/nim-lang/Nim/issues/12973))
+- Fixed "ARC: Unpacking tuple with seq causes segfault"
+  ([#12989](https://github.com/nim-lang/Nim/issues/12989))
+- Fixed "ARC/newruntime: strutils.join on seq with only empty strings causes segfault"
+  ([#12965](https://github.com/nim-lang/Nim/issues/12965))
+- Fixed "regression (1.0.4): `{.push exportc.}` wrongly affects generic instantiations, causing codegen errors"
+  ([#12985](https://github.com/nim-lang/Nim/issues/12985))
+- Fixed "cdt, crash with --gc:arc, works fine with default gc"
+  ([#12978](https://github.com/nim-lang/Nim/issues/12978))
+- Fixed "ARC: No indexError thrown on out-of-bound seq access, SIGSEGV instead"
+  ([#12961](https://github.com/nim-lang/Nim/issues/12961))
+- Fixed "ARC/async: Returning in a try-block results in wrong codegen"
+  ([#12956](https://github.com/nim-lang/Nim/issues/12956))
+- Fixed "asm keyword is generating wrong output C code when --cc:tcc"
+  ([#12988](https://github.com/nim-lang/Nim/issues/12988))
+- Fixed "Destructor not invoked"
+  ([#13026](https://github.com/nim-lang/Nim/issues/13026))
+- Fixed "ARC/newruntime: Adding inherited var ref object to seq with base type causes segfault"
+  ([#12964](https://github.com/nim-lang/Nim/issues/12964))
+- Fixed "Style check error with JS compiler target"
+  ([#13032](https://github.com/nim-lang/Nim/issues/13032))
+- Fixed "regression(1.0.4): undeclared identifier: 'readLines'; plus another regression and bug"
+  ([#13013](https://github.com/nim-lang/Nim/issues/13013))
+- Fixed "regression(1.04) `invalid pragma: since` with nim js"
+  ([#12996](https://github.com/nim-lang/Nim/issues/12996))
+- Fixed "Sink to MemMove optimization in injectdestructors"
+  ([#13002](https://github.com/nim-lang/Nim/issues/13002))
+- Fixed "--gc:arc: `catch` doesn't work with exception subclassing"
+  ([#13072](https://github.com/nim-lang/Nim/issues/13072))
+- Fixed "nim c --gc:arc --exceptions:{setjmp,goto} incorrectly handles raise; `nim cpp --gc:arc` is ok"
+  ([#13070](https://github.com/nim-lang/Nim/issues/13070))
+- Fixed "typetraits feature request - get subtype of a generic type"
+  ([#6454](https://github.com/nim-lang/Nim/issues/6454))
+- Fixed "CountTable inconsistencies between keys() and len() after setting value to 0"
+  ([#12813](https://github.com/nim-lang/Nim/issues/12813))
+- Fixed "{.align.} pragma is not applied if there is a generic field"
+  ([#13122](https://github.com/nim-lang/Nim/issues/13122))
+- Fixed "ARC, finalizer, allow rebinding the same function multiple times"
+  ([#13112](https://github.com/nim-lang/Nim/issues/13112))
+- Fixed "`nim doc` treats `export localSymbol` incorrectly"
+  ([#13100](https://github.com/nim-lang/Nim/issues/13100))
+- Fixed "--gc:arc SIGSEGV (double free?)"
+  ([#13119](https://github.com/nim-lang/Nim/issues/13119))
+- Fixed "codegen bug with arc"
+  ([#13105](https://github.com/nim-lang/Nim/issues/13105))
+- Fixed "symbols not defined in the grammar"
+  ([#10665](https://github.com/nim-lang/Nim/issues/10665))
+- Fixed "[JS] Move is not defined"
+  ([#9674](https://github.com/nim-lang/Nim/issues/9674))
+- Fixed "[TODO] pathutils.`/` can return invalid AbsoluteFile"
+  ([#13121](https://github.com/nim-lang/Nim/issues/13121))
+- Fixed "regression(1.04) `nim doc main.nim` generates broken html (no css)"
+  ([#12998](https://github.com/nim-lang/Nim/issues/12998))
+- Fixed "Wrong supportsCopyMem on string in type section"
+  ([#13095](https://github.com/nim-lang/Nim/issues/13095))
+- Fixed "Arc, finalizer, out of memory"
+  ([#13157](https://github.com/nim-lang/Nim/issues/13157))
+- Fixed "`--genscript` messes up nimcache and future nim invocations"
+  ([#13144](https://github.com/nim-lang/Nim/issues/13144))
+- Fixed "--gc:arc with --exceptions:goto for "nim c" generate invalid c code"
+  ([#13186](https://github.com/nim-lang/Nim/issues/13186))
+- Fixed "[regression] duplicate member `_i1` codegen bug"
+  ([#13195](https://github.com/nim-lang/Nim/issues/13195))
+- Fixed "RTree investigations with valgrind for --gc:arc"
+  ([#13110](https://github.com/nim-lang/Nim/issues/13110))
+- Fixed "relativePath("foo", ".") returns wrong path"
+  ([#13211](https://github.com/nim-lang/Nim/issues/13211))
+- Fixed "asyncftpclient - problem with welcome.msg"
+  ([#4684](https://github.com/nim-lang/Nim/issues/4684))
+- Fixed "Unclear error message, lowest priority"
+  ([#13256](https://github.com/nim-lang/Nim/issues/13256))
+- Fixed "Channel messages are corrupted"
+  ([#13219](https://github.com/nim-lang/Nim/issues/13219))
+- Fixed "Codegen bug with exportc and case objects"
+  ([#13281](https://github.com/nim-lang/Nim/issues/13281))
+- Fixed "[bugfix] fix #11590: c compiler warnings silently ignored, giving undefined behavior"
+  ([#11591](https://github.com/nim-lang/Nim/issues/11591))
+- Fixed "[CI] tnetdial flaky test"
+  ([#13132](https://github.com/nim-lang/Nim/issues/13132))
+- Fixed "Cross-Compiling with -d:mingw fails to locate compiler under OSX"
+  ([#10717](https://github.com/nim-lang/Nim/issues/10717))
+- Fixed "`nim doc --project` broken with imports below main project file or duplicate names"
+  ([#13150](https://github.com/nim-lang/Nim/issues/13150))
+- Fixed "regression: isNamedTuple(MyGenericTuple[int]) is false, should be true"
+  ([#13349](https://github.com/nim-lang/Nim/issues/13349))
+- Fixed "--gc:arc codegen bug copying objects bound to C structs with missing C struct fields"
+  ([#13269](https://github.com/nim-lang/Nim/issues/13269))
+- Fixed "write requires conversion to string"
+  ([#13182](https://github.com/nim-lang/Nim/issues/13182))
+- Fixed "Some remarks to stdlib documentation"
+  ([#13352](https://github.com/nim-lang/Nim/issues/13352))
+- Fixed "a `check` in unittest generated by template doesn't show actual value"
+  ([#6736](https://github.com/nim-lang/Nim/issues/6736))
+- Fixed "Implicit return with case expression fails with 'var' return."
+  ([#3339](https://github.com/nim-lang/Nim/issues/3339))
+- Fixed "Segfault with closure on arc"
+  ([#13314](https://github.com/nim-lang/Nim/issues/13314))
+- Fixed "[Macro] Crash on malformed case statement with multiple else"
+  ([#13255](https://github.com/nim-lang/Nim/issues/13255))
+- Fixed "regression: `echo 'discard' | nim c -r -` generates a file '-' ; `-` should be treated specially"
+  ([#13374](https://github.com/nim-lang/Nim/issues/13374))
+- Fixed "on OSX, debugging (w gdb or lldb) a nim program crashes at the 1st call to `execCmdEx`"
+  ([#9634](https://github.com/nim-lang/Nim/issues/9634))
+- Fixed "Internal error in getTypeDescAux"
+  ([#13378](https://github.com/nim-lang/Nim/issues/13378))
+- Fixed "gc:arc mode breaks tuple let"
+  ([#13368](https://github.com/nim-lang/Nim/issues/13368))
+- Fixed "Nim compiler hangs for certain C/C++ compiler errors"
+  ([#8648](https://github.com/nim-lang/Nim/issues/8648))
+- Fixed "htmlgen does not support `data-*` attributes"
+  ([#13444](https://github.com/nim-lang/Nim/issues/13444))
+- Fixed "[gc:arc] setLen will cause string not to be null-terminated."
+  ([#13457](https://github.com/nim-lang/Nim/issues/13457))
+- Fixed "joinPath("", "") is "/" ; should be """
+  ([#13455](https://github.com/nim-lang/Nim/issues/13455))
+- Fixed "[CI] flaky test on windows: tests/osproc/texitcode.nim"
+  ([#13449](https://github.com/nim-lang/Nim/issues/13449))
+- Fixed "Casting to float32 on NimVM is broken"
+  ([#13479](https://github.com/nim-lang/Nim/issues/13479))
+- Fixed "`--hints:off` doesn't work (doesn't override ~/.config/nim.cfg)"
+  ([#8312](https://github.com/nim-lang/Nim/issues/8312))
+- Fixed "joinPath("", "") is "/" ; should be """
+  ([#13455](https://github.com/nim-lang/Nim/issues/13455))
+- Fixed "tables.values is broken"
+  ([#13496](https://github.com/nim-lang/Nim/issues/13496))
+- Fixed "global user config can override project specific config"
+  ([#9405](https://github.com/nim-lang/Nim/issues/9405))
+- Fixed "Non deterministic macros and id consistency problem"
+  ([#12627](https://github.com/nim-lang/Nim/issues/12627))
+- Fixed "try expression doesn't work with return on expect branch"
+  ([#13490](https://github.com/nim-lang/Nim/issues/13490))
+- Fixed "CI will break every 4 years on feb 28: times doesn't handle leap years properly"
+  ([#13543](https://github.com/nim-lang/Nim/issues/13543))
+- Fixed "[minor] `nimgrep --word` doesn't work with operators (eg misses  `1 +% 2`)"
+  ([#13528](https://github.com/nim-lang/Nim/issues/13528))
+- Fixed "`as` is usable as infix operator but its existence and precedence are not documented"
+  ([#13409](https://github.com/nim-lang/Nim/issues/13409))
+- Fixed "JSON unmarshalling drops seq's items"
+  ([#13531](https://github.com/nim-lang/Nim/issues/13531))
+- Fixed "os.joinPath returns wrong path when head ends '\' or '/' and tail starts '..'."
+  ([#13579](https://github.com/nim-lang/Nim/issues/13579))
+- Fixed "Block-local types with the same name lead to bad codegen (sighashes regression)"
+  ([#5170](https://github.com/nim-lang/Nim/issues/5170))
+- Fixed "tuple codegen error"
+  ([#12704](https://github.com/nim-lang/Nim/issues/12704))
+- Fixed "newHttpHeaders does not accept repeated headers"
+  ([#13573](https://github.com/nim-lang/Nim/issues/13573))
+- Fixed "regression: --incremental:on fails on simplest example"
+  ([#13319](https://github.com/nim-lang/Nim/issues/13319))
+- Fixed "strscan can't get value of last element in format"
+  ([#13605](https://github.com/nim-lang/Nim/issues/13605))
+- Fixed "hashes_examples crashes with "Bus Error" (unaligned access) on sparc64"
+  ([#12508](https://github.com/nim-lang/Nim/issues/12508))
+- Fixed "gc:arc bug with re-used `seq[T]`"
+  ([#13596](https://github.com/nim-lang/Nim/issues/13596))
+- Fixed "`raise CatchableError` is broken with --gc:arc  when throwing inside a proc"
+  ([#13599](https://github.com/nim-lang/Nim/issues/13599))
+- Fixed "cpp --gc:arc --exceptions:goto fails to raise with discard"
+  ([#13436](https://github.com/nim-lang/Nim/issues/13436))
+- Fixed "terminal doesn't compile with -d:useWinAnsi"
+  ([#13607](https://github.com/nim-lang/Nim/issues/13607))
+- Fixed "Parsing "sink ptr T" - region needs to be an object type"
+  ([#12757](https://github.com/nim-lang/Nim/issues/12757))
+- Fixed "gc:arc + threads:on + closures compilation error"
+  ([#13519](https://github.com/nim-lang/Nim/issues/13519))
+- Fixed "[ARC] segmentation fault"
+  ([#13240](https://github.com/nim-lang/Nim/issues/13240))
+- Fixed "times.toDateTime buggy on 29th, 30th and 31th of each month"
+  ([#13558](https://github.com/nim-lang/Nim/issues/13558))
+- Fixed "Deque misbehaves on VM"
+  ([#13310](https://github.com/nim-lang/Nim/issues/13310))
+- Fixed "Nimscript listFiles should throw exception when path is not found"
+  ([#12676](https://github.com/nim-lang/Nim/issues/12676))
+- Fixed "koch boot fails if even an empty config.nims is present in ~/.config/nims/ [devel regression]"
+  ([#13633](https://github.com/nim-lang/Nim/issues/13633))
+- Fixed "nim doc generates lots of false positive LockLevel warnings"
+  ([#13218](https://github.com/nim-lang/Nim/issues/13218))
+- Fixed "Arrays are passed by copy to iterators, causing crashes, unnecessary allocations and slowdowns"
+  ([#12747](https://github.com/nim-lang/Nim/issues/12747))
+- Fixed "Range types always uses signed integer as a base type"
+  ([#13646](https://github.com/nim-lang/Nim/issues/13646))
+- Fixed "Generate c code cannot compile with recent devel version"
+  ([#13645](https://github.com/nim-lang/Nim/issues/13645))
+- Fixed "[regression] VM: Error: cannot convert -1 to uint64"
+  ([#13661](https://github.com/nim-lang/Nim/issues/13661))
+- Fixed "Spurious raiseException(Exception) detected"
+  ([#13654](https://github.com/nim-lang/Nim/issues/13654))
+- Fixed "gc:arc memory leak"
+  ([#13659](https://github.com/nim-lang/Nim/issues/13659))
+- Fixed "Error: cannot convert -1 to uint (inside tuples)"
+  ([#13671](https://github.com/nim-lang/Nim/issues/13671))
+- Fixed "strformat issue with --gc:arc"
+  ([#13622](https://github.com/nim-lang/Nim/issues/13622))
+- Fixed "astToStr doesn't work inside generics"
+  ([#13524](https://github.com/nim-lang/Nim/issues/13524))
+- Fixed "oswalkdir.walkDirRec wont return folders"
+  ([#11458](https://github.com/nim-lang/Nim/issues/11458))
+- Fixed "`echo 'echo 1' | nim c -r -`  silently gives wrong results (nimBetterRun not updated for stdin)"
+  ([#13412](https://github.com/nim-lang/Nim/issues/13412))
+- Fixed "gc:arc destroys the global variable accidentally."
+  ([#13691](https://github.com/nim-lang/Nim/issues/13691))
+- Fixed "[minor] sigmatch errors should be sorted, for reproducible errors"
+  ([#13538](https://github.com/nim-lang/Nim/issues/13538))
+- Fixed "Exception when converting csize to clong"
+  ([#13698](https://github.com/nim-lang/Nim/issues/13698))
+- Fixed "ARC: variables are no copied on the thread spawn causing crashes"
+  ([#13708](https://github.com/nim-lang/Nim/issues/13708))
+- Fixed "Illegal distinct seq causes compiler crash"
+  ([#13720](https://github.com/nim-lang/Nim/issues/13720))
+- Fixed "cyclic seq definition crashes the compiler"
+  ([#13715](https://github.com/nim-lang/Nim/issues/13715))
+- Fixed "Iterator with openArray parameter make the argument evaluated many times"
+  ([#13417](https://github.com/nim-lang/Nim/issues/13417))
+- Fixed "net/asyncnet: Unable to access peer's certificate chain"
+  ([#13299](https://github.com/nim-lang/Nim/issues/13299))
+- Fixed "Accidentally "SIGSEGV: Illegal storage access" error after arc optimizations (#13325)"
+  ([#13709](https://github.com/nim-lang/Nim/issues/13709))
+- Fixed "Base64 Regression"
+  ([#13722](https://github.com/nim-lang/Nim/issues/13722))
+- Fixed "A regression (?) with --gc:arc and repr"
+  ([#13731](https://github.com/nim-lang/Nim/issues/13731))
+- Fixed "Internal compiler error when using the new variable pragmas"
+  ([#13737](https://github.com/nim-lang/Nim/issues/13737))
+- Fixed "bool conversion produces vcc 2019 warning at cpp compilation stage"
+  ([#13744](https://github.com/nim-lang/Nim/issues/13744))
+- Fixed "Compiler "does not detect" a type recursion error in the wrong code, remaining frozen"
+  ([#13763](https://github.com/nim-lang/Nim/issues/13763))
+- Fixed "[minor] regression: `Foo[0.0] is Foo[-0.0]` is now false"
+  ([#13730](https://github.com/nim-lang/Nim/issues/13730))
+- Fixed "`nim doc` - only whitespace on first line causes segfault"
+  ([#13631](https://github.com/nim-lang/Nim/issues/13631))
+- Fixed "hashset regression"
+  ([#13794](https://github.com/nim-lang/Nim/issues/13794))
+- Fixed "`os.getApplFreebsd` could return incorrect paths in the case of a long path"
+  ([#13806](https://github.com/nim-lang/Nim/issues/13806))
+- Fixed "Destructors are not inherited"
+  ([#13810](https://github.com/nim-lang/Nim/issues/13810))
+- Fixed "io.readLines AssertionError on devel"
+  ([#13829](https://github.com/nim-lang/Nim/issues/13829))
+- Fixed "exceptions:goto accidentally reset the variable during exception handling"
+  ([#13782](https://github.com/nim-lang/Nim/issues/13782))

changelogs/changelog_X_XX_X.md

@@ -0,0 +1,20 @@
+# v1.xx.x - yyyy-mm-dd
+
+This is an example file.
+The changes should go to changelog.md!
+
+
+## Standard library additions and changes
+
+- Added `example.exampleProc`.
+- Changed `example.foo` to take additional `bar` parameter.
+
+
+## Language changes
+
+
+## Compiler changes
+
+
+## Tool changes
+

changelogs/readme.md

@@ -0,0 +1,61 @@
+# Update changelog
+After each release, call:
+```
+git mv changelog.md changelogs/changelog_0_19_0.md # use correct version
+cp changelogs/changelog_X_XX_X.md changelog.md
+git add changelog.md
+```
+
+## Recent changelogs are saved here (with their git history)
+
+## Older changelogs are stored in https://github.com/nim-lang/website
+
+## source files:
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2009-12-21-version-086-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2010-03-14-version-088-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2010-10-20-version-0810-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2011-07-10-version-0812-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2012-02-09-version-0814-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2012-09-23-version-090-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2013-05-20-version-092-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2014-04-21-version-094-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2014-10-19-version-096-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2014-12-29-version-0102-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2015-04-30-version-0110-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2015-05-04-version-0112-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2015-10-27-version-0120-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2016-01-18-version-0130-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2016-06-07-version-0140-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2016-06-09-version-0142-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2016-09-30-version-0150-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2016-10-23-version-0152-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2017-01-08-version-0160-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2017-05-17-version-0170-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2017-09-07-version-0172-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2018-03-01-version-0180-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2018-09-26-version-0190-released.md
+
+## urls:
+* https://nim-lang.org/blog/2009/12/21/version-086-released.html
+* https://nim-lang.org/blog/2010/03/14/version-088-released.html
+* https://nim-lang.org/blog/2010/10/20/version-0810-released.html
+* https://nim-lang.org/blog/2011/07/10/version-0812-released.html
+* https://nim-lang.org/blog/2012/02/09/version-0814-released.html
+* https://nim-lang.org/blog/2012/09/23/version-090-released.html
+* https://nim-lang.org/blog/2013/05/20/version-092-released.html
+* https://nim-lang.org/blog/2014/04/21/version-094-released.html
+* https://nim-lang.org/blog/2014/10/19/version-096-released.html
+* https://nim-lang.org/blog/2014/12/29/version-0102-released.html
+* https://nim-lang.org/blog/2015/04/30/version-0110-released.html
+* https://nim-lang.org/blog/2015/05/04/version-0112-released.html
+* https://nim-lang.org/blog/2015/10/27/version-0120-released.html
+* https://nim-lang.org/blog/2016/01/18/version-0130-released.html
+* https://nim-lang.org/blog/2016/06/07/version-0140-released.html
+* https://nim-lang.org/blog/2016/06/09/version-0142-released.html
+* https://nim-lang.org/blog/2016/09/30/version-0150-released.html
+* https://nim-lang.org/blog/2016/10/23/version-0152-released.html
+* https://nim-lang.org/blog/2017/01/08/version-0160-released.html
+* https://nim-lang.org/blog/2017/05/17/version-0170-released.html
+* https://nim-lang.org/blog/2017/09/07/version-0172-released.html
+* https://nim-lang.org/blog/2018/03/01/version-0180-released.html
+* https://nim-lang.org/blog/2018/09/26/version-0190-released.html

ci/nsis_build.bat

@@ -1,4 +1,4 @@
-REM - Run the full testsuite;  tests\testament\tester all
+REM - Run the full testsuite;  testament\tester all
 
 REM - Uncomment the list of changes in news.txt
 REM - write a news ticker entry

compiler.nimble

@@ -4,6 +4,6 @@ author = "Andreas Rumpf"
 description = "Compiler package providing the compiler sources as a library."
 license = "MIT"
 
-installDirs = @["compiler"]
+installDirs = @["compiler", "nimsuggest"]
 
 requires "nim >= 0.14.0"

compiler/aliases.nim

@@ -10,7 +10,7 @@
 ## Simple alias analysis for the HLO and the code generators.
 
 import
-  ast, astalgo, types, trees, intsets, msgs
+  ast, astalgo, types, trees, intsets
 
 type
   TAnalysisResult* = enum
@@ -22,17 +22,17 @@ proc isPartOfAux(n: PNode, b: PType, marker: var IntSet): TAnalysisResult =
   result = arNo
   case n.kind
   of nkRecList:
-    for i in countup(0, sonsLen(n) - 1):
-      result = isPartOfAux(n.sons[i], b, marker)
+    for i in 0..<n.len:
+      result = isPartOfAux(n[i], b, marker)
       if result == arYes: return
   of nkRecCase:
-    assert(n.sons[0].kind == nkSym)
-    result = isPartOfAux(n.sons[0], b, marker)
+    assert(n[0].kind == nkSym)
+    result = isPartOfAux(n[0], b, marker)
     if result == arYes: return
-    for i in countup(1, sonsLen(n) - 1):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = isPartOfAux(lastSon(n.sons[i]), b, marker)
+        result = isPartOfAux(lastSon(n[i]), b, marker)
         if result == arYes: return
       else: discard "isPartOfAux(record case branch)"
   of nkSym:
@@ -46,14 +46,14 @@ proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult =
   if compareTypes(a, b, dcEqIgnoreDistinct): return arYes
   case a.kind
   of tyObject:
-    if a.sons[0] != nil:
-      result = isPartOfAux(a.sons[0].skipTypes(skipPtrs), b, marker)
+    if a[0] != nil:
+      result = isPartOfAux(a[0].skipTypes(skipPtrs), b, marker)
     if result == arNo: result = isPartOfAux(a.n, b, marker)
   of tyGenericInst, tyDistinct, tyAlias, tySink:
     result = isPartOfAux(lastSon(a), b, marker)
   of tyArray, tySet, tyTuple:
-    for i in countup(0, sonsLen(a) - 1):
-      result = isPartOfAux(a.sons[i], b, marker)
+    for i in 0..<a.len:
+      result = isPartOfAux(a[i], b, marker)
       if result == arYes: return
   else: discard
 
@@ -108,7 +108,7 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
           result = arMaybe
     of nkBracketExpr:
       result = isPartOf(a[0], b[0])
-      if len(a) >= 2 and len(b) >= 2:
+      if a.len >= 2 and b.len >= 2:
         # array accesses:
         if result == arYes and isDeepConstExpr(a[1]) and isDeepConstExpr(b[1]):
           # we know it's the same array and we have 2 constant indexes;
@@ -186,4 +186,14 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
         if res != arNo:
           result = res
           if res == arYes: break
+    of nkCallKinds:
+      result = arNo
+      for i in 1..<b.len:
+        let res = isPartOf(a, b[i])
+        if res != arNo:
+          result = res
+          if res == arYes: break
+    of nkBracket:
+      if b.len > 0:
+        result = isPartOf(a, b[0])
     else: discard

compiler/asciitables.nim

@@ -0,0 +1,83 @@
+#[
+move to std/asciitables.nim once stable, or to a nimble paackage
+once compiler can depend on nimble
+]#
+
+type Cell* = object
+  text*: string
+  width*, row*, col*, ncols*, nrows*: int
+
+iterator parseTableCells*(s: string, delim = '\t'): Cell =
+  ## iterates over all cells in a `delim`-delimited `s`, after a 1st
+  ## pass that computes number of rows, columns, and width of each column.
+  var widths: seq[int]
+  var cell: Cell
+  template update() =
+    if widths.len<=cell.col:
+      widths.setLen cell.col+1
+      widths[cell.col] = cell.width
+    else:
+      widths[cell.col] = max(widths[cell.col], cell.width)
+    cell.width = 0
+
+  for a in s:
+    case a
+    of '\n':
+      update()
+      cell.col = 0
+      cell.row.inc
+    elif a == delim:
+      update()
+      cell.col.inc
+    else:
+      # todo: consider multi-width chars when porting to non-ascii implementation
+      cell.width.inc
+  if s.len > 0 and s[^1] != '\n':
+    update()
+
+  cell.ncols = widths.len
+  cell.nrows = cell.row + 1
+  cell.row = 0
+  cell.col = 0
+  cell.width = 0
+
+  template update2() =
+    cell.width = widths[cell.col]
+    yield cell
+    cell.text = ""
+    cell.width = 0
+    cell.col.inc
+
+  template finishRow() =
+    for col in cell.col..<cell.ncols:
+      cell.col = col
+      update2()
+    cell.col = 0
+
+  for a in s:
+    case a
+    of '\n':
+      finishRow()
+      cell.row.inc
+    elif a == delim:
+      update2()
+    else:
+      cell.width.inc
+      cell.text.add a
+
+  if s.len > 0 and s[^1] != '\n':
+    finishRow()
+
+proc alignTable*(s: string, delim = '\t', fill = ' ', sep = " "): string =
+  ## formats a `delim`-delimited `s` representing a table; each cell is aligned
+  ## to a width that's computed for each column; consecutive columns are
+  ## delimited by `sep`, and alignment space is filled using `fill`.
+  ## More customized formatting can be done by calling `parseTableCells` directly.
+  for cell in parseTableCells(s, delim):
+    result.add cell.text
+    for i in cell.text.len..<cell.width:
+      result.add fill
+    if cell.col < cell.ncols-1:
+      result.add sep
+    if cell.col == cell.ncols-1 and cell.row < cell.nrows - 1:
+      result.add '\n'

compiler/astalgo.nim

@@ -24,46 +24,55 @@ proc typeToYaml*(conf: ConfigRef; n: PType, indent: int = 0, maxRecDepth: int =
 proc symToYaml*(conf: ConfigRef; n: PSym, indent: int = 0, maxRecDepth: int = - 1): Rope
 proc lineInfoToStr*(conf: ConfigRef; info: TLineInfo): Rope
 
-when declared(echo):
-  # these are for debugging only: They are not really deprecated, but I want
-  # the warning so that release versions do not contain debugging statements:
-  proc debug*(n: PSym; conf: ConfigRef = nil) {.exportc: "debugSym", deprecated.}
-  proc debug*(n: PType; conf: ConfigRef = nil) {.exportc: "debugType", deprecated.}
-  proc debug*(n: PNode; conf: ConfigRef = nil) {.exportc: "debugNode", deprecated.}
 
-  template debug*(x: PSym|PType|PNode) {.deprecated.} =
-    when compiles(c.config):
-      debug(c.config, x)
-    elif compiles(c.graph.config):
-      debug(c.graph.config, x)
+# these are for debugging only: They are not really deprecated, but I want
+# the warning so that release versions do not contain debugging statements:
+proc debug*(n: PSym; conf: ConfigRef = nil) {.exportc: "debugSym", deprecated.}
+proc debug*(n: PType; conf: ConfigRef = nil) {.exportc: "debugType", deprecated.}
+proc debug*(n: PNode; conf: ConfigRef = nil) {.exportc: "debugNode", deprecated.}
+
+proc typekinds*(t: PType) {.deprecated.} =
+  var t = t
+  var s = ""
+  while t != nil and t.len > 0:
+    s.add $t.kind
+    s.add " "
+    t = t.lastSon
+  echo s
+
+template debug*(x: PSym|PType|PNode) {.deprecated.} =
+  when compiles(c.config):
+    debug(c.config, x)
+  elif compiles(c.graph.config):
+    debug(c.graph.config, x)
+  else:
+    error()
+
+template debug*(x: auto) {.deprecated.} =
+  echo x
+
+template mdbg*: bool {.deprecated.} =
+  when compiles(c.graph):
+    c.module.fileIdx == c.graph.config.projectMainIdx
+  elif compiles(c.module):
+    c.module.fileIdx == c.config.projectMainIdx
+  elif compiles(c.c.module):
+    c.c.module.fileIdx == c.c.config.projectMainIdx
+  elif compiles(m.c.module):
+    m.c.module.fileIdx == m.c.config.projectMainIdx
+  elif compiles(cl.c.module):
+    cl.c.module.fileIdx == cl.c.config.projectMainIdx
+  elif compiles(p):
+    when compiles(p.lex):
+      p.lex.fileIdx == p.lex.config.projectMainIdx
     else:
-      error()
-
-  template debug*(x: auto) {.deprecated.} =
-    echo x
-
-  template mdbg*: bool {.deprecated.} =
-    when compiles(c.graph):
-      c.module.fileIdx == c.graph.config.projectMainIdx
-    elif compiles(c.module):
-      c.module.fileIdx == c.config.projectMainIdx
-    elif compiles(c.c.module):
-      c.c.module.fileIdx == c.c.config.projectMainIdx
-    elif compiles(m.c.module):
-      m.c.module.fileIdx == m.c.config.projectMainIdx
-    elif compiles(cl.c.module):
-      cl.c.module.fileIdx == cl.c.config.projectMainIdx
-    elif compiles(p):
-      when compiles(p.lex):
-        p.lex.fileIdx == p.lex.config.projectMainIdx
-      else:
-        p.module.module.fileIdx == p.config.projectMainIdx
-    elif compiles(m.module.fileIdx):
-      m.module.fileIdx == m.config.projectMainIdx
-    elif compiles(L.fileIdx):
-      L.fileIdx == L.config.projectMainIdx
-    else:
-      error()
+      p.module.module.fileIdx == p.config.projectMainIdx
+  elif compiles(m.module.fileIdx):
+    m.module.fileIdx == m.config.projectMainIdx
+  elif compiles(L.fileIdx):
+    L.fileIdx == L.config.projectMainIdx
+  else:
+    error()
 
 # --------------------------- ident tables ----------------------------------
 proc idTableGet*(t: TIdTable, key: PIdObj): RootRef
@@ -77,7 +86,6 @@ proc idNodeTablePut*(t: var TIdNodeTable, key: PIdObj, val: PNode)
 
 # ---------------------------------------------------------------------------
 
-proc getSymFromList*(list: PNode, ident: PIdent, start: int = 0): PSym
 proc lookupInRecord*(n: PNode, field: PIdent): PSym
 proc mustRehash*(length, counter: int): bool
 proc nextTry*(h, maxHash: Hash): Hash {.inline.}
@@ -108,16 +116,16 @@ proc skipConvAndClosure*(n: PNode): PNode =
     case result.kind
     of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64,
        nkClosure:
-      result = result.sons[0]
+      result = result[0]
     of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-      result = result.sons[1]
+      result = result[1]
     else: break
 
 proc sameValue*(a, b: PNode): bool =
   result = false
   case a.kind
   of nkCharLit..nkUInt64Lit:
-    if b.kind in {nkCharLit..nkUInt64Lit}: result = a.intVal == b.intVal
+    if b.kind in {nkCharLit..nkUInt64Lit}: result = getInt(a) == getInt(b)
   of nkFloatLit..nkFloat64Lit:
     if b.kind in {nkFloatLit..nkFloat64Lit}: result = a.floatVal == b.floatVal
   of nkStrLit..nkTripleStrLit:
@@ -131,8 +139,8 @@ proc leValue*(a, b: PNode): bool =
   # a <= b?
   result = false
   case a.kind
-  of nkCharLit..nkUInt32Lit:
-    if b.kind in {nkCharLit..nkUInt32Lit}: result = a.intVal <= b.intVal
+  of nkCharLit..nkUInt64Lit:
+    if b.kind in {nkCharLit..nkUInt64Lit}: result = getInt(a) <= getInt(b)
   of nkFloatLit..nkFloat64Lit:
     if b.kind in {nkFloatLit..nkFloat64Lit}: result = a.floatVal <= b.floatVal
   of nkStrLit..nkTripleStrLit:
@@ -152,17 +160,17 @@ proc lookupInRecord(n: PNode, field: PIdent): PSym =
   result = nil
   case n.kind
   of nkRecList:
-    for i in countup(0, sonsLen(n) - 1):
-      result = lookupInRecord(n.sons[i], field)
+    for i in 0..<n.len:
+      result = lookupInRecord(n[i], field)
       if result != nil: return
   of nkRecCase:
-    if (n.sons[0].kind != nkSym): return nil
-    result = lookupInRecord(n.sons[0], field)
+    if (n[0].kind != nkSym): return nil
+    result = lookupInRecord(n[0], field)
     if result != nil: return
-    for i in countup(1, sonsLen(n) - 1):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = lookupInRecord(lastSon(n.sons[i]), field)
+        result = lookupInRecord(lastSon(n[i]), field)
         if result != nil: return
       else: return nil
   of nkSym:
@@ -174,14 +182,55 @@ proc getModule*(s: PSym): PSym =
   assert((result.kind == skModule) or (result.owner != result))
   while result != nil and result.kind != skModule: result = result.owner
 
-proc getSymFromList(list: PNode, ident: PIdent, start: int = 0): PSym =
-  for i in countup(start, sonsLen(list) - 1):
-    if list.sons[i].kind == nkSym:
-      result = list.sons[i].sym
+proc fromSystem*(op: PSym): bool {.inline.} = sfSystemModule in getModule(op).flags
+proc getSymFromList*(list: PNode, ident: PIdent, start: int = 0): PSym =
+  for i in start..<list.len:
+    if list[i].kind == nkSym:
+      result = list[i].sym
       if result.name.id == ident.id: return
     else: return nil
   result = nil
 
+proc sameIgnoreBacktickGensymInfo(a, b: string): bool =
+  if a[0] != b[0]: return false
+  var alen = a.len - 1
+  while alen > 0 and a[alen] != '`': dec(alen)
+  if alen <= 0: alen = a.len
+
+  var i = 1
+  var j = 1
+  while true:
+    while i < alen and a[i] == '_': inc i
+    while j < b.len and b[j] == '_': inc j
+    var aa = if i < alen: toLowerAscii(a[i]) else: '\0'
+    var bb = if j < b.len: toLowerAscii(b[j]) else: '\0'
+    if aa != bb: return false
+
+    # the characters are identical:
+    if i >= alen:
+      # both cursors at the end:
+      if j >= b.len: return true
+      # not yet at the end of 'b':
+      return false
+    elif j >= b.len:
+      return false
+    inc i
+    inc j
+
+proc getNamedParamFromList*(list: PNode, ident: PIdent): PSym =
+  ## Named parameters are special because a named parameter can be
+  ## gensym'ed and then they have '\`<number>' suffix that we need to
+  ## ignore, see compiler / evaltempl.nim, snippet:
+  ##
+  ##..code-block:: nim
+  ##
+  ##    result.add newIdentNode(getIdent(c.ic, x.name.s & "\`gensym" & $x.id),
+  ##            if c.instLines: actual.info else: templ.info)
+  for i in 1..<list.len:
+    let it = list[i].sym
+    if it.name.id == ident.id or
+        sameIgnoreBacktickGensymInfo(it.name.s, ident.s): return it
+
 proc hashNode(p: RootRef): Hash =
   result = hash(cast[pointer](p))
 
@@ -206,15 +255,15 @@ proc makeYamlString*(s: string): Rope =
   const MaxLineLength = 64
   result = nil
   var res = "\""
-  for i in 0 ..< s.len:
+  for i in 0..<s.len:
     if (i + 1) mod MaxLineLength == 0:
-      add(res, '\"')
-      add(res, "\n")
-      add(result, rope(res))
+      res.add('\"')
+      res.add("\n")
+      result.add(rope(res))
       res = "\""              # reset
-    add(res, toYamlChar(s[i]))
-  add(res, '\"')
-  add(result, rope(res))
+    res.add(toYamlChar(s[i]))
+  res.add('\"')
+  result.add(rope(res))
 
 proc flagsToStr[T](flags: set[T]): Rope =
   if flags == {}:
@@ -222,8 +271,8 @@ proc flagsToStr[T](flags: set[T]): Rope =
   else:
     result = nil
     for x in items(flags):
-      if result != nil: add(result, ", ")
-      add(result, makeYamlString($x))
+      if result != nil: result.add(", ")
+      result.add(makeYamlString($x))
     result = "[" & result & "]"
 
 proc lineInfoToStr(conf: ConfigRef; info: TLineInfo): Rope =
@@ -238,65 +287,68 @@ proc symToYamlAux(conf: ConfigRef; n: PSym, marker: var IntSet,
 proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet,
                    indent, maxRecDepth: int): Rope
 
-proc ropeConstr(indent: int, c: openArray[Rope]): Rope =
-  # array of (name, value) pairs
-  var istr = rspaces(indent + 2)
-  result = rope("{")
-  var i = 0
-  while i <= high(c):
-    if i > 0: add(result, ",")
-    addf(result, "$N$1\"$2\": $3", [istr, c[i], c[i + 1]])
-    inc(i, 2)
-  addf(result, "$N$1}", [rspaces(indent)])
-
 proc symToYamlAux(conf: ConfigRef; n: PSym, marker: var IntSet, indent: int,
                   maxRecDepth: int): Rope =
   if n == nil:
     result = rope("null")
   elif containsOrIncl(marker, n.id):
-    result = "\"$1 @$2\"" % [rope(n.name.s), rope(
-        strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))]
+    result = "\"$1\"" % [rope(n.name.s)]
   else:
     var ast = treeToYamlAux(conf, n.ast, marker, indent + 2, maxRecDepth - 1)
-    result = ropeConstr(indent, [rope("kind"),
-                                 makeYamlString($n.kind),
-                                 rope("name"), makeYamlString(n.name.s),
-                                 rope("typ"), typeToYamlAux(conf, n.typ, marker,
-                                   indent + 2, maxRecDepth - 1),
-                                 rope("info"), lineInfoToStr(conf, n.info),
-                                 rope("flags"), flagsToStr(n.flags),
-                                 rope("magic"), makeYamlString($n.magic),
-                                 rope("ast"), ast, rope("options"),
-                                 flagsToStr(n.options), rope("position"),
-                                 rope(n.position)])
+                                 #rope("typ"), typeToYamlAux(conf, n.typ, marker,
+                                 #  indent + 2, maxRecDepth - 1),
+    let istr = rspaces(indent + 2)
+    result = rope("{")
+    result.addf("$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
+    result.addf("$N$1\"name\": $2", [istr, makeYamlString(n.name.s)])
+    result.addf("$N$1\"typ\": $2", [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth - 1)])
+    if conf != nil:
+      # if we don't pass the config, we probably don't care about the line info
+      result.addf("$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
+    if card(n.flags) > 0:
+      result.addf("$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
+    result.addf("$N$1\"magic\": $2", [istr, makeYamlString($n.magic)])
+    result.addf("$N$1\"ast\": $2", [istr, ast])
+    result.addf("$N$1\"options\": $2", [istr, flagsToStr(n.options)])
+    result.addf("$N$1\"position\": $2", [istr, rope(n.position)])
+    result.addf("$N$1\"k\": $2", [istr, makeYamlString($n.loc.k)])
+    result.addf("$N$1\"storage\": $2", [istr, makeYamlString($n.loc.storage)])
+    if card(n.loc.flags) > 0:
+      result.addf("$N$1\"flags\": $2", [istr, makeYamlString($n.loc.flags)])
+    result.addf("$N$1\"r\": $2", [istr, n.loc.r])
+    result.addf("$N$1\"lode\": $2", [istr, treeToYamlAux(conf, n.loc.lode, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1}", [rspaces(indent)])
 
 proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int,
                    maxRecDepth: int): Rope =
+  var sonsRope: Rope
   if n == nil:
-    result = rope("null")
+    sonsRope = rope("null")
   elif containsOrIncl(marker, n.id):
-    result = "\"$1 @$2\"" % [rope($n.kind), rope(
+    sonsRope = "\"$1 @$2\"" % [rope($n.kind), rope(
         strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))]
   else:
-    if sonsLen(n) > 0:
-      result = rope("[")
-      for i in countup(0, sonsLen(n) - 1):
-        if i > 0: add(result, ",")
-        addf(result, "$N$1$2", [rspaces(indent + 4), typeToYamlAux(conf, n.sons[i],
+    if n.len > 0:
+      sonsRope = rope("[")
+      for i in 0..<n.len:
+        if i > 0: sonsRope.add(",")
+        sonsRope.addf("$N$1$2", [rspaces(indent + 4), typeToYamlAux(conf, n[i],
             marker, indent + 4, maxRecDepth - 1)])
-      addf(result, "$N$1]", [rspaces(indent + 2)])
+      sonsRope.addf("$N$1]", [rspaces(indent + 2)])
     else:
-      result = rope("null")
-    result = ropeConstr(indent, [rope("kind"),
-                                 makeYamlString($n.kind),
-                                 rope("sym"), symToYamlAux(conf, n.sym, marker,
-        indent + 2, maxRecDepth - 1), rope("n"), treeToYamlAux(conf, n.n, marker,
-        indent + 2, maxRecDepth - 1), rope("flags"), flagsToStr(n.flags),
-                                 rope("callconv"),
-                                 makeYamlString(CallingConvToStr[n.callConv]),
-                                 rope("size"), rope(n.size),
-                                 rope("align"), rope(n.align),
-                                 rope("sons"), result])
+      sonsRope = rope("null")
+
+    let istr = rspaces(indent + 2)
+    result = rope("{")
+    result.addf("$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
+    result.addf("$N$1\"sym\": $2",  [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1\"n\": $2",     [istr, treeToYamlAux(conf, n.n, marker, indent + 2, maxRecDepth - 1)])
+    if card(n.flags) > 0:
+      result.addf("$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
+    result.addf("$N$1\"callconv\": $2", [istr, makeYamlString(CallingConvToStr[n.callConv])])
+    result.addf("$N$1\"size\": $2", [istr, rope(n.size)])
+    result.addf("$N$1\"align\": $2", [istr, rope(n.align)])
+    result.addf("$N$1\"sons\": $2", [istr, sonsRope])
 
 proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
                    maxRecDepth: int): Rope =
@@ -306,34 +358,35 @@ proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
     var istr = rspaces(indent + 2)
     result = "{$N$1\"kind\": $2" % [istr, makeYamlString($n.kind)]
     if maxRecDepth != 0:
-      addf(result, ",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
+      if conf != nil:
+        result.addf(",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
       case n.kind
       of nkCharLit..nkInt64Lit:
-        addf(result, ",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
+        result.addf(",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
       of nkFloatLit, nkFloat32Lit, nkFloat64Lit:
-        addf(result, ",$N$1\"floatVal\": $2",
+        result.addf(",$N$1\"floatVal\": $2",
             [istr, rope(n.floatVal.toStrMaxPrecision)])
       of nkStrLit..nkTripleStrLit:
-        addf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
+        result.addf(",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
       of nkSym:
-        addf(result, ",$N$1\"sym\": $2",
+        result.addf(",$N$1\"sym\": $2",
              [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth)])
       of nkIdent:
         if n.ident != nil:
-          addf(result, ",$N$1\"ident\": $2", [istr, makeYamlString(n.ident.s)])
+          result.addf(",$N$1\"ident\": $2", [istr, makeYamlString(n.ident.s)])
         else:
-          addf(result, ",$N$1\"ident\": null", [istr])
+          result.addf(",$N$1\"ident\": null", [istr])
       else:
-        if sonsLen(n) > 0:
-          addf(result, ",$N$1\"sons\": [", [istr])
-          for i in countup(0, sonsLen(n) - 1):
-            if i > 0: add(result, ",")
-            addf(result, "$N$1$2", [rspaces(indent + 4), treeToYamlAux(conf, n.sons[i],
+        if n.len > 0:
+          result.addf(",$N$1\"sons\": [", [istr])
+          for i in 0..<n.len:
+            if i > 0: result.add(",")
+            result.addf("$N$1$2", [rspaces(indent + 4), treeToYamlAux(conf, n[i],
                 marker, indent + 4, maxRecDepth - 1)])
-          addf(result, "$N$1]", [istr])
-      addf(result, ",$N$1\"typ\": $2",
+          result.addf("$N$1]", [istr])
+      result.addf(",$N$1\"typ\": $2",
            [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth)])
-    addf(result, "$N$1}", [rspaces(indent)])
+    result.addf("$N$1}", [rspaces(indent)])
 
 proc treeToYaml(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope =
   var marker = initIntSet()
@@ -347,93 +400,258 @@ proc symToYaml(conf: ConfigRef; n: PSym, indent: int = 0, maxRecDepth: int = - 1
   var marker = initIntSet()
   result = symToYamlAux(conf, n, marker, indent, maxRecDepth)
 
-proc debugTree*(conf: ConfigRef; n: PNode, indent: int, maxRecDepth: int; renderType=false): Rope
-proc debugType(conf: ConfigRef; n: PType, maxRecDepth=100): Rope =
+import tables
+
+const backrefStyle = "\e[90m"
+const enumStyle = "\e[34m"
+const numberStyle = "\e[33m"
+const stringStyle = "\e[32m"
+const resetStyle  = "\e[0m"
+
+type
+  DebugPrinter = object
+    conf: ConfigRef
+    visited: Table[pointer, int]
+    renderSymType: bool
+    indent: int
+    currentLine: int
+    firstItem: bool
+    useColor: bool
+    res: string
+
+proc indentMore(this: var DebugPrinter) =
+  this.indent += 2
+
+proc indentLess(this: var DebugPrinter) =
+  this.indent -= 2
+
+proc newlineAndIndent(this: var DebugPrinter) =
+  this.res.add "\n"
+  this.currentLine += 1
+  for i in 0..<this.indent:
+    this.res.add ' '
+
+proc openCurly(this: var DebugPrinter) =
+  this.res.add "{"
+  this.indentMore
+  this.firstItem = true
+
+proc closeCurly(this: var DebugPrinter) =
+  this.indentLess
+  this.newlineAndIndent
+  this.res.add "}"
+
+proc comma(this: var DebugPrinter) =
+  this.res.add ", "
+
+proc openBracket(this: var DebugPrinter) =
+  this.res.add "["
+  #this.indentMore
+
+proc closeBracket(this: var DebugPrinter) =
+  #this.indentLess
+  this.res.add "]"
+
+proc key(this: var DebugPrinter; key: string) =
+  if not this.firstItem:
+    this.res.add ","
+  this.firstItem = false
+
+  this.newlineAndIndent
+  this.res.add "\""
+  this.res.add key
+  this.res.add "\": "
+
+proc value(this: var DebugPrinter; value: string) =
+  if this.useColor:
+    this.res.add stringStyle
+  this.res.add "\""
+  this.res.add value
+  this.res.add "\""
+  if this.useColor:
+    this.res.add resetStyle
+
+proc value(this: var DebugPrinter; value: BiggestInt) =
+  if this.useColor:
+    this.res.add numberStyle
+  this.res.addInt value
+  if this.useColor:
+    this.res.add resetStyle
+
+proc value[T: enum](this: var DebugPrinter; value: T) =
+  if this.useColor:
+    this.res.add enumStyle
+  this.res.add "\""
+  this.res.add $value
+  this.res.add "\""
+  if this.useColor:
+    this.res.add resetStyle
+
+proc value[T: enum](this: var DebugPrinter; value: set[T]) =
+  this.openBracket
+  let high = card(value)-1
+  var i = 0
+  for v in value:
+    this.value v
+    if i != high:
+      this.comma
+    inc i
+  this.closeBracket
+
+template earlyExit(this: var DebugPrinter; n: PType | PNode | PSym) =
   if n == nil:
-    result = rope("null")
+    this.res.add "null"
+    return
+  let index = this.visited.getOrDefault(cast[pointer](n), -1)
+  if index < 0:
+    this.visited[cast[pointer](n)] = this.currentLine
   else:
-    result = rope($n.kind)
-    if n.sym != nil:
-      add(result, " ")
-      add(result, n.sym.name.s)
-    if n.kind in IntegralTypes and n.n != nil:
-      add(result, ", node: ")
-      add(result, debugTree(conf, n.n, 2, maxRecDepth-1, renderType=true))
-    if (n.kind != tyString) and (sonsLen(n) > 0) and maxRecDepth != 0:
-      add(result, "(")
-      for i in countup(0, sonsLen(n) - 1):
-        if i > 0: add(result, ", ")
-        if n.sons[i] == nil:
-          add(result, "null")
-        else:
-          add(result, debugType(conf, n.sons[i], maxRecDepth-1))
-      if n.kind == tyObject and n.n != nil:
-        add(result, ", node: ")
-        add(result, debugTree(conf, n.n, 2, maxRecDepth-1, renderType=true))
-      add(result, ")")
+    if this.useColor:
+      this.res.add backrefStyle
+    this.res.add "<defined "
+    this.res.addInt(this.currentLine - index)
+    this.res.add " lines upwards>"
+    if this.useColor:
+      this.res.add resetStyle
+    return
 
-proc debugTree(conf: ConfigRef; n: PNode, indent: int, maxRecDepth: int;
-               renderType=false): Rope =
-  if n == nil:
-    result = rope("null")
+proc value(this: var DebugPrinter; value: PType)
+proc value(this: var DebugPrinter; value: PNode)
+proc value(this: var DebugPrinter; value: PSym) =
+  earlyExit(this, value)
+
+  this.openCurly
+  this.key("kind")
+  this.value(value.kind)
+  this.key("name")
+  this.value(value.name.s)
+  this.key("id")
+  this.value(value.id)
+  if value.kind in {skField, skEnumField, skParam}:
+    this.key("position")
+    this.value(value.position)
+
+  if card(value.flags) > 0:
+    this.key("flags")
+    this.value(value.flags)
+
+  if this.renderSymType and value.typ != nil:
+    this.key "typ"
+    this.value(value.typ)
+
+  this.closeCurly
+
+proc value(this: var DebugPrinter; value: PType) =
+  earlyExit(this, value)
+
+  this.openCurly
+  this.key "kind"
+  this.value value.kind
+
+  this.key "id"
+  this.value value.id
+
+  if value.sym != nil:
+    this.key "sym"
+    this.value value.sym
+    #this.value value.sym.name.s
+
+  if card(value.flags) > 0:
+    this.key "flags"
+    this.value value.flags
+
+  if value.kind in IntegralTypes and value.n != nil:
+    this.key "n"
+    this.value value.n
+
+  if value.len > 0:
+    this.key "sons"
+    this.openBracket
+    for i in 0..<value.len:
+      this.value value[i]
+      if i != value.len - 1:
+        this.comma
+    this.closeBracket
+
+  if value.n != nil:
+    this.key "n"
+    this.value value.n
+
+  this.closeCurly
+
+proc value(this: var DebugPrinter; value: PNode) =
+  earlyExit(this, value)
+
+  this.openCurly
+  this.key "kind"
+  this.value  value.kind
+  when defined(useNodeIds):
+    this.key "id"
+    this.value value.id
+  if this.conf != nil:
+    this.key "info"
+    this.value $lineInfoToStr(this.conf, value.info)
+  if card(value.flags) > 0:
+    this.key "flags"
+    this.value value.flags
+
+  case value.kind
+  of nkCharLit..nkUInt64Lit:
+    this.key "intVal"
+    this.value value.intVal
+  of nkFloatLit, nkFloat32Lit, nkFloat64Lit:
+    this.key "floatVal"
+    this.value value.floatVal.toStrMaxPrecision
+  of nkStrLit..nkTripleStrLit:
+    this.key "strVal"
+    this.value value.strVal
+  of nkSym:
+    this.key "sym"
+    this.value value.sym
+    #this.value value.sym.name.s
+  of nkIdent:
+    if value.ident != nil:
+      this.key "ident"
+      this.value value.ident.s
   else:
-    var istr = rspaces(indent + 2)
-    result = "{$N$1\"kind\": $2" %
-             [istr, makeYamlString($n.kind)]
-    when defined(useNodeIds):
-      addf(result, ",$N$1\"id\": $2", [istr, rope(n.id)])
-    addf(result, ",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
-    if maxRecDepth != 0:
-      addf(result, ",$N$1\"flags\": $2", [istr, rope($n.flags)])
-      case n.kind
-      of nkCharLit..nkUInt64Lit:
-        addf(result, ",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
-      of nkFloatLit, nkFloat32Lit, nkFloat64Lit:
-        addf(result, ",$N$1\"floatVal\": $2",
-            [istr, rope(n.floatVal.toStrMaxPrecision)])
-      of nkStrLit..nkTripleStrLit:
-        addf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
-      of nkSym:
-        addf(result, ",$N$1\"sym\": $2_$3",
-            [istr, rope(n.sym.name.s), rope(n.sym.id)])
-        #     [istr, symToYaml(n.sym, indent, maxRecDepth),
-        #     rope(n.sym.id)])
-        if renderType and n.sym.typ != nil:
-          addf(result, ",$N$1\"typ\": $2", [istr, debugType(conf, n.sym.typ, 2)])
-      of nkIdent:
-        if n.ident != nil:
-          addf(result, ",$N$1\"ident\": $2", [istr, makeYamlString(n.ident.s)])
-        else:
-          addf(result, ",$N$1\"ident\": null", [istr])
-      else:
-        if renderType and n.typ != nil:
-          addf(result, ",$N$1\"typ\": $2", [istr, debugType(conf, n.typ, 2)])
-        if sonsLen(n) > 0:
-          addf(result, ",$N$1\"sons\": [", [istr])
-          for i in countup(0, sonsLen(n) - 1):
-            if i > 0: add(result, ",")
-            addf(result, "$N$1$2", [rspaces(indent + 4), debugTree(conf, n.sons[i],
-                indent + 4, maxRecDepth - 1, renderType)])
-          addf(result, "$N$1]", [istr])
-    addf(result, "$N$1}", [rspaces(indent)])
+    if this.renderSymType and value.typ != nil:
+      this.key "typ"
+      this.value value.typ
+    if value.len > 0:
+      this.key "sons"
+      this.openBracket
+      for i in 0..<value.len:
+        this.value value[i]
+        if i != value.len - 1:
+          this.comma
+      this.closeBracket
 
-when declared(echo):
-  proc debug(n: PSym; conf: ConfigRef) =
-    if n == nil:
-      echo("null")
-    elif n.kind == skUnknown:
-      echo("skUnknown")
-    else:
-      #writeLine(stdout, $symToYaml(n, 0, 1))
-      echo("$1_$2: $3, $4, $5, $6" % [
-        n.name.s, $n.id, $flagsToStr(n.flags), $flagsToStr(n.loc.flags),
-        $lineInfoToStr(conf, n.info), $n.kind])
+  this.closeCurly
 
-  proc debug(n: PType; conf: ConfigRef) =
-    echo($debugType(conf, n))
 
-  proc debug(n: PNode; conf: ConfigRef) =
-    echo($debugTree(conf, n, 0, 100))
+proc debug(n: PSym; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
+
+proc debug(n: PType; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
+
+proc debug(n: PNode; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  #this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
 
 proc nextTry(h, maxHash: Hash): Hash =
   result = ((5 * h) + 1) and maxHash
@@ -460,13 +678,13 @@ proc objectSetRawInsert(data: var TObjectSeq, obj: RootRef) =
 
 proc objectSetEnlarge(t: var TObjectSet) =
   var n: TObjectSeq
-  newSeq(n, len(t.data) * GrowthFactor)
-  for i in countup(0, high(t.data)):
+  newSeq(n, t.data.len * GrowthFactor)
+  for i in 0..high(t.data):
     if t.data[i] != nil: objectSetRawInsert(n, t.data[i])
   swap(t.data, n)
 
 proc objectSetIncl*(t: var TObjectSet, obj: RootRef) =
-  if mustRehash(len(t.data), t.counter): objectSetEnlarge(t)
+  if mustRehash(t.data.len, t.counter): objectSetEnlarge(t)
   objectSetRawInsert(t.data, obj)
   inc(t.counter)
 
@@ -479,7 +697,7 @@ proc objectSetContainsOrIncl*(t: var TObjectSet, obj: RootRef): bool =
     if it == obj:
       return true             # found it
     h = nextTry(h, high(t.data))
-  if mustRehash(len(t.data), t.counter):
+  if mustRehash(t.data.len, t.counter):
     objectSetEnlarge(t)
     objectSetRawInsert(t.data, obj)
   else:
@@ -496,35 +714,18 @@ proc strTableContains*(t: TStrTable, n: PSym): bool =
     h = nextTry(h, high(t.data))
   result = false
 
-proc strTableRawInsert(data: var TSymSeq, n: PSym) =
+proc strTableRawInsert(data: var seq[PSym], n: PSym) =
   var h: Hash = n.name.h and high(data)
-  if sfImmediate notin n.flags:
-    # fast path:
-    while data[h] != nil:
-      if data[h] == n:
-        # allowed for 'export' feature:
-        #InternalError(n.info, "StrTableRawInsert: " & n.name.s)
-        return
-      h = nextTry(h, high(data))
-    assert(data[h] == nil)
-    data[h] = n
-  else:
-    # slow path; we have to ensure immediate symbols are preferred for
-    # symbol lookups.
-    # consider the chain: foo (immediate), foo, bar, bar (immediate)
-    # then bar (immediate) gets replaced with foo (immediate) and the non
-    # immediate foo is picked! Thus we need to replace it with the first
-    # slot that has in fact the same identifier stored in it!
-    var favPos = -1
-    while data[h] != nil:
-      if data[h] == n: return
-      if favPos < 0 and data[h].name.id == n.name.id: favPos = h
-      h = nextTry(h, high(data))
-    assert(data[h] == nil)
-    data[h] = n
-    if favPos >= 0: swap data[h], data[favPos]
+  while data[h] != nil:
+    if data[h] == n:
+      # allowed for 'export' feature:
+      #InternalError(n.info, "StrTableRawInsert: " & n.name.s)
+      return
+    h = nextTry(h, high(data))
+  assert(data[h] == nil)
+  data[h] = n
 
-proc symTabReplaceRaw(data: var TSymSeq, prevSym: PSym, newSym: PSym) =
+proc symTabReplaceRaw(data: var seq[PSym], prevSym: PSym, newSym: PSym) =
   assert prevSym.name.h == newSym.name.h
   var h: Hash = prevSym.name.h and high(data)
   while data[h] != nil:
@@ -538,14 +739,14 @@ proc symTabReplace*(t: var TStrTable, prevSym: PSym, newSym: PSym) =
   symTabReplaceRaw(t.data, prevSym, newSym)
 
 proc strTableEnlarge(t: var TStrTable) =
-  var n: TSymSeq
-  newSeq(n, len(t.data) * GrowthFactor)
-  for i in countup(0, high(t.data)):
+  var n: seq[PSym]
+  newSeq(n, t.data.len * GrowthFactor)
+  for i in 0..high(t.data):
     if t.data[i] != nil: strTableRawInsert(n, t.data[i])
   swap(t.data, n)
 
 proc strTableAdd*(t: var TStrTable, n: PSym) =
-  if mustRehash(len(t.data), t.counter): strTableEnlarge(t)
+  if mustRehash(t.data.len, t.counter): strTableEnlarge(t)
   strTableRawInsert(t.data, n)
   inc(t.counter)
 
@@ -572,7 +773,7 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym;
     if not onConflictKeepOld:
       t.data[replaceSlot] = n # overwrite it with newer definition!
     return t.data[replaceSlot] # found it
-  elif mustRehash(len(t.data), t.counter):
+  elif mustRehash(t.data.len, t.counter):
     strTableEnlarge(t)
     strTableRawInsert(t.data, n)
   else:
@@ -676,7 +877,7 @@ iterator items*(tab: TStrTable): PSym =
     s = nextIter(it, tab)
 
 proc hasEmptySlot(data: TIdPairSeq): bool =
-  for h in countup(0, high(data)):
+  for h in 0..high(data):
     if data[h].key == nil:
       return true
   result = false
@@ -729,9 +930,9 @@ proc idTablePut(t: var TIdTable, key: PIdObj, val: RootRef) =
     assert(t.data[index].key != nil)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in countup(0, high(t.data)):
+    if mustRehash(t.data.len, t.counter):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(t.data):
         if t.data[i].key != nil:
           idTableRawInsert(n, t.data[i].key, t.data[i].val)
       assert(hasEmptySlot(n))
@@ -740,7 +941,7 @@ proc idTablePut(t: var TIdTable, key: PIdObj, val: RootRef) =
     inc(t.counter)
 
 iterator idTablePairs*(t: TIdTable): tuple[key: PIdObj, val: RootRef] =
-  for i in 0 .. high(t.data):
+  for i in 0..high(t.data):
     if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val)
 
 proc idNodeTableRawGet(t: TIdNodeTable, key: PIdObj): int =
@@ -774,10 +975,10 @@ proc idNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) =
     assert(t.data[index].key != nil)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
+    if mustRehash(t.data.len, t.counter):
       var n: TIdNodePairSeq
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in countup(0, high(t.data)):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(t.data):
         if t.data[i].key != nil:
           idNodeTableRawInsert(n, t.data[i].key, t.data[i].val)
       swap(t.data, n)
@@ -785,13 +986,13 @@ proc idNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) =
     inc(t.counter)
 
 iterator pairs*(t: TIdNodeTable): tuple[key: PIdObj, val: PNode] =
-  for i in 0 .. high(t.data):
+  for i in 0..high(t.data):
     if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val)
 
 proc initIITable(x: var TIITable) =
   x.counter = 0
   newSeq(x.data, StartSize)
-  for i in countup(0, StartSize - 1): x.data[i].key = InvalidKey
+  for i in 0..<StartSize: x.data[i].key = InvalidKey
 
 proc iiTableRawGet(t: TIITable, key: int): int =
   var h: Hash
@@ -822,13 +1023,28 @@ proc iiTablePut(t: var TIITable, key, val: int) =
     assert(t.data[index].key != InvalidKey)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
+    if mustRehash(t.data.len, t.counter):
       var n: TIIPairSeq
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in countup(0, high(n)): n[i].key = InvalidKey
-      for i in countup(0, high(t.data)):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(n): n[i].key = InvalidKey
+      for i in 0..high(t.data):
         if t.data[i].key != InvalidKey:
           iiTableRawInsert(n, t.data[i].key, t.data[i].val)
       swap(t.data, n)
     iiTableRawInsert(t.data, key, val)
     inc(t.counter)
+
+proc isAddrNode*(n: PNode): bool =
+  case n.kind
+    of nkAddr, nkHiddenAddr: true
+    of nkCallKinds:
+      if n[0].kind == nkSym and n[0].sym.magic == mAddr: true
+      else: false
+    else: false
+
+proc listSymbolNames*(symbols: openArray[PSym]): string =
+  for sym in symbols:
+    if result.len > 0:
+      result.add ", "
+    result.add sym.name.s
+

compiler/bitsets.nim

@@ -11,12 +11,17 @@
 # the code here should be reused in the Nim standard library
 
 type
-  TBitSet* = seq[int8]        # we use byte here to avoid issues with
+  ElemType = byte
+  TBitSet* = seq[ElemType]    # we use byte here to avoid issues with
                               # cross-compiling; uint would be more efficient
                               # however
-
 const
-  ElemSize* = sizeof(int8) * 8
+  ElemSize* = 8
+  One = ElemType(1)
+  Zero = ElemType(0)
+
+template modElemSize(arg: untyped): untyped = arg and 7
+template divElemSize(arg: untyped): untyped = arg shr 3
 
 proc bitSetInit*(b: var TBitSet, length: int)
 proc bitSetUnion*(x: var TBitSet, y: TBitSet)
@@ -32,66 +37,65 @@ proc bitSetCard*(x: TBitSet): BiggestInt
 # implementation
 
 proc bitSetIn(x: TBitSet, e: BiggestInt): bool =
-  result = (x[int(e div ElemSize)] and toU8(int(1 shl (e mod ElemSize)))) !=
-      toU8(0)
+  result = (x[int(e.divElemSize)] and (One shl e.modElemSize)) != Zero
 
 proc bitSetIncl(x: var TBitSet, elem: BiggestInt) =
   assert(elem >= 0)
-  x[int(elem div ElemSize)] = x[int(elem div ElemSize)] or
-      toU8(int(1 shl (elem mod ElemSize)))
+  x[int(elem.divElemSize)] = x[int(elem.divElemSize)] or
+      (One shl elem.modElemSize)
 
 proc bitSetExcl(x: var TBitSet, elem: BiggestInt) =
-  x[int(elem div ElemSize)] = x[int(elem div ElemSize)] and
-      not toU8(int(1 shl (elem mod ElemSize)))
+  x[int(elem.divElemSize)] = x[int(elem.divElemSize)] and
+      not(One shl elem.modElemSize)
 
 proc bitSetInit(b: var TBitSet, length: int) =
   newSeq(b, length)
 
 proc bitSetUnion(x: var TBitSet, y: TBitSet) =
-  for i in countup(0, high(x)): x[i] = x[i] or y[i]
+  for i in 0..high(x): x[i] = x[i] or y[i]
 
 proc bitSetDiff(x: var TBitSet, y: TBitSet) =
-  for i in countup(0, high(x)): x[i] = x[i] and not y[i]
+  for i in 0..high(x): x[i] = x[i] and not y[i]
 
 proc bitSetSymDiff(x: var TBitSet, y: TBitSet) =
-  for i in countup(0, high(x)): x[i] = x[i] xor y[i]
+  for i in 0..high(x): x[i] = x[i] xor y[i]
 
 proc bitSetIntersect(x: var TBitSet, y: TBitSet) =
-  for i in countup(0, high(x)): x[i] = x[i] and y[i]
+  for i in 0..high(x): x[i] = x[i] and y[i]
 
 proc bitSetEquals(x, y: TBitSet): bool =
-  for i in countup(0, high(x)):
+  for i in 0..high(x):
     if x[i] != y[i]:
       return false
   result = true
 
 proc bitSetContains(x, y: TBitSet): bool =
-  for i in countup(0, high(x)):
-    if (x[i] and not y[i]) != int8(0):
+  for i in 0..high(x):
+    if (x[i] and not y[i]) != Zero:
       return false
   result = true
 
 # Number of set bits for all values of int8
-const populationCount: array[low(int8)..high(int8), int8] = block:
-    var arr: array[low(int8)..high(int8), int8]
+const populationCount: array[uint8, uint8] = block:
+    var arr: array[uint8, uint8]
 
-    proc countSetBits(x: int8): int8 =
+    proc countSetBits(x: uint8): uint8 =
       return
-        ( x and 0b00000001'i8) +
-        ((x and 0b00000010'i8) shr 1) +
-        ((x and 0b00000100'i8) shr 2) +
-        ((x and 0b00001000'i8) shr 3) +
-        ((x and 0b00010000'i8) shr 4) +
-        ((x and 0b00100000'i8) shr 5) +
-        ((x and 0b01000000'i8) shr 6) +
-        ((x and 0b10000000'i8) shr 7)
-        
+        ( x and 0b00000001'u8) +
+        ((x and 0b00000010'u8) shr 1) +
+        ((x and 0b00000100'u8) shr 2) +
+        ((x and 0b00001000'u8) shr 3) +
+        ((x and 0b00010000'u8) shr 4) +
+        ((x and 0b00100000'u8) shr 5) +
+        ((x and 0b01000000'u8) shr 6) +
+        ((x and 0b10000000'u8) shr 7)
 
-    for it in low(int8)..high(int8):
-      arr[it] = countSetBits(it)
+
+    for it in low(uint8)..high(uint8):
+      arr[it] = countSetBits(cast[uint8](it))
 
     arr
 
 proc bitSetCard(x: TBitSet): BiggestInt =
   for it in x:
-    result.inc populationCount[it]
+    result.inc int(populationCount[it])

compiler/btrees.nim

@@ -37,34 +37,34 @@ template eq(a, b): bool = cmp(a, b) == 0
 proc getOrDefault*[Key, Val](b: BTree[Key, Val], key: Key): Val =
   var x = b.root
   while x.isInternal:
-    for j in 0 ..< x.entries:
+    for j in 0..<x.entries:
       if j+1 == x.entries or less(key, x.keys[j+1]):
         x = x.links[j]
         break
   assert(not x.isInternal)
-  for j in 0 ..< x.entries:
+  for j in 0..<x.entries:
     if eq(key, x.keys[j]): return x.vals[j]
 
 proc contains*[Key, Val](b: BTree[Key, Val], key: Key): bool =
   var x = b.root
   while x.isInternal:
-    for j in 0 ..< x.entries:
+    for j in 0..<x.entries:
       if j+1 == x.entries or less(key, x.keys[j+1]):
         x = x.links[j]
         break
   assert(not x.isInternal)
-  for j in 0 ..< x.entries:
+  for j in 0..<x.entries:
     if eq(key, x.keys[j]): return true
   return false
 
 proc copyHalf[Key, Val](h, result: Node[Key, Val]) =
-  for j in 0 ..< Mhalf:
+  for j in 0..<Mhalf:
     result.keys[j] = h.keys[Mhalf + j]
   if h.isInternal:
-    for j in 0 ..< Mhalf:
+    for j in 0..<Mhalf:
       result.links[j] = h.links[Mhalf + j]
   else:
-    for j in 0 ..< Mhalf:
+    for j in 0..<Mhalf:
       shallowCopy(result.vals[j], h.vals[Mhalf + j])
 
 proc split[Key, Val](h: Node[Key, Val]): Node[Key, Val] =
@@ -191,7 +191,7 @@ when isMainModule:
     st.add("www.yahoo.com",        "216.109.118.65")
 
     assert st.getOrDefault("www.cs.princeton.edu") == "abc"
-    assert st.getOrDefault("www.harvardsucks.com") == nil
+    assert st.getOrDefault("www.harvardsucks.com") == ""
 
     assert st.getOrDefault("www.simpsons.com") == "209.052.165.60"
     assert st.getOrDefault("www.apple.com") == "17.112.152.32"

compiler/canonicalizer.nim

@@ -102,7 +102,7 @@ proc hashTree(c: var MD5Context, n: PNode) =
   of nkStrLit..nkTripleStrLit:
     c &= n.strVal
   else:
-    for i in 0..<n.len: hashTree(c, n.sons[i])
+    for i in 0..<n.len: hashTree(c, n[i])
 
 proc hashType(c: var MD5Context, t: PType) =
   # modelled after 'typeToString'
@@ -120,44 +120,44 @@ proc hashType(c: var MD5Context, t: PType) =
 
   case t.kind
   of tyGenericBody, tyGenericInst, tyGenericInvocation:
-    for i in countup(0, sonsLen(t) -1 -ord(t.kind != tyGenericInvocation)):
-      c.hashType t.sons[i]
+    for i in 0..<t.len-ord(t.kind != tyGenericInvocation):
+      c.hashType t[i]
   of tyUserTypeClass:
     internalAssert t.sym != nil and t.sym.owner != nil
     c &= t.sym.owner.name.s
   of tyUserTypeClassInst:
     let body = t.base
     c.hashSym body.sym
-    for i in countup(1, sonsLen(t) - 2):
-      c.hashType t.sons[i]
+    for i in 1..<t.len-1:
+      c.hashType t[i]
   of tyFromExpr:
     c.hashTree(t.n)
   of tyArray:
-    c.hashTree(t.sons[0].n)
-    c.hashType(t.sons[1])
+    c.hashTree(t[0].n)
+    c.hashType(t[1])
   of tyTuple:
     if t.n != nil:
-      assert(sonsLen(t.n) == sonsLen(t))
-      for i in countup(0, sonsLen(t.n) - 1):
-        assert(t.n.sons[i].kind == nkSym)
-        c &= t.n.sons[i].sym.name.s
+      assert(t.n.len == t.len)
+      for i in 0..<t.n.len:
+        assert(t.n[i].kind == nkSym)
+        c &= t.n[i].sym.name.s
         c &= ":"
-        c.hashType(t.sons[i])
+        c.hashType(t[i])
         c &= ","
     else:
-      for i in countup(0, sonsLen(t) - 1): c.hashType t.sons[i]
+      for i in 0..<t.len: c.hashType t[i]
   of tyRange:
     c.hashTree(t.n)
-    c.hashType(t.sons[0])
+    c.hashType(t[0])
   of tyProc:
     c &= (if tfIterator in t.flags: "iterator " else: "proc ")
-    for i in 0..<t.len: c.hashType(t.sons[i])
+    for i in 0..<t.len: c.hashType(t[i])
     md5Update(c, cast[cstring](addr(t.callConv)), 1)
 
     if tfNoSideEffect in t.flags: c &= ".noSideEffect"
     if tfThread in t.flags: c &= ".thread"
   else:
-    for i in 0..<t.len: c.hashType(t.sons[i])
+    for i in 0..<t.len: c.hashType(t[i])
   if tfNotNil in t.flags: c &= "not nil"
 
 proc canonConst(n: PNode): TUid =
@@ -238,35 +238,35 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
     encodeVInt(n.sym.id, result)
     pushSym(w, n.sym)
   else:
-    for i in countup(0, sonsLen(n) - 1):
-      encodeNode(w, n.info, n.sons[i], result)
-  add(result, ')')
+    for i in 0..<n.len:
+      encodeNode(w, n.info, n[i], result)
+  result.add(')')
 
 proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
   var oldLen = result.len
   result.add('<')
   if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
   if loc.s != low(loc.s):
-    add(result, '*')
+    result.add('*')
     encodeVInt(ord(loc.s), result)
   if loc.flags != {}:
-    add(result, '$')
+    result.add('$')
     encodeVInt(cast[int32](loc.flags), result)
   if loc.t != nil:
-    add(result, '^')
+    result.add('^')
     encodeVInt(cast[int32](loc.t.id), result)
     pushType(w, loc.t)
   if loc.r != nil:
-    add(result, '!')
+    result.add('!')
     encodeStr($loc.r, result)
   if loc.a != 0:
-    add(result, '?')
+    result.add('?')
     encodeVInt(loc.a, result)
   if oldLen + 1 == result.len:
     # no data was necessary, so remove the '<' again:
     setLen(result, oldLen)
   else:
-    add(result, '>')
+    result.add('>')
 
 proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t == nil:
@@ -277,47 +277,47 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t.kind == tyForward: internalError("encodeType: tyForward")
   # for the new rodfile viewer we use a preceding [ so that the data section
   # can easily be disambiguated:
-  add(result, '[')
+  result.add('[')
   encodeVInt(ord(t.kind), result)
-  add(result, '+')
+  result.add('+')
   encodeVInt(t.id, result)
   if t.n != nil:
-    encodeNode(w, unknownLineInfo(), t.n, result)
+    encodeNode(w, unknownLineInfo, t.n, result)
   if t.flags != {}:
-    add(result, '$')
+    result.add('$')
     encodeVInt(cast[int32](t.flags), result)
   if t.callConv != low(t.callConv):
-    add(result, '?')
+    result.add('?')
     encodeVInt(ord(t.callConv), result)
   if t.owner != nil:
-    add(result, '*')
+    result.add('*')
     encodeVInt(t.owner.id, result)
     pushSym(w, t.owner)
   if t.sym != nil:
-    add(result, '&')
+    result.add('&')
     encodeVInt(t.sym.id, result)
     pushSym(w, t.sym)
   if t.size != - 1:
-    add(result, '/')
+    result.add('/')
     encodeVBiggestInt(t.size, result)
-  if t.align != 2:
-    add(result, '=')
+  if t.align != - 1:
+    result.add('=')
     encodeVInt(t.align, result)
   encodeLoc(w, t.loc, result)
-  for i in countup(0, sonsLen(t) - 1):
-    if t.sons[i] == nil:
-      add(result, "^()")
+  for i in 0..<t.len:
+    if t[i] == nil:
+      result.add("^()")
     else:
-      add(result, '^')
-      encodeVInt(t.sons[i].id, result)
-      pushType(w, t.sons[i])
+      result.add('^')
+      encodeVInt(t[i].id, result)
+      pushType(w, t[i])
 
 proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) =
-  add(result, '|')
+  result.add('|')
   encodeVInt(ord(lib.kind), result)
-  add(result, '|')
+  result.add('|')
   encodeStr($lib.name, result)
-  add(result, '|')
+  result.add('|')
   encodeNode(w, info, lib.path, result)
 
 proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
@@ -363,8 +363,8 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
   encodeLoc(w, s.loc, result)
   if s.annex != nil: encodeLib(w, s.annex, s.info, result)
   if s.constraint != nil:
-    add(result, '#')
-    encodeNode(w, unknownLineInfo(), s.constraint, result)
+    result.add('#')
+    encodeNode(w, unknownLineInfo, s.constraint, result)
   # lazy loading will soon reload the ast lazily, so the ast needs to be
   # the last entry of a symbol:
   if s.ast != nil:

compiler/ccgcalls.nim

@@ -9,88 +9,126 @@
 #
 # included from cgen.nim
 
+proc canRaiseDisp(p: BProc; n: PNode): bool =
+  # we assume things like sysFatal cannot raise themselves
+  if n.kind == nkSym and {sfNeverRaises, sfImportc, sfCompilerProc} * n.sym.flags != {}:
+    result = false
+  elif optPanics in p.config.globalOptions or
+      (n.kind == nkSym and sfSystemModule in getModule(n.sym).flags):
+    # we know we can be strict:
+    result = canRaise(n)
+  else:
+    # we have to be *very* conservative:
+    result = canRaiseConservative(n)
+
 proc leftAppearsOnRightSide(le, ri: PNode): bool =
   if le != nil:
-    for i in 1 ..< ri.len:
+    for i in 1..<ri.len:
       let r = ri[i]
       if isPartOf(le, r) != arNo: return true
 
 proc hasNoInit(call: PNode): bool {.inline.} =
-  result = call.sons[0].kind == nkSym and sfNoInit in call.sons[0].sym.flags
+  result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
+
+proc isHarmlessStore(p: BProc; canRaise: bool; d: TLoc): bool =
+  if d.k in {locTemp, locNone} or not canRaise:
+    result = true
+  elif d.k == locLocalVar and p.withinTryWithExcept == 0:
+    # we cannot observe a store to a local variable if the current proc
+    # has no error handler:
+    result = true
+  else:
+    result = false
 
 proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
                callee, params: Rope) =
+  let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
+  genLineDir(p, ri)
   var pl = callee & ~"(" & params
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
+  var typ = skipTypes(ri[0].typ, abstractInst)
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
       if params != nil: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
       if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
         # Great, we can use 'd':
-        if d.k == locNone: getTemp(p, typ.sons[0], d, needsInit=true)
+        if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
           discard "resetLoc(p, d)"
-        add(pl, addrLoc(p.config, d))
-        add(pl, ~");$n")
+        pl.add(addrLoc(p.config, d))
+        pl.add(~");$n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
-        add(pl, ~");$n")
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
+        pl.add(~");$n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
+      if canRaise: raiseExit(p)
     else:
-      add(pl, ~")")
-      if p.module.compileToCpp and lfSingleUse in d.flags:
-        # do not generate spurious temporaries for C++! For C we're better off
-        # with them to prevent undefined behaviour and because the codegen
-        # is free to emit expressions multiple times!
-        d.k = locCall
-        d.r = pl
-        excl d.flags, lfSingleUse
-      else:
-        if d.k == locNone: getTemp(p, typ.sons[0], d)
+      pl.add(~")")
+      if p.module.compileToCpp:
+        if lfSingleUse in d.flags:
+          # do not generate spurious temporaries for C++! For C we're better off
+          # with them to prevent undefined behaviour and because the codegen
+          # is free to emit expressions multiple times!
+          d.k = locCall
+          d.r = pl
+          excl d.flags, lfSingleUse
+        else:
+          if d.k == locNone and p.splitDecls == 0:
+            getTempCpp(p, typ[0], d, pl)
+          else:
+            if d.k == locNone: getTemp(p, typ[0], d)
+            var list: TLoc
+            initLoc(list, locCall, d.lode, OnUnknown)
+            list.r = pl
+            genAssignment(p, d, list, {}) # no need for deep copying
+            if canRaise: raiseExit(p)
+
+      elif isHarmlessStore(p, canRaise, d):
+        if d.k == locNone: getTemp(p, typ[0], d)
         assert(d.t != nil)        # generate an assignment to d:
         var list: TLoc
         initLoc(list, locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
+        if canRaise: raiseExit(p)
+      else:
+        var tmp: TLoc
+        getTemp(p, typ[0], tmp, needsInit=true)
+        var list: TLoc
+        initLoc(list, locCall, d.lode, OnUnknown)
+        list.r = pl
+        genAssignment(p, tmp, list, {}) # no need for deep copying
+        if canRaise: raiseExit(p)
+        genAssignment(p, d, tmp, {})
   else:
-    add(pl, ~");$n")
+    pl.add(~");$n")
     line(p, cpsStmts, pl)
-
-proc isInCurrentFrame(p: BProc, n: PNode): bool =
-  # checks if `n` is an expression that refers to the current frame;
-  # this does not work reliably because of forwarding + inlining can break it
-  case n.kind
-  of nkSym:
-    if n.sym.kind in {skVar, skResult, skTemp, skLet} and p.prc != nil:
-      result = p.prc.id == n.sym.owner.id
-  of nkDotExpr, nkBracketExpr:
-    if skipTypes(n.sons[0].typ, abstractInst).kind notin {tyVar,tyLent,tyPtr,tyRef}:
-      result = isInCurrentFrame(p, n.sons[0])
-  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-    result = isInCurrentFrame(p, n.sons[1])
-  of nkHiddenDeref, nkDerefExpr:
-    # what about: var x = addr(y); callAsOpenArray(x[])?
-    # *shrug* ``addr`` is unsafe anyway.
-    result = false
-  of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
-    result = isInCurrentFrame(p, n.sons[0])
-  else: discard
+    if canRaise: raiseExit(p)
 
 proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
 
 proc openArrayLoc(p: BProc, n: PNode): Rope =
   var a: TLoc
 
-  let q = skipConv(n)
+  var q = skipConv(n)
+  var skipped = false
+  while q.kind == nkStmtListExpr and q.len > 0:
+    skipped = true
+    q = q.lastSon
   if getMagic(q) == mSlice:
     # magic: pass slice to openArray:
+    if skipped:
+      q = skipConv(n)
+      while q.kind == nkStmtListExpr and q.len > 0:
+        for i in 0..<q.len-1:
+          genStmts(p, q[i])
+        q = q.lastSon
     var b, c: TLoc
     initLocExpr(p, q[1], a)
     initLocExpr(p, q[2], b)
@@ -99,21 +137,23 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
     if optBoundsCheck in p.options:
       genBoundsCheck(p, a, b, c)
     let ty = skipTypes(a.t, abstractVar+{tyPtr})
+    let dest = getTypeDesc(p.module, n.typ[0])
     case ty.kind
     of tyArray:
-      let first = firstOrd(p.config, ty)
+      let first = toInt64(firstOrd(p.config, ty))
       if first == 0:
-        result = "($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c)]
+        result = "($4*)(($1)+($2)), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dest]
       else:
-        result = "($1)+(($2)-($4)), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), intLiteral(first)]
-    of tyOpenArray, tyVarargs:
-      result = "($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c)]
+        result = "($5*)($1)+(($2)-($4)), ($3)-($2)+1" %
+          [rdLoc(a), rdLoc(b), rdLoc(c), intLiteral(first), dest]
+    of tyOpenArray, tyVarargs, tyUncheckedArray, tyCString:
+      result = "($4*)($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dest]
     of tyString, tySequence:
       if skipTypes(n.typ, abstractInst).kind == tyVar and
           not compileToCpp(p.module):
-        result = "(*$1)$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p)]
+        result = "($5*)(*$1)$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
       else:
-        result = "$1$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p)]
+        result = "($5*)$1$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
     else:
       internalError(p.config, "openArrayLoc: " & typeToString(a.t))
   else:
@@ -145,7 +185,7 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
 
 proc genArgStringToCString(p: BProc, n: PNode): Rope {.inline.} =
   var a: TLoc
-  initLocExpr(p, n.sons[0], a)
+  initLocExpr(p, n[0], a)
   result = ropecg(p.module, "#nimToCStringConv($1)", [a.rdLoc])
 
 proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
@@ -153,19 +193,19 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
   if n.kind == nkStringToCString:
     result = genArgStringToCString(p, n)
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
-    var n = if n.kind != nkHiddenAddr: n else: n.sons[0]
+    var n = if n.kind != nkHiddenAddr: n else: n[0]
     result = openArrayLoc(p, n)
-  elif ccgIntroducedPtr(p.config, param):
+  elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
     result = addrLoc(p.config, a)
   elif p.module.compileToCpp and param.typ.kind == tyVar and
       n.kind == nkHiddenAddr:
-    initLocExprSingleUse(p, n.sons[0], a)
+    initLocExprSingleUse(p, n[0], a)
     # if the proc is 'importc'ed but not 'importcpp'ed then 'var T' still
     # means '*T'. See posix.nim for lots of examples that do that in the wild.
-    let callee = call.sons[0]
+    let callee = call[0]
     if callee.kind == nkSym and
-        {sfImportC, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportC} and
+        {sfImportc, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportc} and
         {lfHeader, lfNoDecl} * callee.sym.loc.flags != {}:
       result = addrLoc(p.config, a)
     else:
@@ -183,29 +223,36 @@ proc genArgNoParam(p: BProc, n: PNode): Rope =
     result = rdLoc(a)
 
 template genParamLoop(params) {.dirty.} =
-  if i < sonsLen(typ):
-    assert(typ.n.sons[i].kind == nkSym)
-    let paramType = typ.n.sons[i]
+  if i < typ.len:
+    assert(typ.n[i].kind == nkSym)
+    let paramType = typ.n[i]
     if not paramType.typ.isCompileTimeOnly:
-      if params != nil: add(params, ~", ")
-      add(params, genArg(p, ri.sons[i], paramType.sym, ri))
+      if params != nil: params.add(~", ")
+      params.add(genArg(p, ri[i], paramType.sym, ri))
   else:
-    if params != nil: add(params, ~", ")
-    add(params, genArgNoParam(p, ri.sons[i]))
+    if params != nil: params.add(~", ")
+    params.add(genArgNoParam(p, ri[i]))
+
+proc addActualSuffixForHCR(res: var Rope, module: PSym, sym: PSym) =
+  if sym.flags * {sfImportc, sfNonReloadable} == {} and sym.loc.k == locProc and
+      (sym.typ.callConv == ccInline or sym.owner.id == module.id):
+    res = res & "_actual".rope
 
 proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var op: TLoc
   # this is a hotspot in the compiler
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   var params: Rope
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var typ = skipTypes(ri[0].typ, abstractInstOwned)
   assert(typ.kind == tyProc)
-  assert(sonsLen(typ) == sonsLen(typ.n))
-  var length = sonsLen(ri)
-  for i in countup(1, length - 1):
+  assert(typ.len == typ.n.len)
+  for i in 1..<ri.len:
     genParamLoop(params)
-  fixupCall(p, le, ri, d, op.r, params)
+  var callee = rdLoc(op)
+  if p.hcrOn and ri[0].kind == nkSym:
+    callee.addActualSuffixForHCR(p.module.module, ri[0].sym)
+  fixupCall(p, le, ri, d, callee, params)
 
 proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
 
@@ -218,68 +265,89 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
   const PatProc = "$1.ClE_0? $1.ClP_0($3$1.ClE_0):(($4)($1.ClP_0))($2)"
   const PatIter = "$1.ClP_0($3$1.ClE_0)" # we know the env exists
   var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   var pl: Rope
 
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
-  var length = sonsLen(ri)
-  for i in countup(1, length - 1):
-    assert(sonsLen(typ) == sonsLen(typ.n))
+  for i in 1..<ri.len:
+    assert(typ.len == typ.n.len)
     genParamLoop(pl)
 
   template genCallPattern {.dirty.} =
-    lineF(p, cpsStmts, callPattern & ";$n", [op.r, pl, pl.addComma, rawProc])
+    if tfIterator in typ.flags:
+      lineF(p, cpsStmts, PatIter & ";$n", [rdLoc(op), pl, pl.addComma, rawProc])
+    else:
+      lineF(p, cpsStmts, PatProc & ";$n", [rdLoc(op), pl, pl.addComma, rawProc])
 
   let rawProc = getRawProcType(p, typ)
-  let callPattern = if tfIterator in typ.flags: PatIter else: PatProc
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
-      if sonsLen(ri) > 1: add(pl, ~", ")
+  let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
+      if ri.len > 1: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
       if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
-          getTemp(p, typ.sons[0], d, needsInit=true)
+          getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
           discard "resetLoc(p, d)"
-        add(pl, addrLoc(p.config, d))
+        pl.add(addrLoc(p.config, d))
         genCallPattern()
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
         genCallPattern()
+        if canRaise: raiseExit(p)
         genAssignment(p, d, tmp, {}) # no need for deep copying
-    else:
-      if d.k == locNone: getTemp(p, typ.sons[0], d)
+    elif isHarmlessStore(p, canRaise, d):
+      if d.k == locNone: getTemp(p, typ[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
       initLoc(list, locCall, d.lode, OnUnknown)
-      list.r = callPattern % [op.r, pl, pl.addComma, rawProc]
+      if tfIterator in typ.flags:
+        list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
+      else:
+        list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
       genAssignment(p, d, list, {}) # no need for deep copying
+      if canRaise: raiseExit(p)
+    else:
+      var tmp: TLoc
+      getTemp(p, typ[0], tmp)
+      assert(d.t != nil)        # generate an assignment to d:
+      var list: TLoc
+      initLoc(list, locCall, d.lode, OnUnknown)
+      if tfIterator in typ.flags:
+        list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
+      else:
+        list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
+      genAssignment(p, tmp, list, {})
+      if canRaise: raiseExit(p)
+      genAssignment(p, d, tmp, {})
   else:
     genCallPattern()
+    if canRaise: raiseExit(p)
 
 proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
-  if i < sonsLen(typ):
+  if i < typ.len:
     # 'var T' is 'T&' in C++. This means we ignore the request of
     # any nkHiddenAddr when it's a 'var T'.
-    let paramType = typ.n.sons[i]
+    let paramType = typ.n[i]
     assert(paramType.kind == nkSym)
     if paramType.typ.isCompileTimeOnly:
       result = nil
-    elif typ.sons[i].kind == tyVar and ri.sons[i].kind == nkHiddenAddr:
-      result = genArgNoParam(p, ri.sons[i][0])
+    elif typ[i].kind == tyVar and ri[i].kind == nkHiddenAddr:
+      result = genArgNoParam(p, ri[i][0])
     else:
-      result = genArgNoParam(p, ri.sons[i]) #, typ.n.sons[i].sym)
+      result = genArgNoParam(p, ri[i]) #, typ.n[i].sym)
   else:
     if tfVarargs notin typ.flags:
       localError(p.config, ri.info, "wrong argument count")
       result = nil
     else:
-      result = genArgNoParam(p, ri.sons[i])
+      result = genArgNoParam(p, ri[i])
 
 discard """
 Dot call syntax in C++
@@ -323,16 +391,16 @@ proc skipAddrDeref(node: PNode): PNode =
   var isAddr = false
   case n.kind
   of nkAddr, nkHiddenAddr:
-    n = n.sons[0]
+    n = n[0]
     isAddr = true
   of nkDerefExpr, nkHiddenDeref:
-    n = n.sons[0]
+    n = n[0]
   else: return n
-  if n.kind == nkObjDownConv: n = n.sons[0]
+  if n.kind == nkObjDownConv: n = n[0]
   if isAddr and n.kind in {nkDerefExpr, nkHiddenDeref}:
-    result = n.sons[0]
+    result = n[0]
   elif n.kind in {nkAddr, nkHiddenAddr}:
-    result = n.sons[0]
+    result = n[0]
   else:
     result = node
 
@@ -340,13 +408,13 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   # for better or worse c2nim translates the 'this' argument to a 'var T'.
   # However manual wrappers may also use 'ptr T'. In any case we support both
   # for convenience.
-  internalAssert p.config, i < sonsLen(typ)
-  assert(typ.n.sons[i].kind == nkSym)
+  internalAssert p.config, i < typ.len
+  assert(typ.n[i].kind == nkSym)
   # if the parameter is lying (tyVar) and thus we required an additional deref,
   # skip the deref:
   var ri = ri[i]
   while ri.kind == nkObjDownConv: ri = ri[0]
-  let t = typ.sons[i].skipTypes({tyGenericInst, tyAlias, tySink})
+  let t = typ[i].skipTypes({tyGenericInst, tyAlias, tySink})
   if t.kind == tyVar:
     let x = if ri.kind == nkHiddenAddr: ri[0] else: ri
     if x.typ.kind == tyPtr:
@@ -368,7 +436,7 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   else:
     ri = skipAddrDeref(ri)
     if ri.kind in {nkAddr, nkHiddenAddr}: ri = ri[0]
-    result = genArgNoParam(p, ri) #, typ.n.sons[i].sym)
+    result = genArgNoParam(p, ri) #, typ.n[i].sym)
     result.add(".")
 
 proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
@@ -377,33 +445,36 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
   while i < pat.len:
     case pat[i]
     of '@':
-      if j < ri.len:
-        result.add genOtherArg(p, ri, j, typ)
-        for k in j+1 ..< ri.len:
-          result.add(~", ")
-          result.add genOtherArg(p, ri, k, typ)
+      var first = true
+      for k in j..<ri.len:
+        let arg = genOtherArg(p, ri, k, typ)
+        if arg.len > 0:
+          if not first:
+            result.add(~", ")
+          first = false
+          result.add arg
       inc i
     of '#':
-      if pat[i+1] in {'+', '@'}:
+      if i+1 < pat.len and pat[i+1] in {'+', '@'}:
         let ri = ri[j]
         if ri.kind in nkCallKinds:
-          let typ = skipTypes(ri.sons[0].typ, abstractInst)
-          if pat[i+1] == '+': result.add genArgNoParam(p, ri.sons[0])
+          let typ = skipTypes(ri[0].typ, abstractInst)
+          if pat[i+1] == '+': result.add genArgNoParam(p, ri[0])
           result.add(~"(")
           if 1 < ri.len:
             result.add genOtherArg(p, ri, 1, typ)
-          for k in j+1 ..< ri.len:
+          for k in j+1..<ri.len:
             result.add(~", ")
             result.add genOtherArg(p, ri, k, typ)
           result.add(~")")
         else:
           localError(p.config, ri.info, "call expression expected for C++ pattern")
         inc i
-      elif pat[i+1] == '.':
+      elif i+1 < pat.len and pat[i+1] == '.':
         result.add genThisArg(p, ri, j, typ)
         inc i
-      elif pat[i+1] == '[':
-        var arg = ri.sons[j].skipAddrDeref
+      elif i+1 < pat.len and pat[i+1] == '[':
+        var arg = ri[j].skipAddrDeref
         while arg.kind in {nkAddr, nkHiddenAddr, nkObjDownConv}: arg = arg[0]
         result.add genArgNoParam(p, arg)
         #result.add debugTree(arg, 0, 10)
@@ -423,24 +494,23 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
         if pat[i] notin {'@', '#', '\''}: inc(i)
         else: break
       if i - 1 >= start:
-        add(result, substr(pat, start, i - 1))
+        result.add(substr(pat, start, i - 1))
 
 proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
-  var length = sonsLen(ri)
-  assert(sonsLen(typ) == sonsLen(typ.n))
+  assert(typ.len == typ.n.len)
   # don't call '$' here for efficiency:
-  let pat = ri.sons[0].sym.loc.r.data
+  let pat = ri[0].sym.loc.r.data
   internalAssert p.config, pat.len > 0
   if pat.contains({'#', '(', '@', '\''}):
     var pl = genPatternCall(p, ri, pat, typ)
     # simpler version of 'fixupCall' that works with the pl+params combination:
-    var typ = skipTypes(ri.sons[0].typ, abstractInst)
-    if typ.sons[0] != nil:
+    var typ = skipTypes(ri[0].typ, abstractInst)
+    if typ[0] != nil:
       if p.module.compileToCpp and lfSingleUse in d.flags:
         # do not generate spurious temporaries for C++! For C we're better off
         # with them to prevent undefined behaviour and because the codegen
@@ -449,115 +519,114 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
         d.r = pl
         excl d.flags, lfSingleUse
       else:
-        if d.k == locNone: getTemp(p, typ.sons[0], d)
+        if d.k == locNone: getTemp(p, typ[0], d)
         assert(d.t != nil)        # generate an assignment to d:
         var list: TLoc
         initLoc(list, locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
     else:
-      add(pl, ~";$n")
+      pl.add(~";$n")
       line(p, cpsStmts, pl)
   else:
     var pl: Rope = nil
-    #var param = typ.n.sons[1].sym
+    #var param = typ.n[1].sym
     if 1 < ri.len:
-      add(pl, genThisArg(p, ri, 1, typ))
-    add(pl, op.r)
+      pl.add(genThisArg(p, ri, 1, typ))
+    pl.add(op.r)
     var params: Rope
-    for i in countup(2, length - 1):
+    for i in 2..<ri.len:
       if params != nil: params.add(~", ")
-      assert(sonsLen(typ) == sonsLen(typ.n))
-      add(params, genOtherArg(p, ri, i, typ))
+      assert(typ.len == typ.n.len)
+      params.add(genOtherArg(p, ri, i, typ))
     fixupCall(p, le, ri, d, pl, params)
 
 proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
   # generates a crappy ObjC call
   var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   var pl = ~"["
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
-  var length = sonsLen(ri)
-  assert(sonsLen(typ) == sonsLen(typ.n))
+  assert(typ.len == typ.n.len)
 
   # don't call '$' here for efficiency:
-  let pat = ri.sons[0].sym.loc.r.data
+  let pat = ri[0].sym.loc.r.data
   internalAssert p.config, pat.len > 0
   var start = 3
   if ' ' in pat:
     start = 1
-    add(pl, op.r)
-    if length > 1:
-      add(pl, ~": ")
-      add(pl, genArg(p, ri.sons[1], typ.n.sons[1].sym, ri))
+    pl.add(op.r)
+    if ri.len > 1:
+      pl.add(~": ")
+      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
       start = 2
   else:
-    if length > 1:
-      add(pl, genArg(p, ri.sons[1], typ.n.sons[1].sym, ri))
-      add(pl, ~" ")
-    add(pl, op.r)
-    if length > 2:
-      add(pl, ~": ")
-      add(pl, genArg(p, ri.sons[2], typ.n.sons[2].sym, ri))
-  for i in countup(start, length-1):
-    assert(sonsLen(typ) == sonsLen(typ.n))
-    if i >= sonsLen(typ):
+    if ri.len > 1:
+      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
+      pl.add(~" ")
+    pl.add(op.r)
+    if ri.len > 2:
+      pl.add(~": ")
+      pl.add(genArg(p, ri[2], typ.n[2].sym, ri))
+  for i in start..<ri.len:
+    assert(typ.len == typ.n.len)
+    if i >= typ.len:
       internalError(p.config, ri.info, "varargs for objective C method?")
-    assert(typ.n.sons[i].kind == nkSym)
-    var param = typ.n.sons[i].sym
-    add(pl, ~" ")
-    add(pl, param.name.s)
-    add(pl, ~": ")
-    add(pl, genArg(p, ri.sons[i], param, ri))
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
-      if sonsLen(ri) > 1: add(pl, ~" ")
+    assert(typ.n[i].kind == nkSym)
+    var param = typ.n[i].sym
+    pl.add(~" ")
+    pl.add(param.name.s)
+    pl.add(~": ")
+    pl.add(genArg(p, ri[i], param, ri))
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
+      if ri.len > 1: pl.add(~" ")
       # beware of 'result = p(result)'. We always allocate a temporary:
       if d.k in {locTemp, locNone}:
         # We already got a temp. Great, special case it:
-        if d.k == locNone: getTemp(p, typ.sons[0], d, needsInit=true)
-        add(pl, ~"Result: ")
-        add(pl, addrLoc(p.config, d))
-        add(pl, ~"];$n")
+        if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
+        pl.add(~"Result: ")
+        pl.add(addrLoc(p.config, d))
+        pl.add(~"];$n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
-        add(pl, ~"];$n")
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
+        pl.add(~"];$n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
     else:
-      add(pl, ~"]")
-      if d.k == locNone: getTemp(p, typ.sons[0], d)
+      pl.add(~"]")
+      if d.k == locNone: getTemp(p, typ[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
       initLoc(list, locCall, ri, OnUnknown)
       list.r = pl
       genAssignment(p, d, list, {}) # no need for deep copying
   else:
-    add(pl, ~"];$n")
+    pl.add(~"];$n")
     line(p, cpsStmts, pl)
 
 proc genCall(p: BProc, e: PNode, d: var TLoc) =
-  if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure:
+  if e[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
     genClosureCall(p, nil, e, d)
-  elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags:
+  elif e[0].kind == nkSym and sfInfixCall in e[0].sym.flags:
     genInfixCall(p, nil, e, d)
-  elif e.sons[0].kind == nkSym and sfNamedParamCall in e.sons[0].sym.flags:
+  elif e[0].kind == nkSym and sfNamedParamCall in e[0].sym.flags:
     genNamedParamCall(p, e, d)
   else:
     genPrefixCall(p, nil, e, d)
   postStmtActions(p)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
-  if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure:
+  if ri[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
-  elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags:
+  elif ri[0].kind == nkSym and sfInfixCall in ri[0].sym.flags:
     genInfixCall(p, le, ri, d)
-  elif ri.sons[0].kind == nkSym and sfNamedParamCall in ri.sons[0].sym.flags:
+  elif ri[0].kind == nkSym and sfNamedParamCall in ri[0].sym.flags:
     genNamedParamCall(p, ri, d)
   else:
     genPrefixCall(p, le, ri, d)

compiler/ccgliterals.nim

@@ -7,6 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
+# included from cgen.nim
+
 ## This include file contains the logic to produce constant string
 ## and seq literals. The code here is responsible that
 ## ``const x = ["a", "b"]`` works without hidden runtime creation code.
@@ -19,7 +21,7 @@ template detectVersion(field, corename) =
     if core == nil or core.kind != skConst:
       m.g.field = 1
     else:
-      m.g.field = int ast.getInt(core.ast)
+      m.g.field = toInt(ast.getInt(core.ast))
   result = m.g.field
 
 proc detectStrVersion(m: BModule): int =
@@ -33,8 +35,8 @@ proc detectSeqVersion(m: BModule): int =
 proc genStringLiteralDataOnlyV1(m: BModule, s: string): Rope =
   discard cgsym(m, "TGenericSeq")
   result = getTempName(m)
-  addf(m.s[cfsData], "STRING_LITERAL($1, $2, $3);$n",
-       [result, makeCString(s), rope(len(s))])
+  m.s[cfsData].addf("STRING_LITERAL($1, $2, $3);$n",
+       [result, makeCString(s), rope(s.len)])
 
 proc genStringLiteralV1(m: BModule; n: PNode): Rope =
   if s.isNil:
@@ -47,61 +49,65 @@ proc genStringLiteralV1(m: BModule; n: PNode): Rope =
                       [genStringLiteralDataOnlyV1(m, n.strVal)])
     else:
       result = ropecg(m, "((#NimStringDesc*) &$1$2)",
-                      [m.tmpBase, rope(id)])
+                      [m.tmpBase, id])
 
 # ------ Version 2: destructor based strings and seqs -----------------------
 
-proc genStringLiteralDataOnlyV2(m: BModule, s: string): Rope =
-  result = getTempName(m)
-  addf(m.s[cfsData], "static const struct {$n" &
-       "  NI cap; void* allocator; NIM_CHAR data[$2];$n" &
-       "} $1 = { $2, NIM_NIL, $3 };$n",
-       [result, rope(len(s)), makeCString(s)])
+proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope; isConst: bool) =
+  m.s[cfsData].addf("static $4 struct {$n" &
+       "  NI cap; NIM_CHAR data[$2+1];$n" &
+       "} $1 = { $2 | NIM_STRLIT_FLAG, $3 };$n",
+       [result, rope(s.len), makeCString(s),
+       rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2(m: BModule; n: PNode): Rope =
+proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   if id == m.labels:
+    let pureLit = getTempName(m)
+    genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
+    result = getTempName(m)
     discard cgsym(m, "NimStrPayload")
     discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    let pureLit = genStringLiteralDataOnlyV2(m, n.strVal)
-    result = getTempName(m)
-    addf(m.s[cfsData], "static const NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [result, rope(len(n.strVal)), pureLit])
+    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [result, rope(n.strVal.len), pureLit, rope(if isConst: "const" else: "")])
   else:
-    result = m.tmpBase & rope(id)
+    result = getTempName(m)
+    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [result, rope(n.strVal.len), m.tmpBase & rope(id),
+          rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2Const(m: BModule; n: PNode): Rope =
+proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   var pureLit: Rope
   if id == m.labels:
+    pureLit = getTempName(m)
     discard cgsym(m, "NimStrPayload")
     discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    pureLit = genStringLiteralDataOnlyV2(m, n.strVal)
+    genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
   else:
     pureLit = m.tmpBase & rope(id)
-  result = "{$1, (NimStrPayload*)&$2}" % [rope(len(n.strVal)), pureLit]
+  result = "{$1, (NimStrPayload*)&$2}" % [rope(n.strVal.len), pureLit]
 
 # ------ Version selector ---------------------------------------------------
 
-proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo): Rope =
+proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo;
+                              isConst: bool): Rope =
   case detectStrVersion(m)
   of 0, 1: result = genStringLiteralDataOnlyV1(m, s)
-  of 2: result = genStringLiteralDataOnlyV2(m, s)
+  of 2:
+    result = getTempName(m)
+    genStringLiteralDataOnlyV2(m, s, result, isConst)
   else:
     localError(m.config, info, "cannot determine how to produce code for string literal")
 
-proc genStringLiteralFromData(m: BModule; data: Rope; info: TLineInfo): Rope =
-  result = ropecg(m, "((#NimStringDesc*) &$1)",
-                [data])
-
 proc genNilStringLiteral(m: BModule; info: TLineInfo): Rope =
   result = ropecg(m, "((#NimStringDesc*) NIM_NIL)", [])
 
 proc genStringLiteral(m: BModule; n: PNode): Rope =
   case detectStrVersion(m)
   of 0, 1: result = genStringLiteralV1(m, n)
-  of 2: result = genStringLiteralV2(m, n)
+  of 2: result = genStringLiteralV2(m, n, isConst = true)
   else:
     localError(m.config, n.info, "cannot determine how to produce code for string literal")

compiler/ccgmerge.nim

@@ -11,8 +11,8 @@
 ## is needed for incremental compilation.
 
 import
-  ast, astalgo, ropes, options, strutils, nimlexbase, msgs, cgendata, rodutils,
-  intsets, platform, llstream, tables, sighashes, pathutils
+  ast, ropes, options, strutils, nimlexbase, cgendata, rodutils,
+  intsets, llstream, tables, modulegraphs, pathutils
 
 # Careful! Section marks need to contain a tabulator so that they cannot
 # be part of C string literals.
@@ -21,16 +21,18 @@ const
   CFileSectionNames: array[TCFileSection, string] = [
     cfsMergeInfo: "",
     cfsHeaders: "NIM_merge_HEADERS",
+    cfsFrameDefines: "NIM_merge_FRAME_DEFINES",
     cfsForwardTypes: "NIM_merge_FORWARD_TYPES",
     cfsTypes: "NIM_merge_TYPES",
     cfsSeqTypes: "NIM_merge_SEQ_TYPES",
     cfsFieldInfo: "NIM_merge_FIELD_INFO",
     cfsTypeInfo: "NIM_merge_TYPE_INFO",
     cfsProcHeaders: "NIM_merge_PROC_HEADERS",
-    cfsVars: "NIM_merge_VARS",
     cfsData: "NIM_merge_DATA",
+    cfsVars: "NIM_merge_VARS",
     cfsProcs: "NIM_merge_PROCS",
     cfsInitProc: "NIM_merge_INIT_PROC",
+    cfsDatInitProc: "NIM_merge_DATINIT_PROC",
     cfsTypeInit1: "NIM_merge_TYPE_INIT1",
     cfsTypeInit2: "NIM_merge_TYPE_INIT2",
     cfsTypeInit3: "NIM_merge_TYPE_INIT3",
@@ -48,9 +50,9 @@ const
 proc genSectionStart*(fs: TCFileSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
     result = nil
-    add(result, "\n/*\t")
-    add(result, CFileSectionNames[fs])
-    add(result, ":*/\n")
+    result.add("\n/*\t")
+    result.add(CFileSectionNames[fs])
+    result.add(":*/\n")
 
 proc genSectionEnd*(fs: TCFileSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
@@ -59,9 +61,9 @@ proc genSectionEnd*(fs: TCFileSection; conf: ConfigRef): Rope =
 proc genSectionStart*(ps: TCProcSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
     result = rope("")
-    add(result, "\n/*\t")
-    add(result, CProcSectionNames[ps])
-    add(result, ":*/\n")
+    result.add("\n/*\t")
+    result.add(CProcSectionNames[ps])
+    result.add(":*/\n")
 
 proc genSectionEnd*(ps: TCProcSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
@@ -144,38 +146,34 @@ proc atEndMark(buf: cstring, pos: int): bool =
 
 proc readVerbatimSection(L: var TBaseLexer): Rope =
   var pos = L.bufpos
-  var buf = L.buf
   var r = newStringOfCap(30_000)
   while true:
-    case buf[pos]
+    case L.buf[pos]
     of CR:
       pos = nimlexbase.handleCR(L, pos)
-      buf = L.buf
       r.add('\L')
     of LF:
       pos = nimlexbase.handleLF(L, pos)
-      buf = L.buf
       r.add('\L')
     of '\0':
       doAssert(false, "ccgmerge: expected: " & NimMergeEndMark)
       break
     else:
-      if atEndMark(buf, pos):
+      if atEndMark(L.buf, pos):
         inc pos, NimMergeEndMark.len
         break
-      r.add(buf[pos])
+      r.add(L.buf[pos])
       inc pos
   L.bufpos = pos
   result = r.rope
 
 proc readKey(L: var TBaseLexer, result: var string) =
   var pos = L.bufpos
-  var buf = L.buf
   setLen(result, 0)
-  while buf[pos] in IdentChars:
-    result.add(buf[pos])
+  while L.buf[pos] in IdentChars:
+    result.add(L.buf[pos])
     inc pos
-  if buf[pos] != ':': doAssert(false, "ccgmerge: ':' expected")
+  if L.buf[pos] != ':': doAssert(false, "ccgmerge: ':' expected")
   L.bufpos = pos + 1 # skip ':'
 
 proc newFakeType(id: int): PType =

compiler/ccgthreadvars.nim

@@ -16,12 +16,12 @@ proc emulatedThreadVars(conf: ConfigRef): bool =
   result = {optThreads, optTlsEmulation} <= conf.globalOptions
 
 proc accessThreadLocalVar(p: BProc, s: PSym) =
-  if emulatedThreadVars(p.config) and not p.threadVarAccessed:
-    p.threadVarAccessed = true
+  if emulatedThreadVars(p.config) and threadVarAccessed notin p.flags:
+    p.flags.incl threadVarAccessed
     incl p.module.flags, usesThreadVars
-    addf(p.procSec(cpsLocals), "\tNimThreadVars* NimTV_;$n", [])
-    add(p.procSec(cpsInit),
-      ropecg(p.module, "\tNimTV_ = (NimThreadVars*) #GetThreadLocalVars();$n"))
+    p.procSec(cpsLocals).addf("\tNimThreadVars* NimTV_;$n", [])
+    p.procSec(cpsInit).add(
+      ropecg(p.module, "\tNimTV_ = (NimThreadVars*) #GetThreadLocalVars();$n", []))
 
 proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
   if emulatedThreadVars(m.config):
@@ -30,23 +30,25 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
     # allocator for it :-(
     if not containsOrIncl(m.g.nimtvDeclared, s.id):
       m.g.nimtvDeps.add(s.loc.t)
-      addf(m.g.nimtv, "$1 $2;$n", [getTypeDesc(m, s.loc.t), s.loc.r])
+      m.g.nimtv.addf("$1 $2;$n", [getTypeDesc(m, s.loc.t), s.loc.r])
   else:
-    if isExtern: add(m.s[cfsVars], "extern ")
-    if optThreads in m.config.globalOptions: add(m.s[cfsVars], "NIM_THREADVAR ")
-    add(m.s[cfsVars], getTypeDesc(m, s.loc.t))
-    addf(m.s[cfsVars], " $1;$n", [s.loc.r])
+    if isExtern: m.s[cfsVars].add("extern ")
+    elif lfExportLib in s.loc.flags: m.s[cfsVars].add("N_LIB_EXPORT_VAR ")
+    else: m.s[cfsVars].add("N_LIB_PRIVATE ")
+    if optThreads in m.config.globalOptions: m.s[cfsVars].add("NIM_THREADVAR ")
+    m.s[cfsVars].add(getTypeDesc(m, s.loc.t))
+    m.s[cfsVars].addf(" $1;$n", [s.loc.r])
 
 proc generateThreadLocalStorage(m: BModule) =
   if m.g.nimtv != nil and (usesThreadVars in m.flags or sfMainModule in m.module.flags):
     for t in items(m.g.nimtvDeps): discard getTypeDesc(m, t)
-    addf(m.s[cfsSeqTypes], "typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
+    m.s[cfsSeqTypes].addf("typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
 
 proc generateThreadVarsSize(m: BModule) =
   if m.g.nimtv != nil:
     let externc = if m.config.cmd == cmdCompileToCpp or
                        sfCompileToCpp in m.module.flags: "extern \"C\" "
                   else: ""
-    addf(m.s[cfsProcs],
+    m.s[cfsProcs].addf(
       "$#NI NimThreadVarsSize(){return (NI)sizeof(NimThreadVars);}$n",
       [externc.rope])

compiler/ccgtrav.nim

@@ -16,6 +16,9 @@ type
     p: BProc
     visitorFrmt: string
 
+const
+  visitorFrmt = "#nimGCvisit((void*)$1, $2);$n"
+
 proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType)
 proc genCaseRange(p: BProc, branch: PNode)
 proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false)
@@ -25,18 +28,18 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
   if n == nil: return
   case n.kind
   of nkRecList:
-    for i in countup(0, sonsLen(n) - 1):
-      genTraverseProc(c, accessor, n.sons[i], typ)
+    for i in 0..<n.len:
+      genTraverseProc(c, accessor, n[i], typ)
   of nkRecCase:
-    if (n.sons[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
+    if (n[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
     var p = c.p
-    let disc = n.sons[0].sym
+    let disc = n[0].sym
     if disc.loc.r == nil: fillObjectFields(c.p.module, typ)
     if disc.loc.t == nil:
       internalError(c.p.config, n.info, "genTraverseProc()")
     lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
-    for i in countup(1, sonsLen(n) - 1):
-      let branch = n.sons[i]
+    for i in 1..<n.len:
+      let branch = n[i]
       assert branch.kind in {nkOfBranch, nkElse}
       if branch.kind == nkOfBranch:
         genCaseRange(c.p, branch)
@@ -67,47 +70,48 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType) =
   var p = c.p
   case typ.kind
   of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred,
-     tySink:
+     tySink, tyOwned:
     genTraverseProc(c, accessor, lastSon(typ))
   of tyArray:
-    let arraySize = lengthOrd(c.p.config, typ.sons[0])
+    let arraySize = lengthOrd(c.p.config, typ[0])
     var i: TLoc
-    getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo(), tyInt), i)
+    getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
     let oldCode = p.s(cpsStmts)
     linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
-            i.r, arraySize.rope)
+            [i.r, arraySize])
     let oldLen = p.s(cpsStmts).len
-    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", accessor, i.r), typ.sons[1])
+    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", [accessor, i.r]), typ[1])
     if p.s(cpsStmts).len == oldLen:
       # do not emit dummy long loops for faster debug builds:
       p.s(cpsStmts) = oldCode
     else:
       lineF(p, cpsStmts, "}$n", [])
   of tyObject:
-    for i in countup(0, sonsLen(typ) - 1):
-      var x = typ.sons[i]
+    for i in 0..<typ.len:
+      var x = typ[i]
       if x != nil: x = x.skipTypes(skipPtrs)
       genTraverseProc(c, accessor.parentObj(c.p.module), x)
     if typ.n != nil: genTraverseProc(c, accessor, typ.n, typ)
   of tyTuple:
     let typ = getUniqueType(typ)
-    for i in countup(0, sonsLen(typ) - 1):
-      genTraverseProc(c, ropecg(c.p.module, "$1.Field$2", accessor, i.rope), typ.sons[i])
+    for i in 0..<typ.len:
+      genTraverseProc(c, ropecg(c.p.module, "$1.Field$2", [accessor, i]), typ[i])
   of tyRef:
-    lineCg(p, cpsStmts, c.visitorFrmt, accessor)
+    lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
   of tySequence:
-    if tfHasAsgn notin typ.flags:
-      lineCg(p, cpsStmts, c.visitorFrmt, accessor)
+    if optSeqDestructors notin c.p.module.config.globalOptions:
+      lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
     elif containsGarbageCollectedRef(typ.lastSon):
       # destructor based seqs are themselves not traced but their data is, if
       # they contain a GC'ed type:
-      genTraverseProcSeq(c, accessor, typ)
+      lineCg(p, cpsStmts, "#nimGCvisitSeq((void*)$1, $2);$n", [accessor, c.visitorFrmt])
+      #genTraverseProcSeq(c, accessor, typ)
   of tyString:
     if tfHasAsgn notin typ.flags:
-      lineCg(p, cpsStmts, c.visitorFrmt, accessor)
+      lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
   of tyProc:
     if typ.callConv == ccClosure:
-      lineCg(p, cpsStmts, c.visitorFrmt, ropecg(c.p.module, "$1.ClE_0", accessor))
+      lineCg(p, cpsStmts, visitorFrmt, [ropecg(c.p.module, "$1.ClE_0", [accessor]), c.visitorFrmt])
   else:
     discard
 
@@ -115,7 +119,7 @@ proc genTraverseProcSeq(c: TTraversalClosure, accessor: Rope, typ: PType) =
   var p = c.p
   assert typ.kind == tySequence
   var i: TLoc
-  getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo(), tyInt), i)
+  getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
   let oldCode = p.s(cpsStmts)
   var a: TLoc
   a.r = accessor
@@ -123,7 +127,7 @@ proc genTraverseProcSeq(c: TTraversalClosure, accessor: Rope, typ: PType) =
   lineF(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
       [i.r, lenExpr(c.p, a)])
   let oldLen = p.s(cpsStmts).len
-  genTraverseProc(c, "$1$3[$2]" % [accessor, i.r, dataField(c.p)], typ.sons[0])
+  genTraverseProc(c, "$1$3[$2]" % [accessor, i.r, dataField(c.p)], typ[0])
   if p.s(cpsStmts).len == oldLen:
     # do not emit dummy long loops for faster debug builds:
     p.s(cpsStmts) = oldCode
@@ -134,46 +138,53 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope =
   var c: TTraversalClosure
   var p = newProc(nil, m)
   result = "Marker_" & getTypeName(m, origTyp, sig)
-  var typ = origTyp.skipTypes(abstractInst)
+  let
+    hcrOn = m.hcrOn
+    typ = origTyp.skipTypes(abstractInstOwned)
+    markerName = if hcrOn: result & "_actual" else: result
+    header = "static N_NIMCALL(void, $1)(void* p, NI op)" % [markerName]
+    t = getTypeDesc(m, typ)
 
-  let header = "static N_NIMCALL(void, $1)(void* p, NI op)" % [result]
-
-  let t = getTypeDesc(m, typ)
   lineF(p, cpsLocals, "$1 a;$n", [t])
   lineF(p, cpsInit, "a = ($1)p;$n", [t])
 
   c.p = p
-  c.visitorFrmt = "#nimGCvisit((void*)$1, op);$n"
+  c.visitorFrmt = "op" # "#nimGCvisit((void*)$1, op);$n"
 
   assert typ.kind != tyTypeDesc
   if typ.kind == tySequence:
     genTraverseProcSeq(c, "a".rope, typ)
   else:
-    if skipTypes(typ.sons[0], typedescInst).kind == tyArray:
+    if skipTypes(typ[0], typedescInst+{tyOwned}).kind == tyArray:
       # C's arrays are broken beyond repair:
-      genTraverseProc(c, "a".rope, typ.sons[0])
+      genTraverseProc(c, "a".rope, typ[0])
     else:
-      genTraverseProc(c, "(*a)".rope, typ.sons[0])
+      genTraverseProc(c, "(*a)".rope, typ[0])
 
-  let generatedProc = "$1 {$n$2$3$4}$n" %
+  let generatedProc = "$1 {$n$2$3$4}\n" %
         [header, p.s(cpsLocals), p.s(cpsInit), p.s(cpsStmts)]
 
-  m.s[cfsProcHeaders].addf("$1;$n", [header])
+  m.s[cfsProcHeaders].addf("$1;\n", [header])
   m.s[cfsProcs].add(generatedProc)
 
+  if hcrOn:
+    m.s[cfsProcHeaders].addf("N_NIMCALL_PTR(void, $1)(void*, NI);\n", [result])
+    m.s[cfsDynLibInit].addf("\t$1 = (N_NIMCALL_PTR(void, )(void*, NI)) hcrRegisterProc($3, \"$1\", (void*)$2);\n",
+         [result, markerName, getModuleDllPath(m)])
+
 proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope =
   discard genTypeInfo(m, s.loc.t, info)
 
   var c: TTraversalClosure
   var p = newProc(nil, m)
-  var sLoc = s.loc.r
+  var sLoc = rdLoc(s.loc)
   result = getTempName(m)
 
   if sfThread in s.flags and emulatedThreadVars(m.config):
     accessThreadLocalVar(p, s)
     sLoc = "NimTV_->" & sLoc
 
-  c.visitorFrmt = "#nimGCvisit((void*)$1, 0);$n"
+  c.visitorFrmt = "0" # "#nimGCvisit((void*)$1, 0);$n"
   c.p = p
   let header = "static N_NIMCALL(void, $1)(void)" % [result]
   genTraverseProc(c, sLoc, s.loc.t)

compiler/ccgutils.nim

@@ -10,17 +10,17 @@
 # This module declares some helpers for the C code generator.
 
 import
-  ast, astalgo, ropes, hashes, strutils, types, msgs, wordrecg,
+  ast, hashes, strutils, msgs, wordrecg,
   platform, trees, options
 
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
   case n.kind
   of nkStmtList:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       result = getPragmaStmt(n[i], w)
       if result != nil: break
   of nkPragma:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       if whichPragma(n[i]) == w: return n[i]
   else: discard
 
@@ -28,29 +28,29 @@ proc stmtsContainPragma*(n: PNode, w: TSpecialWord): bool =
   result = getPragmaStmt(n, w) != nil
 
 proc hashString*(conf: ConfigRef; s: string): BiggestInt =
-  # has to be the same algorithm as system.hashString!
+  # has to be the same algorithm as strmantle.hashString!
   if CPU[conf.target.targetCPU].bit == 64:
     # we have to use the same bitwidth
     # as the target CPU
-    var b = 0'i64
-    for i in countup(0, len(s) - 1):
-      b = b +% ord(s[i])
-      b = b +% `shl`(b, 10)
-      b = b xor `shr`(b, 6)
-    b = b +% `shl`(b, 3)
-    b = b xor `shr`(b, 11)
-    b = b +% `shl`(b, 15)
-    result = b
+    var b = 0'u64
+    for i in 0..<s.len:
+      b = b + uint(s[i])
+      b = b + (b shl 10)
+      b = b xor (b shr 6)
+    b = b + (b shl 3)
+    b = b xor (b shr 11)
+    b = b + (b shl 15)
+    result = cast[Hash](b)
   else:
-    var a = 0'i32
-    for i in countup(0, len(s) - 1):
-      a = a +% ord(s[i]).int32
-      a = a +% `shl`(a, 10'i32)
-      a = a xor `shr`(a, 6'i32)
-    a = a +% `shl`(a, 3'i32)
-    a = a xor `shr`(a, 11'i32)
-    a = a +% `shl`(a, 15'i32)
-    result = a
+    var a = 0'u32
+    for i in 0..<s.len:
+      a = a + uint32(s[i])
+      a = a + (a shl 10)
+      a = a xor (a shr 6)
+    a = a + (a shl 3)
+    a = a xor (a shr 11)
+    a = a + (a shl 15)
+    result = cast[Hash](a)
 
 template getUniqueType*(key: PType): PType = key
 
@@ -70,18 +70,18 @@ proc mangle*(name: string): string =
   template special(x) =
     result.add x
     requiresUnderscore = true
-  for i in start..(name.len-1):
+  for i in start..<name.len:
     let c = name[i]
     case c
     of 'a'..'z', '0'..'9', 'A'..'Z':
-      add(result, c)
+      result.add(c)
     of '_':
       # we generate names like 'foo_9' for scope disambiguations and so
       # disallow this here:
       if i > 0 and i < name.len-1 and name[i+1] in Digits:
         discard
       else:
-        add(result, c)
+        result.add(c)
     of '$': special "dollar"
     of '%': special "percent"
     of '&': special "amp"
@@ -102,7 +102,7 @@ proc mangle*(name: string): string =
     of '@': special "at"
     of '|': special "bar"
     else:
-      add(result, "X" & toHex(ord(c), 2))
+      result.add("X" & toHex(ord(c), 2))
       requiresUnderscore = true
   if requiresUnderscore:
     result.add "_"

compiler/cgendata.nim

@@ -10,16 +10,15 @@
 ## This module contains the data structures for the C code generation phase.
 
 import
-  ast, astalgo, ropes, passes, options, intsets, platform, sighashes,
-  tables, ndi, lineinfos, pathutils
-
-from modulegraphs import ModuleGraph
+  ast, ropes, options, intsets,
+  tables, ndi, lineinfos, pathutils, modulegraphs
 
 type
   TLabel* = Rope              # for the C generator a label is just a rope
   TCFileSection* = enum       # the sections a generated C file consists of
     cfsMergeInfo,             # section containing merge information
     cfsHeaders,               # section for C include file headers
+    cfsFrameDefines           # section for nim frame macros
     cfsForwardTypes,          # section for C forward typedefs
     cfsTypes,                 # section for C typedefs
     cfsSeqTypes,              # section for sequence types only
@@ -28,10 +27,11 @@ type
     cfsFieldInfo,             # section for field information
     cfsTypeInfo,              # section for type information
     cfsProcHeaders,           # section for C procs prototypes
-    cfsVars,                  # section for C variable declarations
     cfsData,                  # section for C constant data
+    cfsVars,                  # section for C variable declarations
     cfsProcs,                 # section for C procs that are not inline
     cfsInitProc,              # section for the C init proc
+    cfsDatInitProc,           # section for the C datInit proc
     cfsTypeInit1,             # section 1 for declarations of type information
     cfsTypeInit2,             # section 2 for init of type information
     cfsTypeInit3,             # section 3 for init of type information
@@ -58,21 +58,26 @@ type
     id*: int                  # the ID of the label; positive means that it
     label*: Rope              # generated text for the label
                               # nil if label is not used
-    sections*: TCProcSections # the code beloging
+    sections*: TCProcSections # the code belonging
     isLoop*: bool             # whether block is a loop
     nestedTryStmts*: int16    # how many try statements is it nested into
     nestedExceptStmts*: int16 # how many except statements is it nested into
     frameLen*: int16
 
+  TCProcFlag* = enum
+    beforeRetNeeded,
+    threadVarAccessed,
+    hasCurFramePointer,
+    noSafePoints,
+    nimErrorFlagAccessed,
+    nimErrorFlagDeclared
+
   TCProc = object             # represents C proc that is currently generated
     prc*: PSym                # the Nim proc that this C proc belongs to
-    beforeRetNeeded*: bool    # true iff 'BeforeRet' label for proc is needed
-    threadVarAccessed*: bool  # true if the proc already accessed some threadvar
-    hasCurFramePointer*: bool # true if _nimCurFrame var needed to recover after
-                              # exception is generated
+    flags*: set[TCProcFlag]
     lastLineInfo*: TLineInfo  # to avoid generating excessive 'nimln' statements
     currLineInfo*: TLineInfo  # AST codegen will make this superfluous
-    nestedTryStmts*: seq[tuple[n: PNode, inExcept: bool]]
+    nestedTryStmts*: seq[tuple[fin: PNode, inExcept: bool, label: Natural]]
                               # in how many nested try statements we are
                               # (the vars must be volatile then)
                               # bool is true when are in the except part of a try block
@@ -85,20 +90,19 @@ type
     options*: TOptions        # options that should be used for code
                               # generation; this is the same as prc.options
                               # unless prc == nil
-    maxFrameLen*: int         # max length of frame descriptor
     module*: BModule          # used to prevent excessive parameter passing
     withinLoop*: int          # > 0 if we are within a loop
     splitDecls*: int          # > 0 if we are in some context for C++ that
                               # requires 'T x = T()' to become 'T x; x = T()'
                               # (yes, C++ is weird like that)
-    gcFrameId*: Natural       # for the GC stack marking
-    gcFrameType*: Rope        # the struct {} we put the GC markers into
+    withinTryWithExcept*: int # required for goto based exception handling
     sigConflicts*: CountTable[string]
 
   TTypeSeq* = seq[PType]
   TypeCache* = Table[SigHash, Rope]
+  TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: PSym]]
 
-  Codegenflag* = enum
+  CodegenFlag* = enum
     preventStackTrace,  # true if stack traces need to be prevented
     usesThreadVars,     # true if the module uses a thread var
     frameDeclared,      # hack for ROD support so that we don't declare
@@ -112,11 +116,10 @@ type
     mainModProcs*, mainModInit*, otherModsInit*, mainDatInit*: Rope
     mapping*: Rope             # the generated mapping file (if requested)
     modules*: seq[BModule]     # list of all compiled modules
-    forwardedProcsCounter*: int
+    modulesClosed*: seq[BModule] # list of the same compiled modules, but in the order they were closed
+    forwardedProcs*: seq[PSym] # proc:s that did not yet have a body
     generatedHeader*: BModule
-    breakPointId*: int
-    breakpoints*: Rope # later the breakpoints are inserted into the main proc
-    typeInfoMarker*: TypeCache
+    typeInfoMarker*: TypeCacheWithOwner
     config*: ConfigRef
     graph*: ModuleGraph
     strVersion*, seqVersion*: int # version of the string/seq implementation to use
@@ -132,9 +135,9 @@ type
                             # nimtvDeps is VERY hard to cache because it's
                             # not a list of IDs nor can it be made to be one.
 
-  TCGen = object of TPassContext # represents a C source file
+  TCGen = object of PPassContext # represents a C source file
     s*: TCFileSections        # sections of the C file
-    flags*: set[Codegenflag]
+    flags*: set[CodegenFlag]
     module*: PSym
     filename*: AbsoluteFile
     cfilename*: AbsoluteFile  # filename of the module (including path,
@@ -148,14 +151,15 @@ type
     typeInfoMarker*: TypeCache # needed for generating type information
     initProc*: BProc          # code for init procedure
     preInitProc*: BProc       # code executed before the init proc
+    hcrCreateTypeInfosProc*: Rope # type info globals are in here when HCR=on
+    inHcrInitGuard*: bool     # We are currently within a HCR reloading guard.
     typeStack*: TTypeSeq      # used for type generation
     dataCache*: TNodeTable
-    forwardedProcs*: TSymSeq  # keep forwarded procs here
     typeNodes*, nimTypes*: int # used for type info generation
     typeNodesName*, nimTypesName*: Rope # used for type info generation
     labels*: Natural          # for generating unique module-scope names
     extensionLoaders*: array['0'..'9', Rope] # special procs for the
-                                              # OpenGL wrapper
+                                             # OpenGL wrapper
     injectStmt*: Rope
     sigConflicts*: CountTable[SigHash]
     g*: BModuleList
@@ -170,7 +174,7 @@ proc includeHeader*(this: BModule; header: string) =
 
 proc s*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # section in the current block
-  result = p.blocks[p.blocks.len-1].sections[s]
+  result = p.blocks[^1].sections[s]
 
 proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # top level proc sections
@@ -188,12 +192,10 @@ proc newProc*(prc: PSym, module: BModule): BProc =
   result.sigConflicts = initCountTable[string]()
 
 proc newModuleList*(g: ModuleGraph): BModuleList =
-  BModuleList(modules: @[], typeInfoMarker: initTable[SigHash, Rope](), config: g.config,
-    graph: g, nimtvDeps: @[], nimtvDeclared: initIntSet())
+  BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: PSym]](),
+    config: g.config, graph: g, nimtvDeclared: initIntSet())
 
 iterator cgenModules*(g: BModuleList): BModule =
-  for i in 0..high(g.modules):
-    # ultimately, we are iterating over the file ids here.
-    # some "files" won't have an associated cgen module (like stdin)
-    # and we must skip over them.
-    if g.modules[i] != nil: yield g.modules[i]
+  for m in g.modulesClosed:
+    # iterate modules in the order they were closed
+    yield m

compiler/cgmeth.nim

@@ -7,10 +7,10 @@
 #    distribution, for details about the copyright.
 #
 
-## This module implements code generation for multi methods.
+## This module implements code generation for methods.
 
 import
-  intsets, options, ast, astalgo, msgs, idents, renderer, types, magicsys,
+  intsets, options, ast, msgs, idents, renderer, types, magicsys,
   sempass2, strutils, modulegraphs, lineinfos
 
 proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
@@ -23,11 +23,11 @@ proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
       result = n
     elif diff < 0:
       result = newNodeIT(nkObjUpConv, n.info, d)
-      addSon(result, n)
+      result.add n
       if downcast: internalError(conf, n.info, "cgmeth.genConv: no upcast allowed")
     elif diff > 0:
       result = newNodeIT(nkObjDownConv, n.info, d)
-      addSon(result, n)
+      result.add n
       if not downcast:
         internalError(conf, n.info, "cgmeth.genConv: no downcast allowed")
     else:
@@ -37,34 +37,33 @@ proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
 
 proc getDispatcher*(s: PSym): PSym =
   ## can return nil if is has no dispatcher.
-  let dispn = lastSon(s.ast)
-  if dispn.kind == nkSym:
-    let disp = dispn.sym
-    if sfDispatcher in disp.flags: result = disp
+  if dispatcherPos < s.ast.len:
+    result = s.ast[dispatcherPos].sym
+    doAssert sfDispatcher in result.flags
 
 proc methodCall*(n: PNode; conf: ConfigRef): PNode =
   result = n
   # replace ordinary method by dispatcher method:
-  let disp = getDispatcher(result.sons[0].sym)
+  let disp = getDispatcher(result[0].sym)
   if disp != nil:
-    result.sons[0].sym = disp
+    result[0].sym = disp
     # change the arguments to up/downcasts to fit the dispatcher's parameters:
-    for i in countup(1, sonsLen(result)-1):
-      result.sons[i] = genConv(result.sons[i], disp.typ.sons[i], true, conf)
+    for i in 1..<result.len:
+      result[i] = genConv(result[i], disp.typ[i], true, conf)
   else:
-    localError(conf, n.info, "'" & $result.sons[0] & "' lacks a dispatcher")
+    localError(conf, n.info, "'" & $result[0] & "' lacks a dispatcher")
 
 type
   MethodResult = enum No, Invalid, Yes
 
-proc sameMethodBucket(a, b: PSym): MethodResult =
+proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
   if a.name.id != b.name.id: return
-  if sonsLen(a.typ) != sonsLen(b.typ):
+  if a.typ.len != b.typ.len:
     return
 
-  for i in countup(1, sonsLen(a.typ) - 1):
-    var aa = a.typ.sons[i]
-    var bb = b.typ.sons[i]
+  for i in 1..<a.typ.len:
+    var aa = a.typ[i]
+    var bb = b.typ[i]
     while true:
       aa = skipTypes(aa, {tyGenericInst, tyAlias})
       bb = skipTypes(bb, {tyGenericInst, tyAlias})
@@ -73,10 +72,10 @@ proc sameMethodBucket(a, b: PSym): MethodResult =
         bb = bb.lastSon
       else:
         break
-    if sameType(a.typ.sons[i], b.typ.sons[i]):
+    if sameType(a.typ[i], b.typ[i]):
       if aa.kind == tyObject and result != Invalid:
         result = Yes
-    elif aa.kind == tyObject and bb.kind == tyObject:
+    elif aa.kind == tyObject and bb.kind == tyObject and (i == 1 or multiMethods):
       let diff = inheritanceDiff(bb, aa)
       if diff < 0:
         if result != Invalid:
@@ -91,21 +90,22 @@ proc sameMethodBucket(a, b: PSym): MethodResult =
       return No
   if result == Yes:
     # check for return type:
-    if not sameTypeOrNil(a.typ.sons[0], b.typ.sons[0]):
-      if b.typ.sons[0] != nil and b.typ.sons[0].kind == tyExpr:
+    if not sameTypeOrNil(a.typ[0], b.typ[0]):
+      if b.typ[0] != nil and b.typ[0].kind == tyUntyped:
         # infer 'auto' from the base to make it consistent:
-        b.typ.sons[0] = a.typ.sons[0]
+        b.typ[0] = a.typ[0]
       else:
         return No
 
 proc attachDispatcher(s: PSym, dispatcher: PNode) =
-  var L = s.ast.len-1
-  var x = s.ast.sons[L]
-  if x.kind == nkSym and sfDispatcher in x.sym.flags:
+  if dispatcherPos < s.ast.len:
     # we've added a dispatcher already, so overwrite it
-    s.ast.sons[L] = dispatcher
+    s.ast[dispatcherPos] = dispatcher
   else:
-    s.ast.add(dispatcher)
+    setLen(s.ast.sons, dispatcherPos+1)
+    if s.ast[resultPos] == nil:
+      s.ast[resultPos] = newNodeI(nkEmpty, s.info)
+    s.ast[dispatcherPos] = dispatcher
 
 proc createDispatcher(s: PSym): PSym =
   var disp = copySym(s)
@@ -115,16 +115,16 @@ proc createDispatcher(s: PSym): PSym =
   # we can't inline the dispatcher itself (for now):
   if disp.typ.callConv == ccInline: disp.typ.callConv = ccDefault
   disp.ast = copyTree(s.ast)
-  disp.ast.sons[bodyPos] = newNodeI(nkEmpty, s.info)
+  disp.ast[bodyPos] = newNodeI(nkEmpty, s.info)
   disp.loc.r = nil
-  if s.typ.sons[0] != nil:
-    if disp.ast.sonsLen > resultPos:
-      disp.ast.sons[resultPos].sym = copySym(s.ast.sons[resultPos].sym)
+  if s.typ[0] != nil:
+    if disp.ast.len > resultPos:
+      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym)
     else:
       # We've encountered a method prototype without a filled-in
       # resultPos slot. We put a placeholder in there that will
       # be updated in fixupDispatcher().
-      disp.ast.addSon(newNodeI(nkEmpty, s.info))
+      disp.ast.add newNodeI(nkEmpty, s.info)
   attachDispatcher(s, newSymNode(disp))
   # attach to itself to prevent bugs:
   attachDispatcher(disp, newSymNode(disp))
@@ -136,9 +136,9 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
   # from later definitions, particularly the resultPos slot. Also,
   # the lock level of the dispatcher needs to be updated/checked
   # against that of the method.
-  if disp.ast.sonsLen > resultPos and meth.ast.sonsLen > resultPos and
-     disp.ast.sons[resultPos].kind == nkEmpty:
-    disp.ast.sons[resultPos] = copyTree(meth.ast.sons[resultPos])
+  if disp.ast.len > resultPos and meth.ast.len > resultPos and
+     disp.ast[resultPos].kind == nkEmpty:
+    disp.ast[resultPos] = copyTree(meth.ast[resultPos])
 
   # The following code works only with lock levels, so we disable
   # it when they're not available.
@@ -158,14 +158,13 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
         disp.typ.lockLevel = meth.typ.lockLevel
 
 proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
-  let L = len(g.methods)
   var witness: PSym
-  for i in countup(0, L - 1):
+  for i in 0..<g.methods.len:
     let disp = g.methods[i].dispatcher
-    case sameMethodBucket(disp, s)
+    case sameMethodBucket(disp, s, multimethods = optMultiMethods in g.config.globalOptions)
     of Yes:
-      add(g.methods[i].methods, s)
-      attachDispatcher(s, lastSon(disp.ast))
+      g.methods[i].methods.add(s)
+      attachDispatcher(s, disp.ast[dispatcherPos])
       fixupDispatcher(s, disp, g.config)
       #echo "fixup ", disp.name.s, " ", disp.id
       when useEffectSystem: checkMethodEffects(g, disp, s)
@@ -178,7 +177,7 @@ proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
     of Invalid:
       if witness.isNil: witness = g.methods[i].methods[0]
   # create a new dispatcher:
-  add(g.methods, (methods: @[s], dispatcher: createDispatcher(s)))
+  g.methods.add((methods: @[s], dispatcher: createDispatcher(s)))
   #echo "adding ", s.info
   #if fromCache:
   #  internalError(s.info, "no method dispatcher found")
@@ -188,34 +187,34 @@ proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
   elif sfBase notin s.flags:
     message(g.config, s.info, warnUseBase)
 
-proc relevantCol(methods: TSymSeq, col: int): bool =
+proc relevantCol(methods: seq[PSym], col: int): bool =
   # returns true iff the position is relevant
-  var t = methods[0].typ.sons[col].skipTypes(skipPtrs)
+  var t = methods[0].typ[col].skipTypes(skipPtrs)
   if t.kind == tyObject:
-    for i in countup(1, high(methods)):
-      let t2 = skipTypes(methods[i].typ.sons[col], skipPtrs)
+    for i in 1..high(methods):
+      let t2 = skipTypes(methods[i].typ[col], skipPtrs)
       if not sameType(t2, t):
         return true
 
 proc cmpSignatures(a, b: PSym, relevantCols: IntSet): int =
-  for col in countup(1, sonsLen(a.typ) - 1):
+  for col in 1..<a.typ.len:
     if contains(relevantCols, col):
-      var aa = skipTypes(a.typ.sons[col], skipPtrs)
-      var bb = skipTypes(b.typ.sons[col], skipPtrs)
+      var aa = skipTypes(a.typ[col], skipPtrs)
+      var bb = skipTypes(b.typ[col], skipPtrs)
       var d = inheritanceDiff(aa, bb)
       if (d != high(int)) and d != 0:
         return d
 
-proc sortBucket(a: var TSymSeq, relevantCols: IntSet) =
+proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
   # we use shellsort here; fast and simple
-  var n = len(a)
+  var n = a.len
   var h = 1
   while true:
     h = 3 * h + 1
     if h > n: break
   while true:
     h = h div 3
-    for i in countup(h, n - 1):
+    for i in h..<n:
       var v = a[i]
       var j = i
       while cmpSignatures(a[j - h], v, relevantCols) >= 0:
@@ -225,70 +224,73 @@ proc sortBucket(a: var TSymSeq, relevantCols: IntSet) =
       a[j] = v
     if h == 1: break
 
-proc genDispatcher(g: ModuleGraph; methods: TSymSeq, relevantCols: IntSet): PSym =
-  var base = lastSon(methods[0].ast).sym
+proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet): PSym =
+  var base = methods[0].ast[dispatcherPos].sym
   result = base
-  var paramLen = sonsLen(base.typ)
+  var paramLen = base.typ.len
   var nilchecks = newNodeI(nkStmtList, base.info)
   var disp = newNodeI(nkIfStmt, base.info)
-  var ands = getSysMagic(g, unknownLineInfo(), "and", mAnd)
-  var iss = getSysMagic(g, unknownLineInfo(), "of", mOf)
-  let boolType = getSysType(g, unknownLineInfo(), tyBool)
-  for col in countup(1, paramLen - 1):
+  var ands = getSysMagic(g, unknownLineInfo, "and", mAnd)
+  var iss = getSysMagic(g, unknownLineInfo, "of", mOf)
+  let boolType = getSysType(g, unknownLineInfo, tyBool)
+  for col in 1..<paramLen:
     if contains(relevantCols, col):
-      let param = base.typ.n.sons[col].sym
+      let param = base.typ.n[col].sym
       if param.typ.skipTypes(abstractInst).kind in {tyRef, tyPtr}:
-        addSon(nilchecks, newTree(nkCall,
-            newSymNode(getCompilerProc(g, "chckNilDisp")), newSymNode(param)))
-  for meth in countup(0, high(methods)):
+        nilchecks.add newTree(nkCall,
+            newSymNode(getCompilerProc(g, "chckNilDisp")), newSymNode(param))
+  for meth in 0..high(methods):
     var curr = methods[meth]      # generate condition:
     var cond: PNode = nil
-    for col in countup(1, paramLen - 1):
+    for col in 1..<paramLen:
       if contains(relevantCols, col):
         var isn = newNodeIT(nkCall, base.info, boolType)
-        addSon(isn, newSymNode(iss))
-        let param = base.typ.n.sons[col].sym
-        addSon(isn, newSymNode(param))
-        addSon(isn, newNodeIT(nkType, base.info, curr.typ.sons[col]))
+        isn.add newSymNode(iss)
+        let param = base.typ.n[col].sym
+        isn.add newSymNode(param)
+        isn.add newNodeIT(nkType, base.info, curr.typ[col])
         if cond != nil:
           var a = newNodeIT(nkCall, base.info, boolType)
-          addSon(a, newSymNode(ands))
-          addSon(a, cond)
-          addSon(a, isn)
+          a.add newSymNode(ands)
+          a.add cond
+          a.add isn
           cond = a
         else:
           cond = isn
-    let retTyp = base.typ.sons[0]
+    let retTyp = base.typ[0]
     let call = newNodeIT(nkCall, base.info, retTyp)
-    addSon(call, newSymNode(curr))
-    for col in countup(1, paramLen - 1):
-      addSon(call, genConv(newSymNode(base.typ.n.sons[col].sym),
-                           curr.typ.sons[col], false, g.config))
+    call.add newSymNode(curr)
+    for col in 1..<paramLen:
+      call.add genConv(newSymNode(base.typ.n[col].sym),
+                           curr.typ[col], false, g.config)
     var ret: PNode
     if retTyp != nil:
       var a = newNodeI(nkFastAsgn, base.info)
-      addSon(a, newSymNode(base.ast.sons[resultPos].sym))
-      addSon(a, call)
+      a.add newSymNode(base.ast[resultPos].sym)
+      a.add call
       ret = newNodeI(nkReturnStmt, base.info)
-      addSon(ret, a)
+      ret.add a
     else:
       ret = call
     if cond != nil:
       var a = newNodeI(nkElifBranch, base.info)
-      addSon(a, cond)
-      addSon(a, ret)
-      addSon(disp, a)
+      a.add cond
+      a.add ret
+      disp.add a
     else:
       disp = ret
   nilchecks.add disp
-  result.ast.sons[bodyPos] = nilchecks
+  nilchecks.flags.incl nfTransf # should not be further transformed
+  result.ast[bodyPos] = nilchecks
 
 proc generateMethodDispatchers*(g: ModuleGraph): PNode =
   result = newNode(nkStmtList)
-  for bucket in countup(0, len(g.methods) - 1):
+  for bucket in 0..<g.methods.len:
     var relevantCols = initIntSet()
-    for col in countup(1, sonsLen(g.methods[bucket].methods[0].typ) - 1):
+    for col in 1..<g.methods[bucket].methods[0].typ.len:
       if relevantCol(g.methods[bucket].methods, col): incl(relevantCols, col)
+      if optMultiMethods notin g.config.globalOptions:
+        # if multi-methods are not enabled, we are interested only in the first field
+        break
     sortBucket(g.methods[bucket].methods, relevantCols)
-    addSon(result,
-           newSymNode(genDispatcher(g, g.methods[bucket].methods, relevantCols)))
+    result.add newSymNode(genDispatcher(g, g.methods[bucket].methods, relevantCols))

compiler/closureiters.nim

@@ -90,9 +90,6 @@
 #  :unrollFinally = true
 #  goto nearestFinally (or -1 if not exists)
 #
-# Every finally block calls closureIterEndFinally() upon its successful
-# completion.
-#
 # Example:
 #
 # try:
@@ -126,13 +123,14 @@
 #     if :curExc.isNil:
 #       return :tmpResult
 #     else:
+#       closureIterSetupExc(nil)
 #       raise
 #   state = -1 # Goto next state. In this case we just exit
 #   break :stateLoop
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, idents,
-  renderer, types, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
+  ast, msgs, idents,
+  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
 
 type
   Ctx = object
@@ -156,8 +154,8 @@ type
                     # is their finally. For finally it is parent finally. Otherwise -1
 
 const
-  nkSkip = { nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
-            nkCommentStmt } + procDefs
+  nkSkip = {nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
+            nkCommentStmt} + procDefs
 
 proc newStateAccess(ctx: var Ctx): PNode =
   if ctx.stateVarSym.isNil:
@@ -174,7 +172,7 @@ proc newStateAssgn(ctx: var Ctx, toValue: PNode): PNode =
 proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
   # Creates state assignment:
   #   :state = stateNo
-  ctx.newStateAssgn(newIntTypeNode(nkIntLit, stateNo, ctx.g.getSysType(TLineInfo(), tyInt)))
+  ctx.newStateAssgn(newIntTypeNode(stateNo, ctx.g.getSysType(TLineInfo(), tyInt)))
 
 proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
   result = newSym(skVar, getIdent(ctx.g.cache, name), ctx.fn, ctx.fn.info)
@@ -210,7 +208,7 @@ proc newUnrollFinallyAccess(ctx: var Ctx, info: TLineInfo): PNode =
 
 proc newCurExcAccess(ctx: var Ctx): PNode =
   if ctx.curExcSym.isNil:
-    ctx.curExcSym = ctx.newEnvVar(":curExc", ctx.g.callCodegenProc("getCurrentException", ctx.g.emptyNode).typ)
+    ctx.curExcSym = ctx.newEnvVar(":curExc", ctx.g.callCodegenProc("getCurrentException").typ)
   ctx.newEnvVarAccess(ctx.curExcSym)
 
 proc newState(ctx: var Ctx, n, gotoOut: PNode): int =
@@ -239,7 +237,7 @@ proc toStmtList(n: PNode): PNode =
 proc addGotoOut(n: PNode, gotoOut: PNode): PNode =
   # Make sure `n` is a stmtlist, and ends with `gotoOut`
   result = toStmtList(n)
-  if result.len == 0 or result.sons[^1].kind != nkGotoState:
+  if result.len == 0 or result[^1].kind != nkGotoState:
     result.add(gotoOut)
 
 proc newTempVar(ctx: var Ctx, typ: PType): PSym =
@@ -275,7 +273,7 @@ proc transformBreaksAndContinuesInWhile(ctx: var Ctx, n: PNode, before, after: P
     if ctx.blockLevel == 0:
       result = after
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformBreaksAndContinuesInWhile(n[i], before, after)
 
 proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode =
@@ -295,7 +293,7 @@ proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode
       if label.kind == nkSym and n[0].sym == label.sym:
         result = after
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformBreaksInBlock(n[i], label, after)
 
 proc newNullifyCurExc(ctx: var Ctx, info: TLineInfo): PNode =
@@ -320,11 +318,11 @@ proc collectExceptState(ctx: var Ctx, n: PNode): PNode {.inline.} =
 
       if c.len > 1:
         var cond: PNode
-        for i in 0 .. c.len - 2:
+        for i in 0..<c.len - 1:
           assert(c[i].kind == nkType)
           let nextCond = newTree(nkCall,
             newSymNode(g.getSysMagic(c.info, "of", mOf)),
-            g.callCodegenProc("getCurrentException", ctx.g.emptyNode),
+            g.callCodegenProc("getCurrentException"),
             c[i])
           nextCond.typ = ctx.g.getSysType(c.info, tyBool)
           nextCond.info = c.info
@@ -359,12 +357,12 @@ proc addElseToExcept(ctx: var Ctx, n: PNode) =
     block: # :unrollFinally = true
       branchBody.add(newTree(nkAsgn,
         ctx.newUnrollFinallyAccess(n.info),
-        newIntTypeNode(nkIntLit, 1, ctx.g.getSysType(n.info, tyBool))))
+        newIntTypeNode(1, ctx.g.getSysType(n.info, tyBool))))
 
     block: # :curExc = getCurrentException()
       branchBody.add(newTree(nkAsgn,
         ctx.newCurExcAccess(),
-        ctx.g.callCodegenProc("getCurrentException", ctx.g.emptyNode)))
+        ctx.g.callCodegenProc("getCurrentException")))
 
     block: # goto nearestFinally
       branchBody.add(newTree(nkGotoState, ctx.g.newIntLit(n.info, ctx.nearestFinally)))
@@ -403,7 +401,7 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
   var n = n
   while n.kind == nkStmtListExpr:
     result.s.sons.add(n.sons)
-    result.s.sons.setLen(result.s.sons.len - 1) # delete last son
+    result.s.sons.setLen(result.s.len - 1) # delete last son
     n = n[^1]
 
   result.res = n
@@ -437,7 +435,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkYieldStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -451,7 +449,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkPar, nkObjConstr, nkTupleConstr, nkBracket:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -461,16 +459,23 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       if n.typ.isNil: internalError(ctx.g.config, "lowerStmtListExprs: constr typ.isNil")
       result.typ = n.typ
 
-      for i in 0 ..< n.len:
-        if n[i].kind == nkStmtListExpr:
+      for i in 0..<n.len:
+        case n[i].kind
+        of nkExprColonExpr:
+          if n[i][1].kind == nkStmtListExpr:
+            let (st, ex) = exprToStmtList(n[i][1])
+            result.add(st)
+            n[i][1] = ex
+        of nkStmtListExpr:
           let (st, ex) = exprToStmtList(n[i])
           result.add(st)
           n[i] = ex
+        else: discard
       result.add(n)
 
   of nkIfStmt, nkIfExpr:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -534,9 +539,9 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
       if isExpr: result.add(ctx.newEnvVarAccess(tmp))
 
-  of nkTryStmt:
+  of nkTryStmt, nkHiddenTryStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -549,7 +554,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         let tmp = ctx.newTempVar(n.typ)
 
         n[0] = ctx.convertExprBodyToAsgn(n[0], tmp)
-        for i in 1 ..< n.len:
+        for i in 1..<n.len:
           let branch = n[i]
           case branch.kind
           of nkExceptBranch:
@@ -566,7 +571,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkCaseStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -584,7 +589,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           result.add(st)
           n[0] = ex
 
-        for i in 1 ..< n.len:
+        for i in 1..<n.len:
           let branch = n[i]
           case branch.kind
           of nkOfBranch:
@@ -596,9 +601,9 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         result.add(n)
         result.add(ctx.newEnvVarAccess(tmp))
 
-  of nkCallKinds:
+  of nkCallKinds, nkChckRange, nkChckRangeF, nkChckRange64:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -638,7 +643,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         result.add(ifNode)
         result.add(ctx.newEnvVarAccess(tmp))
       else:
-        for i in 0 ..< n.len:
+        for i in 0..<n.len:
           if n[i].kind == nkStmtListExpr:
             let (st, ex) = exprToStmtList(n[i])
             result.add(st)
@@ -667,7 +672,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkDiscardStmt, nkReturnStmt, nkRaiseStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -678,9 +683,10 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       n[0] = ex
       result.add(n)
 
-  of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv, nkObjDownConv:
+  of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv, nkObjDownConv,
+      nkDerefExpr, nkHiddenDeref:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -694,7 +700,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkAsgn, nkFastAsgn:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -752,7 +758,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         n[0] = newSymNode(ctx.g.getSysSym(n[0].info, "true"))
         n[1] = newBody
 
-  of nkDotExpr:
+  of nkDotExpr, nkCheckedFieldExpr:
     var ns = false
     n[0] = ctx.lowerStmtListExprs(n[0], ns)
     if ns:
@@ -772,14 +778,14 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       result = newNodeI(nkStmtListExpr, n.info)
       result.typ = n.typ
       let (st, ex) = exprToStmtList(n[1])
-      n.kind = nkBlockStmt
+      n.transitionSonsKind(nkBlockStmt)
       n.typ = nil
       n[1] = st
       result.add(n)
       result.add(ex)
 
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], needsSplit)
 
 proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
@@ -802,10 +808,11 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   let retStmt = newTree(nkReturnStmt, asgn)
   let branch = newTree(nkElifBranch, cmp, retStmt)
 
-  # The C++ backend requires `getCurrentException` here.
-  let raiseStmt = newTree(nkRaiseStmt, ctx.g.callCodegenProc("getCurrentException", ctx.g.emptyNode))
+  let nullifyExc = newTree(nkCall, newSymNode(ctx.g.getCompilerProc("closureIterSetupExc")), nilnode)
+  nullifyExc.info = info
+  let raiseStmt = newTree(nkRaiseStmt, curExc)
   raiseStmt.info = info
-  let elseBranch = newTree(nkElse, raiseStmt)
+  let elseBranch = newTree(nkElse, newTree(nkStmtList, nullifyExc, raiseStmt))
 
   let ifBody = newTree(nkIfStmt, branch, elseBranch)
   let elifBranch = newTree(nkElifBranch, ctx.newUnrollFinallyAccess(info), ifBody)
@@ -825,13 +832,14 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
     block: # :unrollFinally = true
       let asgn = newNodeI(nkAsgn, n.info)
       asgn.add(ctx.newUnrollFinallyAccess(n.info))
-      asgn.add(newIntTypeNode(nkIntLit, 1, ctx.g.getSysType(n.info, tyBool)))
+      asgn.add(newIntTypeNode(1, ctx.g.getSysType(n.info, tyBool)))
       result.add(asgn)
 
     if n[0].kind != nkEmpty:
       let asgnTmpResult = newNodeI(nkAsgn, n.info)
       asgnTmpResult.add(ctx.newTmpResultAccess())
-      asgnTmpResult.add(n[0])
+      let x = if n[0].kind in {nkAsgn, nkFastAsgn}: n[0][1] else: n[0]
+      asgnTmpResult.add(x)
       result.add(asgnTmpResult)
 
     result.add(ctx.newNullifyCurExc(n.info))
@@ -842,174 +850,166 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
   of nkSkip:
     discard
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformReturnsInTry(n[i])
 
 proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode =
   result = n
-  case n.kind:
-    of nkSkip:
-      discard
+  case n.kind
+  of nkSkip: discard
 
-    of nkStmtList, nkStmtListExpr:
-      assert(isEmptyType(n.typ), "nkStmtListExpr not lowered")
+  of nkStmtList, nkStmtListExpr:
+    result = addGotoOut(result, gotoOut)
+    for i in 0..<n.len:
+      if n[i].hasYields:
+        # Create a new split
+        let go = newNodeI(nkGotoState, n[i].info)
+        n[i] = ctx.transformClosureIteratorBody(n[i], go)
 
-      result = addGotoOut(result, gotoOut)
-      for i in 0 ..< n.len:
-        if n[i].hasYieldsInExpressions:
-          # Lower nkStmtListExpr nodes inside `n[i]` first
-          var ns = false
-          n[i] = ctx.lowerStmtListExprs(n[i], ns)
+        let s = newNodeI(nkStmtList, n[i + 1].info)
+        for j in i + 1..<n.len:
+          s.add(n[j])
 
-        if n[i].hasYields:
-          # Create a new split
-          let go = newNodeI(nkGotoState, n[i].info)
-          n[i] = ctx.transformClosureIteratorBody(n[i], go)
+        n.sons.setLen(i + 1)
+        discard ctx.newState(s, go)
+        if ctx.transformClosureIteratorBody(s, gotoOut) != s:
+          internalError(ctx.g.config, "transformClosureIteratorBody != s")
+        break
 
-          let s = newNodeI(nkStmtList, n[i + 1].info)
-          for j in i + 1 ..< n.len:
-            s.add(n[j])
+  of nkYieldStmt:
+    result = newNodeI(nkStmtList, n.info)
+    result.add(n)
+    result.add(gotoOut)
 
-          n.sons.setLen(i + 1)
-          discard ctx.newState(s, go)
-          if ctx.transformClosureIteratorBody(s, gotoOut) != s:
-            internalError(ctx.g.config, "transformClosureIteratorBody != s")
-          break
+  of nkElse, nkElseExpr:
+    result[0] = addGotoOut(result[0], gotoOut)
+    result[0] = ctx.transformClosureIteratorBody(result[0], gotoOut)
 
-    of nkYieldStmt:
-      result = newNodeI(nkStmtList, n.info)
-      result.add(n)
-      result.add(gotoOut)
-
-    of nkElse, nkElseExpr:
-      result[0] = addGotoOut(result[0], gotoOut)
-      result[0] = ctx.transformClosureIteratorBody(result[0], gotoOut)
-
-    of nkElifBranch, nkElifExpr, nkOfBranch:
-      result[^1] = addGotoOut(result[^1], gotoOut)
-      result[^1] = ctx.transformClosureIteratorBody(result[^1], gotoOut)
-
-    of nkIfStmt, nkCaseStmt:
-      for i in 0 ..< n.len:
-        n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
-      if n[^1].kind != nkElse:
-        # We don't have an else branch, but every possible branch has to end with
-        # gotoOut, so add else here.
-        let elseBranch = newTree(nkElse, gotoOut)
-        n.add(elseBranch)
-
-    of nkWhileStmt:
-      # while e:
-      #   s
-      # ->
-      # BEGIN_STATE:
-      #   if e:
-      #     s
-      #     goto BEGIN_STATE
-      #   else:
-      #     goto OUT
-
-      result = newNodeI(nkGotoState, n.info)
-
-      let s = newNodeI(nkStmtList, n.info)
-      discard ctx.newState(s, result)
-      let ifNode = newNodeI(nkIfStmt, n.info)
-      let elifBranch = newNodeI(nkElifBranch, n.info)
-      elifBranch.add(n[0])
-
-      var body = addGotoOut(n[1], result)
-
-      body = ctx.transformBreaksAndContinuesInWhile(body, result, gotoOut)
-      body = ctx.transformClosureIteratorBody(body, result)
-
-      elifBranch.add(body)
-      ifNode.add(elifBranch)
+  of nkElifBranch, nkElifExpr, nkOfBranch:
+    result[^1] = addGotoOut(result[^1], gotoOut)
+    result[^1] = ctx.transformClosureIteratorBody(result[^1], gotoOut)
 
+  of nkIfStmt, nkCaseStmt:
+    for i in 0..<n.len:
+      n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
+    if n[^1].kind != nkElse:
+      # We don't have an else branch, but every possible branch has to end with
+      # gotoOut, so add else here.
       let elseBranch = newTree(nkElse, gotoOut)
-      ifNode.add(elseBranch)
-      s.add(ifNode)
+      n.add(elseBranch)
 
-    of nkBlockStmt:
-      result[1] = addGotoOut(result[1], gotoOut)
-      result[1] = ctx.transformBreaksInBlock(result[1], result[0], gotoOut)
-      result[1] = ctx.transformClosureIteratorBody(result[1], gotoOut)
+  of nkWhileStmt:
+    # while e:
+    #   s
+    # ->
+    # BEGIN_STATE:
+    #   if e:
+    #     s
+    #     goto BEGIN_STATE
+    #   else:
+    #     goto OUT
 
-    of nkTryStmt:
-      # See explanation above about how this works
-      ctx.hasExceptions = true
+    result = newNodeI(nkGotoState, n.info)
 
-      result = newNodeI(nkGotoState, n.info)
-      var tryBody = toStmtList(n[0])
-      var exceptBody = ctx.collectExceptState(n)
-      var finallyBody = newTree(nkStmtList, getFinallyNode(ctx, n))
-      finallyBody = ctx.transformReturnsInTry(finallyBody)
-      finallyBody.add(ctx.newEndFinallyNode(finallyBody.info))
+    let s = newNodeI(nkStmtList, n.info)
+    discard ctx.newState(s, result)
+    let ifNode = newNodeI(nkIfStmt, n.info)
+    let elifBranch = newNodeI(nkElifBranch, n.info)
+    elifBranch.add(n[0])
 
-      # The following index calculation is based on the knowledge how state
-      # indexes are assigned
-      let tryIdx = ctx.states.len
-      var exceptIdx, finallyIdx: int
-      if exceptBody.kind != nkEmpty:
-        exceptIdx = -(tryIdx + 1)
-        finallyIdx = tryIdx + 2
-      else:
-        exceptIdx = tryIdx + 1
-        finallyIdx = tryIdx + 1
+    var body = addGotoOut(n[1], result)
 
-      let outToFinally = newNodeI(nkGotoState, finallyBody.info)
+    body = ctx.transformBreaksAndContinuesInWhile(body, result, gotoOut)
+    body = ctx.transformClosureIteratorBody(body, result)
 
-      block: # Create initial states.
-        let oldExcHandlingState = ctx.curExcHandlingState
-        ctx.curExcHandlingState = exceptIdx
-        let realTryIdx = ctx.newState(tryBody, result)
-        assert(realTryIdx == tryIdx)
+    elifBranch.add(body)
+    ifNode.add(elifBranch)
 
-        if exceptBody.kind != nkEmpty:
-          ctx.curExcHandlingState = finallyIdx
-          let realExceptIdx = ctx.newState(exceptBody, nil)
-          assert(realExceptIdx == -exceptIdx)
+    let elseBranch = newTree(nkElse, gotoOut)
+    ifNode.add(elseBranch)
+    s.add(ifNode)
 
-        ctx.curExcHandlingState = oldExcHandlingState
-        let realFinallyIdx = ctx.newState(finallyBody, outToFinally)
-        assert(realFinallyIdx == finallyIdx)
+  of nkBlockStmt:
+    result[1] = addGotoOut(result[1], gotoOut)
+    result[1] = ctx.transformBreaksInBlock(result[1], result[0], gotoOut)
+    result[1] = ctx.transformClosureIteratorBody(result[1], gotoOut)
 
-      block: # Subdivide the states
-        let oldNearestFinally = ctx.nearestFinally
-        ctx.nearestFinally = finallyIdx
+  of nkTryStmt, nkHiddenTryStmt:
+    # See explanation above about how this works
+    ctx.hasExceptions = true
 
-        let oldExcHandlingState = ctx.curExcHandlingState
-
-        ctx.curExcHandlingState = exceptIdx
-
-        if ctx.transformReturnsInTry(tryBody) != tryBody:
-          internalError(ctx.g.config, "transformReturnsInTry != tryBody")
-        if ctx.transformClosureIteratorBody(tryBody, outToFinally) != tryBody:
-          internalError(ctx.g.config, "transformClosureIteratorBody != tryBody")
-
-        ctx.curExcHandlingState = finallyIdx
-        ctx.addElseToExcept(exceptBody)
-        if ctx.transformReturnsInTry(exceptBody) != exceptBody:
-          internalError(ctx.g.config, "transformReturnsInTry != exceptBody")
-        if ctx.transformClosureIteratorBody(exceptBody, outToFinally) != exceptBody:
-          internalError(ctx.g.config, "transformClosureIteratorBody != exceptBody")
-
-        ctx.curExcHandlingState = oldExcHandlingState
-        ctx.nearestFinally = oldNearestFinally
-        if ctx.transformClosureIteratorBody(finallyBody, gotoOut) != finallyBody:
-          internalError(ctx.g.config, "transformClosureIteratorBody != finallyBody")
-
-    of nkGotoState, nkForStmt:
-      internalError(ctx.g.config, "closure iter " & $n.kind)
+    result = newNodeI(nkGotoState, n.info)
+    var tryBody = toStmtList(n[0])
+    var exceptBody = ctx.collectExceptState(n)
+    var finallyBody = newTree(nkStmtList, getFinallyNode(ctx, n))
+    finallyBody = ctx.transformReturnsInTry(finallyBody)
+    finallyBody.add(ctx.newEndFinallyNode(finallyBody.info))
 
+    # The following index calculation is based on the knowledge how state
+    # indexes are assigned
+    let tryIdx = ctx.states.len
+    var exceptIdx, finallyIdx: int
+    if exceptBody.kind != nkEmpty:
+      exceptIdx = -(tryIdx + 1)
+      finallyIdx = tryIdx + 2
     else:
-      for i in 0 ..< n.len:
-        n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
+      exceptIdx = tryIdx + 1
+      finallyIdx = tryIdx + 1
+
+    let outToFinally = newNodeI(nkGotoState, finallyBody.info)
+
+    block: # Create initial states.
+      let oldExcHandlingState = ctx.curExcHandlingState
+      ctx.curExcHandlingState = exceptIdx
+      let realTryIdx = ctx.newState(tryBody, result)
+      assert(realTryIdx == tryIdx)
+
+      if exceptBody.kind != nkEmpty:
+        ctx.curExcHandlingState = finallyIdx
+        let realExceptIdx = ctx.newState(exceptBody, nil)
+        assert(realExceptIdx == -exceptIdx)
+
+      ctx.curExcHandlingState = oldExcHandlingState
+      let realFinallyIdx = ctx.newState(finallyBody, outToFinally)
+      assert(realFinallyIdx == finallyIdx)
+
+    block: # Subdivide the states
+      let oldNearestFinally = ctx.nearestFinally
+      ctx.nearestFinally = finallyIdx
+
+      let oldExcHandlingState = ctx.curExcHandlingState
+
+      ctx.curExcHandlingState = exceptIdx
+
+      if ctx.transformReturnsInTry(tryBody) != tryBody:
+        internalError(ctx.g.config, "transformReturnsInTry != tryBody")
+      if ctx.transformClosureIteratorBody(tryBody, outToFinally) != tryBody:
+        internalError(ctx.g.config, "transformClosureIteratorBody != tryBody")
+
+      ctx.curExcHandlingState = finallyIdx
+      ctx.addElseToExcept(exceptBody)
+      if ctx.transformReturnsInTry(exceptBody) != exceptBody:
+        internalError(ctx.g.config, "transformReturnsInTry != exceptBody")
+      if ctx.transformClosureIteratorBody(exceptBody, outToFinally) != exceptBody:
+        internalError(ctx.g.config, "transformClosureIteratorBody != exceptBody")
+
+      ctx.curExcHandlingState = oldExcHandlingState
+      ctx.nearestFinally = oldNearestFinally
+      if ctx.transformClosureIteratorBody(finallyBody, gotoOut) != finallyBody:
+        internalError(ctx.g.config, "transformClosureIteratorBody != finallyBody")
+
+  of nkGotoState, nkForStmt:
+    internalError(ctx.g.config, "closure iter " & $n.kind)
+
+  else:
+    for i in 0..<n.len:
+      n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
 
 proc stateFromGotoState(n: PNode): int =
   assert(n.kind == nkGotoState)
   result = n[0].intVal.int
 
-proc tranformStateAssignments(ctx: var Ctx, n: PNode): PNode =
+proc transformStateAssignments(ctx: var Ctx, n: PNode): PNode =
   # This transforms 3 patterns:
   ########################## 1
   # yield e
@@ -1039,19 +1039,19 @@ proc tranformStateAssignments(ctx: var Ctx, n: PNode): PNode =
       result.add(ctx.newStateAssgn(stateFromGotoState(n[1])))
 
       var retStmt = newNodeI(nkReturnStmt, n.info)
-      if n[0].sons[0].kind != nkEmpty:
-        var a = newNodeI(nkAsgn, n[0].sons[0].info)
-        var retVal = n[0].sons[0] #liftCapturedVars(n.sons[0], owner, d, c)
-        addSon(a, newSymNode(getClosureIterResult(ctx.g, ctx.fn)))
-        addSon(a, retVal)
+      if n[0][0].kind != nkEmpty:
+        var a = newNodeI(nkAsgn, n[0][0].info)
+        var retVal = n[0][0] #liftCapturedVars(n[0], owner, d, c)
+        a.add newSymNode(getClosureIterResult(ctx.g, ctx.fn))
+        a.add retVal
         retStmt.add(a)
       else:
         retStmt.add(ctx.g.emptyNode)
 
       result.add(retStmt)
     else:
-      for i in 0 ..< n.len:
-        n[i] = ctx.tranformStateAssignments(n[i])
+      for i in 0..<n.len:
+        n[i] = ctx.transformStateAssignments(n[i])
 
   of nkSkip:
     discard
@@ -1070,8 +1070,8 @@ proc tranformStateAssignments(ctx: var Ctx, n: PNode): PNode =
     result.add(breakState)
 
   else:
-    for i in 0 ..< n.len:
-      n[i] = ctx.tranformStateAssignments(n[i])
+    for i in 0..<n.len:
+      n[i] = ctx.transformStateAssignments(n[i])
 
 proc skipStmtList(ctx: Ctx; n: PNode): PNode =
   result = n
@@ -1111,7 +1111,7 @@ proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
     result = copyTree(n)
     result[0].intVal = ctx.skipEmptyStates(result[0].intVal.int)
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.skipThroughEmptyStates(n[i])
 
 proc newArrayType(g: ModuleGraph; n: int, t: PType, owner: PSym): PType =
@@ -1149,7 +1149,6 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
 
   # :state = exceptionTable[:state]
   block:
-
     # exceptionTable[:state]
     let getNextState = newTree(nkBracketExpr,
       ctx.createExceptionTable(),
@@ -1164,7 +1163,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
     let cond = newTree(nkCall,
       ctx.g.getSysMagic(info, "==", mEqI).newSymNode(),
       ctx.newStateAccess(),
-      newIntTypeNode(nkIntLit, 0, intTyp))
+      newIntTypeNode(0, intTyp))
     cond.typ = boolTyp
 
     let raiseStmt = newTree(nkRaiseStmt, ctx.g.emptyNode)
@@ -1176,7 +1175,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
   block:
     let cond = newTree(nkCall,
       ctx.g.getSysMagic(info, "<", mLtI).newSymNode,
-      newIntTypeNode(nkIntLit, 0, intTyp),
+      newIntTypeNode(0, intTyp),
       ctx.newStateAccess())
     cond.typ = boolTyp
 
@@ -1188,7 +1187,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
     let cond = newTree(nkCall,
       ctx.g.getSysMagic(info, "<", mLtI).newSymNode,
       ctx.newStateAccess(),
-      newIntTypeNode(nkIntLit, 0, intTyp))
+      newIntTypeNode(0, intTyp))
     cond.typ = boolTyp
 
     let negateState = newTree(nkCall,
@@ -1204,7 +1203,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
   block:
     result.add(newTree(nkAsgn,
       ctx.newCurExcAccess(),
-      ctx.g.callCodegenProc("getCurrentException", ctx.g.emptyNode)))
+      ctx.g.callCodegenProc("getCurrentException")))
 
 proc wrapIntoTryExcept(ctx: var Ctx, n: PNode): PNode {.inline.} =
   let setupExc = newTree(nkCall,
@@ -1220,18 +1219,20 @@ proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   # while true:
   #   block :stateLoop:
   #     gotoState :state
+  #     local vars decl (if needed)
   #     body # Might get wrapped in try-except
   let loopBody = newNodeI(nkStmtList, n.info)
   result = newTree(nkWhileStmt, newSymNode(ctx.g.getSysSym(n.info, "true")), loopBody)
   result.info = n.info
 
+  let localVars = newNodeI(nkStmtList, n.info)
   if not ctx.stateVarSym.isNil:
     let varSect = newNodeI(nkVarSection, n.info)
     addVar(varSect, newSymNode(ctx.stateVarSym))
-    loopBody.add(varSect)
+    localVars.add(varSect)
 
     if not ctx.tempVars.isNil:
-      loopBody.add(ctx.tempVars)
+      localVars.add(ctx.tempVars)
 
   let blockStmt = newNodeI(nkBlockStmt, n.info)
   blockStmt.add(newSymNode(ctx.stateLoopLabel))
@@ -1240,7 +1241,7 @@ proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   gs.add(ctx.newStateAccess())
   gs.add(ctx.g.newIntLit(n.info, ctx.states.len - 1))
 
-  var blockBody = newTree(nkStmtList, gs, n)
+  var blockBody = newTree(nkStmtList, gs, localVars, n)
   if ctx.hasExceptions:
     blockBody = ctx.wrapIntoTryExcept(blockBody)
 
@@ -1287,25 +1288,30 @@ proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   ctx.fn = fn
 
   if getEnvParam(fn).isNil:
-    # Lambda lifting was not done yet. Use temporary :state sym, which
+    # Lambda lifting was not done yet. Use temporary :state sym, which will
     # be handled specially by lambda lifting. Local temp vars (if needed)
-    # should folllow the same logic.
+    # should follow the same logic.
     ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), fn, fn.info)
     ctx.stateVarSym.typ = g.createClosureIterStateType(fn)
-
   ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), fn, fn.info)
-  let n = n.toStmtList
+  var n = n.toStmtList
 
   discard ctx.newState(n, nil)
   let gotoOut = newTree(nkGotoState, g.newIntLit(n.info, -1))
 
+  var ns = false
+  n = ctx.lowerStmtListExprs(n, ns)
+
+  if n.hasYieldsInExpressions():
+    internalError(ctx.g.config, "yield in expr not lowered")
+
   # Splitting transformation
   discard ctx.transformClosureIteratorBody(n, gotoOut)
 
   # Optimize empty states away
   ctx.deleteEmptyStates()
 
-  # Make new body by concating the list of states
+  # Make new body by concatenating the list of states
   result = newNodeI(nkStmtList, n.info)
   for s in ctx.states:
     assert(s.len == 2)
@@ -1314,7 +1320,7 @@ proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
     result.add(s)
     result.add(body)
 
-  result = ctx.tranformStateAssignments(result)
+  result = ctx.transformStateAssignments(result)
   result = ctx.wrapIntoStateLoop(result)
 
   # echo "TRANSFORM TO STATES: "

compiler/cmdlinehelper.nim

@@ -10,9 +10,11 @@
 ## Helpers for binaries that use compiler passes, eg: nim, nimsuggest, nimfix
 
 import
-  options, idents, nimconf, scriptconfig, extccomp, commands, msgs,
+  options, idents, nimconf, extccomp, commands, msgs,
   lineinfos, modulegraphs, condsyms, os, pathutils
 
+from strutils import normalize
+
 type
   NimProg* = ref object
     suggestMode*: bool
@@ -27,10 +29,7 @@ proc initDefinesProg*(self: NimProg, conf: ConfigRef, name: string) =
 proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   self.processCmdLine(passCmd1, "", conf)
   if self.supportsStdinFile and conf.projectName == "-":
-    conf.projectName = "stdinfile"
-    conf.projectFull = AbsoluteFile "stdinfile"
-    conf.projectPath = AbsoluteDir getCurrentDir()
-    conf.projectIsStdin = true
+    handleStdinInput(conf)
   elif conf.projectName != "":
     try:
       conf.projectFull = canonicalizePath(conf, AbsoluteFile conf.projectName)
@@ -44,44 +43,31 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
     conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir())
 
 proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef): bool =
-  loadConfigs(DefaultConfig, cache, conf) # load all config files
   if self.suggestMode:
     conf.command = "nimsuggest"
+  loadConfigs(DefaultConfig, cache, conf) # load all config files
 
-  proc runNimScriptIfExists(path: AbsoluteFile)=
-    if fileExists(path):
-      runNimScript(cache, path, freshDefines = false, conf)
-
-  # Caution: make sure this stays in sync with `loadConfigs`
-  if optSkipSystemConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(getSystemConfigPath(conf, DefaultConfigNims))
-
-  if optSkipUserConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(getUserConfigPath(DefaultConfigNims))
-
-  if optSkipParentConfigFiles notin conf.globalOptions:
-    for dir in parentDirs(conf.projectPath.string, fromRoot = true, inclusive = false):
-      runNimScriptIfExists(AbsoluteDir(dir) / DefaultConfigNims)
-
-  if optSkipProjConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(conf.projectPath / DefaultConfigNims)
   block:
     let scriptFile = conf.projectFull.changeFileExt("nims")
     if not self.suggestMode:
-      runNimScriptIfExists(scriptFile)
       # 'nim foo.nims' means to just run the NimScript file and do nothing more:
       if fileExists(scriptFile) and scriptFile == conf.projectFull:
-        return false
-    else:
-      if scriptFile != conf.projectFull:
-        runNimScriptIfExists(scriptFile)
-      else:
-        # 'nimsuggest foo.nims' means to just auto-complete the NimScript file
-        discard
+        if conf.command == "":
+          conf.command = "e"
+          return false
+        elif conf.command.normalize == "e":
+          return false
 
   # now process command line arguments again, because some options in the
-  # command line can overwite the config file's settings
+  # command line can overwrite the config file's settings
   extccomp.initVars(conf)
+  # XXX This is hacky. We need to find a better way.
+  case conf.command
+  of "cpp", "compiletocpp":
+    conf.cmd = cmdCompileToCpp
+  else:
+    discard
+
   self.processCmdLine(passCmd2, "", conf)
   if conf.command == "":
     rawMessage(conf, errGenerated, "command missing")

compiler/commands.nim

@@ -17,7 +17,9 @@ template bootSwitch(name, expr, userString) =
   const name = if expr: " " & userString else: ""
 
 bootSwitch(usedRelease, defined(release), "-d:release")
-bootSwitch(usedGnuReadline, defined(useLinenoise), "-d:useLinenoise")
+bootSwitch(usedDanger, defined(danger), "-d:danger")
+# `useLinenoise` deprecated in favor of `nimUseLinenoise`, kept for backward compatibility
+bootSwitch(useLinenoise, defined(nimUseLinenoise) or defined(useLinenoise), "-d:nimUseLinenoise")
 bootSwitch(usedBoehm, defined(boehmgc), "--gc:boehm")
 bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
 bootSwitch(usedGenerational, defined(gcgenerational), "--gc:generational")
@@ -26,15 +28,18 @@ bootSwitch(usedNoGC, defined(nogc), "--gc:none")
 
 import
   os, msgs, options, nversion, condsyms, strutils, extccomp, platform,
-  wordrecg, parseutils, nimblecmd, idents, parseopt, sequtils, lineinfos,
-  pathutils
+  wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos,
+  pathutils, strtabs
+
+from incremental import nimIncremental
+from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect
 
 # but some have deps to imported modules. Yay.
 bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
 bootSwitch(usedNativeStacktrace,
   defined(nativeStackTrace) and nativeStackTraceSupported,
   "-d:nativeStackTrace")
-bootSwitch(usedFFI, hasFFI, "-d:useFFI")
+bootSwitch(usedFFI, hasFFI, "-d:nimHasLibFFI")
 
 type
   TCmdLinePass* = enum
@@ -47,15 +52,16 @@ const
       "Compiled at $4\n" &
       "Copyright (c) 2006-" & copyrightYear & " by Andreas Rumpf\n"
 
+proc genFeatureDesc[T: enum](t: typedesc[T]): string {.compileTime.} =
+  var x = ""
+  for f in low(T)..high(T):
+    if x.len > 0: x.add "|"
+    x.add $f
+  x
+
 const
-  Usage = slurp"../doc/basicopt.txt".replace("//", "")
-  FeatureDesc = block:
-    var x = ""
-    for f in low(Feature)..high(Feature):
-      if x.len > 0: x.add "|"
-      x.add $f
-    x
-  AdvancedUsage = slurp"../doc/advopt.txt".replace("//", "") % FeatureDesc
+  Usage = slurp"../doc/basicopt.txt".replace(" //", " ")
+  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", " ") % [genFeatureDesc(Feature), genFeatureDesc(LegacyFeature)]
 
 proc getCommandLineDesc(conf: ConfigRef): string =
   result = (HelpMessage % [VersionAsString, platform.OS[conf.target.hostOS].name,
@@ -96,25 +102,24 @@ proc writeVersionInfo(conf: ConfigRef; pass: TCmdLinePass) =
     when gitHash.len == 40:
       msgWriteln(conf, "git hash: " & gitHash, {msgStdout})
 
-    msgWriteln(conf, "active boot switches:" & usedRelease &
-      usedTinyC & usedGnuReadline & usedNativeStacktrace &
+    msgWriteln(conf, "active boot switches:" & usedRelease & usedDanger &
+      usedTinyC & useLinenoise & usedNativeStacktrace &
       usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC,
                {msgStdout})
     msgQuit(0)
 
-proc writeCommandLineUsage*(conf: ConfigRef; helpWritten: var bool) =
-  if not helpWritten:
-    msgWriteln(conf, getCommandLineDesc(conf), {msgStdout})
-    helpWritten = true
+proc writeCommandLineUsage*(conf: ConfigRef) =
+  msgWriteln(conf, getCommandLineDesc(conf), {msgStdout})
 
 proc addPrefix(switch: string): string =
-  if len(switch) == 1: result = "-" & switch
+  if switch.len <= 1: result = "-" & switch
   else: result = "--" & switch
 
 const
   errInvalidCmdLineOption = "invalid command line option: '$1'"
   errOnOrOffExpectedButXFound = "'on' or 'off' expected, but '$1' found"
   errOnOffOrListExpectedButXFound = "'on', 'off' or 'list' expected, but '$1' found"
+  errOffHintsError = "'off', 'hint' or 'error' expected, but '$1' found"
 
 proc invalidCmdLineOption(conf: ConfigRef; pass: TCmdLinePass, switch: string, info: TLineInfo) =
   if switch == " ": localError(conf, info, errInvalidCmdLineOption % "-")
@@ -124,14 +129,14 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
                  info: TLineInfo) =
   cmd = ""
   var i = 0
-  if i < len(switch) and switch[i] == '-': inc(i)
-  if i < len(switch) and switch[i] == '-': inc(i)
-  while i < len(switch):
+  if i < switch.len and switch[i] == '-': inc(i)
+  if i < switch.len and switch[i] == '-': inc(i)
+  while i < switch.len:
     case switch[i]
-    of 'a'..'z', 'A'..'Z', '0'..'9', '_', '.': add(cmd, switch[i])
+    of 'a'..'z', 'A'..'Z', '0'..'9', '_', '.': cmd.add(switch[i])
     else: break
     inc(i)
-  if i >= len(switch): arg = ""
+  if i >= switch.len: arg = ""
   # cmd:arg => (cmd,arg)
   elif switch[i] in {':', '='}: arg = substr(switch, i + 1)
   # cmd[sub]:rest => (cmd,[sub]:rest)
@@ -141,7 +146,7 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
 proc processOnOffSwitch(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass,
                         info: TLineInfo) =
   case arg.normalize
-  of "on": conf.options = conf.options + op
+  of "","on": conf.options = conf.options + op
   of "off": conf.options = conf.options - op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
@@ -157,7 +162,7 @@ proc processOnOffSwitchOrList(conf: ConfigRef; op: TOptions, arg: string, pass:
 proc processOnOffSwitchG(conf: ConfigRef; op: TGlobalOptions, arg: string, pass: TCmdLinePass,
                          info: TLineInfo) =
   case arg.normalize
-  of "on": conf.globalOptions = conf.globalOptions + op
+  of "", "on": conf.globalOptions = conf.globalOptions + op
   of "off": conf.globalOptions = conf.globalOptions - op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
@@ -175,17 +180,17 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
   var i = 0
   var n = hintMin
   var isBracket = false
-  if i < len(arg) and arg[i] == '[':
+  if i < arg.len and arg[i] == '[':
     isBracket = true
     inc(i)
-  while i < len(arg) and (arg[i] notin {':', '=', ']'}):
-    add(id, arg[i])
+  while i < arg.len and (arg[i] notin {':', '=', ']'}):
+    id.add(arg[i])
     inc(i)
   if isBracket:
-    if i < len(arg) and arg[i] == ']': inc(i)
+    if i < arg.len and arg[i] == ']': inc(i)
     else: invalidCmdLineOption(conf, pass, orig, info)
 
-  if i < len(arg) and (arg[i] in {':', '='}): inc(i)
+  if i < arg.len and (arg[i] in {':', '='}): inc(i)
   else: invalidCmdLineOption(conf, pass, orig, info)
   if state == wHint:
     let x = findStr(lineinfos.HintsToStr, id)
@@ -195,17 +200,21 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
     let x = findStr(lineinfos.WarningsToStr, id)
     if x >= 0: n = TNoteKind(x + ord(warnMin))
     else: localError(conf, info, "unknown warning: " & id)
-  case substr(arg, i).normalize
-  of "on":
-    incl(conf.notes, n)
-    incl(conf.mainPackageNotes, n)
-    incl(conf.enableNotes, n)
-  of "off":
-    excl(conf.notes, n)
-    excl(conf.mainPackageNotes, n)
-    incl(conf.disableNotes, n)
-    excl(conf.foreignPackageNotes, n)
-  else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
+
+  let val = substr(arg, i).normalize
+  if val notin ["on", "off"]:
+    localError(conf, info, errOnOrOffExpectedButXFound % arg)
+  elif n notin conf.cmdlineNotes or pass == passCmd1:
+    if pass == passCmd1: incl(conf.cmdlineNotes, n)
+    incl(conf.modifiedyNotes, n)
+    case val
+    of "on":
+      incl(conf.notes, n)
+      incl(conf.mainPackageNotes, n)
+    of "off":
+      excl(conf.notes, n)
+      excl(conf.mainPackageNotes, n)
+      excl(conf.foreignPackageNotes, n)
 
 proc processCompile(conf: ConfigRef; filename: string) =
   var found = findFile(conf, filename)
@@ -216,19 +225,22 @@ const
   errNoneBoehmRefcExpectedButXFound = "'none', 'boehm' or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found"
+  errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found"
 
 proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo): bool =
   case switch.normalize
   of "gc":
     case arg.normalize
-    of "boehm":        result = conf.selectedGC == gcBoehm
-    of "refc":         result = conf.selectedGC == gcRefc
-    of "v2":           result = conf.selectedGC == gcV2
+    of "boehm": result = conf.selectedGC == gcBoehm
+    of "refc": result = conf.selectedGC == gcRefc
+    of "v2": result = false
     of "markandsweep": result = conf.selectedGC == gcMarkAndSweep
     of "generational": result = false
-    of "destructors":  result = conf.selectedGC == gcDestructors
-    of "go":           result = conf.selectedGC == gcGo
-    of "none":         result = conf.selectedGC == gcNone
+    of "destructors", "arc": result = conf.selectedGC == gcArc
+    of "orc": result = conf.selectedGC == gcOrc
+    of "hooks": result = conf.selectedGC == gcHooks
+    of "go": result = conf.selectedGC == gcGo
+    of "none": result = conf.selectedGC == gcNone
     of "stack", "regions": result = conf.selectedGC == gcRegions
     else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "opt":
@@ -240,15 +252,22 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
   of "verbosity": result = $conf.verbosity == arg
   of "app":
     case arg.normalize
-    of "gui":       result = contains(conf.globalOptions, optGenGuiApp)
-    of "console":   result = not contains(conf.globalOptions, optGenGuiApp)
-    of "lib":       result = contains(conf.globalOptions, optGenDynLib) and
+    of "gui": result = contains(conf.globalOptions, optGenGuiApp)
+    of "console": result = not contains(conf.globalOptions, optGenGuiApp)
+    of "lib": result = contains(conf.globalOptions, optGenDynLib) and
                       not contains(conf.globalOptions, optGenGuiApp)
     of "staticlib": result = contains(conf.globalOptions, optGenStaticLib) and
                       not contains(conf.globalOptions, optGenGuiApp)
     else: localError(conf, info, errGuiConsoleOrLibExpectedButXFound % arg)
   of "dynliboverride":
     result = isDynlibOverride(conf, arg)
+  of "exceptions":
+    case arg.normalize
+    of "cpp": result = conf.exc == excCpp
+    of "setjmp": result = conf.exc == excSetjmp
+    of "quirky": result = conf.exc == excQuirky
+    of "goto": result = conf.exc == excGoto
+    else: localError(conf, info, errInvalidExceptionSystem % arg)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool =
@@ -262,8 +281,9 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "hints": result = contains(conf.options, optHints)
   of "threadanalysis": result = contains(conf.globalOptions, optThreadAnalysis)
   of "stacktrace": result = contains(conf.options, optStackTrace)
+  of "stacktracemsgs": result = contains(conf.options, optStackTraceMsgs)
   of "linetrace": result = contains(conf.options, optLineTrace)
-  of "debugger": result = contains(conf.options, optEndb)
+  of "debugger": result = contains(conf.globalOptions, optCDebug)
   of "profiler": result = contains(conf.options, optProfiler)
   of "memtracker": result = contains(conf.options, optMemTracker)
   of "checks", "x": result = conf.options * ChecksOptions == ChecksOptions
@@ -276,8 +296,10 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "fieldchecks": result = contains(conf.options, optFieldCheck)
   of "rangechecks": result = contains(conf.options, optRangeCheck)
   of "boundchecks": result = contains(conf.options, optBoundsCheck)
+  of "refchecks": result = contains(conf.options, optRefCheck)
   of "overflowchecks": result = contains(conf.options, optOverflowCheck)
-  of "movechecks": result = contains(conf.options, optMoveCheck)
+  of "staticboundchecks": result = contains(conf.options, optStaticBoundsCheck)
+  of "stylechecks": result = contains(conf.options, optStyleCheck)
   of "linedir": result = contains(conf.options, optLineDir)
   of "assertions", "a": result = contains(conf.options, optAssert)
   of "run", "r": result = contains(conf.globalOptions, optRun)
@@ -287,9 +309,10 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "taintmode": result = contains(conf.globalOptions, optTaintMode)
   of "tlsemulation": result = contains(conf.globalOptions, optTlsEmulation)
   of "implicitstatic": result = contains(conf.options, optImplicitStatic)
-  of "patterns": result = contains(conf.options, optPatterns)
+  of "patterns", "trmacros": result = contains(conf.options, optTrMacros)
   of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace)
   of "nilseqs": result = contains(conf.options, optNilSeqs)
+  of "oldast": result = contains(conf.options, optOldAst)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc processPath(conf: ConfigRef; path: string, info: TLineInfo,
@@ -354,6 +377,14 @@ proc dynlibOverride(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, in
     expectArg(conf, switch, arg, pass, info)
     options.inclDynlibOverride(conf, arg)
 
+proc handleStdinInput*(conf: ConfigRef) =
+  conf.projectName = "stdinfile"
+  conf.projectFull = conf.projectName.AbsoluteFile
+  conf.projectPath = AbsoluteDir getCurrentDir()
+  conf.projectIsStdin = true
+  if conf.outDir.isEmpty:
+    conf.outDir = getNimcacheDir(conf)
+
 proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
                     conf: ConfigRef) =
   var
@@ -361,8 +392,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   case switch.normalize
   of "path", "p":
     expectArg(conf, switch, arg, pass, info)
-    addPath(conf, if pass == passPP: processCfgPath(conf, arg, info)
-                  else: processPath(conf, arg, info), info)
+    for path in nimbleSubs(conf, arg):
+      addPath(conf, if pass == passPP: processCfgPath(conf, path, info)
+                    else: processPath(conf, path, info), info)
   of "nimblepath", "babelpath":
     # keep the old name for compat
     if pass in {passCmd2, passPP} and optNoNimblePath notin conf.globalOptions:
@@ -375,6 +407,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "nonimblepath", "nobabelpath":
     expectNoArg(conf, switch, arg, pass, info)
     disableNimblePath(conf)
+  of "clearnimblepath":
+    expectNoArg(conf, switch, arg, pass, info)
+    clearNimblePath(conf)
   of "excludepath":
     expectArg(conf, switch, arg, pass, info)
     let path = processPath(conf, arg, info)
@@ -382,21 +417,32 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     conf.lazyPaths.keepItIf(it != path)
   of "nimcache":
     expectArg(conf, switch, arg, pass, info)
-    conf.nimcacheDir = processPath(conf, arg, info, true)
+    conf.nimcacheDir = processPath(conf, arg, info, notRelativeToProj=true)
   of "out", "o":
     expectArg(conf, switch, arg, pass, info)
-    conf.outFile = AbsoluteFile arg
+    let f = splitFile(processPath(conf, arg, info, notRelativeToProj=true).string)
+    conf.outFile = RelativeFile f.name & f.ext
+    conf.outDir = toAbsoluteDir f.dir
+  of "outdir":
+    expectArg(conf, switch, arg, pass, info)
+    conf.outDir = processPath(conf, arg, info, notRelativeToProj=true)
   of "docseesrcurl":
     expectArg(conf, switch, arg, pass, info)
     conf.docSeeSrcUrl = arg
+  of "docroot":
+    conf.docRoot = if arg.len == 0: "@default" else: arg
   of "mainmodule", "m":
     discard "allow for backwards compatibility, but don't do anything"
   of "define", "d":
     expectArg(conf, switch, arg, pass, info)
     if {':', '='} in arg:
       splitSwitch(conf, arg, key, val, pass, info)
+      if cmpIgnoreStyle(key, "nimQuirky") == 0:
+        conf.exc = excQuirky
       defineSymbol(conf.symbols, key, val)
     else:
+      if cmpIgnoreStyle(arg, "nimQuirky") == 0:
+        conf.exc = excQuirky
       defineSymbol(conf.symbols, arg)
   of "undef", "u":
     expectArg(conf, switch, arg, pass, info)
@@ -412,52 +458,72 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if pass in {passCmd2, passPP}:
       addExternalFileToLink(conf, AbsoluteFile arg)
   of "debuginfo":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optCDebug)
+    processOnOffSwitchG(conf, {optCDebug}, arg, pass, info)
   of "embedsrc":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optEmbedOrigSrc)
+    processOnOffSwitchG(conf, {optEmbedOrigSrc}, arg, pass, info)
   of "compileonly", "c":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optCompileOnly)
+    processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info)
   of "nolinking":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optNoLinking)
+    processOnOffSwitchG(conf, {optNoLinking}, arg, pass, info)
   of "nomain":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optNoMain)
+    processOnOffSwitchG(conf, {optNoMain}, arg, pass, info)
   of "forcebuild", "f":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optForceFullMake)
+    processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
   of "project":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl conf.globalOptions, optWholeProject
+    processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
   of "gc":
     expectArg(conf, switch, arg, pass, info)
-    case arg.normalize
-    of "boehm":
-      conf.selectedGC = gcBoehm
-      defineSymbol(conf.symbols, "boehmgc")
-    of "refc":
-      conf.selectedGC = gcRefc
-    of "v2":
-      conf.selectedGC = gcV2
-    of "markandsweep":
-      conf.selectedGC = gcMarkAndSweep
-      defineSymbol(conf.symbols, "gcmarkandsweep")
-    of "destructors":
-      conf.selectedGC = gcDestructors
-      defineSymbol(conf.symbols, "gcdestructors")
-    of "go":
-      conf.selectedGC = gcGo
-      defineSymbol(conf.symbols, "gogc")
-    of "none":
-      conf.selectedGC = gcNone
-      defineSymbol(conf.symbols, "nogc")
-    of "stack", "regions":
-      conf.selectedGC= gcRegions
-      defineSymbol(conf.symbols, "gcregions")
-    else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
+    if pass in {passCmd2, passPP}:
+      case arg.normalize
+      of "boehm":
+        conf.selectedGC = gcBoehm
+        defineSymbol(conf.symbols, "boehmgc")
+      of "refc":
+        conf.selectedGC = gcRefc
+      of "v2":
+        message(conf, info, warnDeprecated, "--gc:v2 is deprecated; using default gc")
+      of "markandsweep":
+        conf.selectedGC = gcMarkAndSweep
+        defineSymbol(conf.symbols, "gcmarkandsweep")
+      of "destructors", "arc":
+        conf.selectedGC = gcArc
+        if conf.cmd != cmdCompileToCpp:
+          conf.exc = excGoto
+        defineSymbol(conf.symbols, "gcdestructors")
+        defineSymbol(conf.symbols, "gcarc")
+        incl conf.globalOptions, optSeqDestructors
+        incl conf.globalOptions, optTinyRtti
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+          defineSymbol(conf.symbols, "nimV2")
+      of "orc":
+        conf.selectedGC = gcOrc
+        if conf.cmd != cmdCompileToCpp:
+          conf.exc = excGoto
+        defineSymbol(conf.symbols, "gcdestructors")
+        defineSymbol(conf.symbols, "gcorc")
+        incl conf.globalOptions, optSeqDestructors
+        incl conf.globalOptions, optTinyRtti
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+          defineSymbol(conf.symbols, "nimV2")
+      of "hooks":
+        conf.selectedGC = gcHooks
+        defineSymbol(conf.symbols, "gchooks")
+        incl conf.globalOptions, optSeqDestructors
+        processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+      of "go":
+        conf.selectedGC = gcGo
+        defineSymbol(conf.symbols, "gogc")
+      of "none":
+        conf.selectedGC = gcNone
+        defineSymbol(conf.symbols, "nogc")
+      of "stack", "regions":
+        conf.selectedGC = gcRegions
+        defineSymbol(conf.symbols, "gcregions")
+      else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "warnings", "w":
     if processOnOffSwitchOrList(conf, {optWarns}, arg, pass, info): listWarnings(conf)
   of "warning": processSpecificNote(arg, wWarning, pass, info, switch, conf)
@@ -466,23 +532,23 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if processOnOffSwitchOrList(conf, {optHints}, arg, pass, info): listHints(conf)
   of "threadanalysis": processOnOffSwitchG(conf, {optThreadAnalysis}, arg, pass, info)
   of "stacktrace": processOnOffSwitch(conf, {optStackTrace}, arg, pass, info)
+  of "stacktracemsgs": processOnOffSwitch(conf, {optStackTraceMsgs}, arg, pass, info)
   of "excessivestacktrace": processOnOffSwitchG(conf, {optExcessiveStackTrace}, arg, pass, info)
   of "linetrace": processOnOffSwitch(conf, {optLineTrace}, arg, pass, info)
   of "debugger":
     case arg.normalize
-    of "on", "endb":
-      conf.options.incl optEndb
-      defineSymbol(conf.symbols, "endb")
+    of "on", "native", "gdb":
+      conf.globalOptions.incl optCDebug
+      conf.options.incl optLineDir
+      #defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
     of "off":
-      conf.options.excl optEndb
-      undefSymbol(conf.symbols, "endb")
-    of "native", "gdb":
-      incl(conf.globalOptions, optCDebug)
-      conf.options = conf.options + {optLineDir} - {optEndb}
-      defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
-      undefSymbol(conf.symbols, "endb")
+      conf.globalOptions.excl optCDebug
     else:
-      localError(conf, info, "expected endb|gdb but found " & arg)
+      localError(conf, info, "expected native|gdb|on|off but found " & arg)
+  of "g": # alias for --debugger:native
+    conf.globalOptions.incl optCDebug
+    conf.options.incl optLineDir
+    #defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
   of "profiler":
     processOnOffSwitch(conf, {optProfiler}, arg, pass, info)
     if optProfiler in conf.options: defineSymbol(conf.symbols, "profiler")
@@ -492,12 +558,20 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if optMemTracker in conf.options: defineSymbol(conf.symbols, "memtracker")
     else: undefSymbol(conf.symbols, "memtracker")
   of "hotcodereloading":
-    processOnOffSwitch(conf, {optHotCodeReloading}, arg, pass, info)
-    if optHotCodeReloading in conf.options: defineSymbol(conf.symbols, "hotcodereloading")
-    else: undefSymbol(conf.symbols, "hotcodereloading")
+    processOnOffSwitchG(conf, {optHotCodeReloading}, arg, pass, info)
+    if conf.hcrOn:
+      defineSymbol(conf.symbols, "hotcodereloading")
+      defineSymbol(conf.symbols, "useNimRtl")
+      # hardcoded linking with dynamic runtime for MSVC for smaller binaries
+      # should do the same for all compilers (wherever applicable)
+      if isVSCompatible(conf):
+        extccomp.addCompileOptionCmd(conf, "/MD")
+    else:
+      undefSymbol(conf.symbols, "hotcodereloading")
+      undefSymbol(conf.symbols, "useNimRtl")
   of "oldnewlines":
     case arg.normalize
-    of "on":
+    of "", "on":
       conf.oldNewlines = true
       defineSymbol(conf.symbols, "nimOldNewlines")
     of "off":
@@ -507,6 +581,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       localError(conf, info, errOnOrOffExpectedButXFound % arg)
   of "laxstrings": processOnOffSwitch(conf, {optLaxStrings}, arg, pass, info)
   of "nilseqs": processOnOffSwitch(conf, {optNilSeqs}, arg, pass, info)
+  of "oldast": processOnOffSwitch(conf, {optOldAst}, arg, pass, info)
   of "checks", "x": processOnOffSwitch(conf, ChecksOptions, arg, pass, info)
   of "floatchecks":
     processOnOffSwitch(conf, {optNaNCheck, optInfCheck}, arg, pass, info)
@@ -517,20 +592,22 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "fieldchecks": processOnOffSwitch(conf, {optFieldCheck}, arg, pass, info)
   of "rangechecks": processOnOffSwitch(conf, {optRangeCheck}, arg, pass, info)
   of "boundchecks": processOnOffSwitch(conf, {optBoundsCheck}, arg, pass, info)
+  of "refchecks": processOnOffSwitch(conf, {optRefCheck}, arg, pass, info)
   of "overflowchecks": processOnOffSwitch(conf, {optOverflowCheck}, arg, pass, info)
-  of "movechecks": processOnOffSwitch(conf, {optMoveCheck}, arg, pass, info)
+  of "staticboundchecks": processOnOffSwitch(conf, {optStaticBoundsCheck}, arg, pass, info)
+  of "stylechecks": processOnOffSwitch(conf, {optStyleCheck}, arg, pass, info)
   of "linedir": processOnOffSwitch(conf, {optLineDir}, arg, pass, info)
   of "assertions", "a": processOnOffSwitch(conf, {optAssert}, arg, pass, info)
   of "deadcodeelim": discard # deprecated, dead code elim always on
   of "threads":
     processOnOffSwitchG(conf, {optThreads}, arg, pass, info)
-    #if optThreads in conf.globalOptions: incl(conf.notes, warnGcUnsafe)
+    #if optThreads in conf.globalOptions: conf.setNote(warnGcUnsafe)
   of "tlsemulation": processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info)
   of "taintmode": processOnOffSwitchG(conf, {optTaintMode}, arg, pass, info)
   of "implicitstatic":
     processOnOffSwitch(conf, {optImplicitStatic}, arg, pass, info)
-  of "patterns":
-    processOnOffSwitch(conf, {optPatterns}, arg, pass, info)
+  of "patterns", "trmacros":
+    processOnOffSwitch(conf, {optTrMacros}, arg, pass, info)
   of "opt":
     expectArg(conf, switch, arg, pass, info)
     case arg.normalize
@@ -589,42 +666,57 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitchG(conf, {optGenIndex}, arg, pass, info)
   of "import":
     expectArg(conf, switch, arg, pass, info)
-    if pass in {passCmd2, passPP}: conf.implicitImports.add arg
+    if pass in {passCmd2, passPP}:
+      conf.implicitImports.add findModule(conf, arg, toFullPath(conf, info)).string
   of "include":
     expectArg(conf, switch, arg, pass, info)
-    if pass in {passCmd2, passPP}: conf.implicitIncludes.add arg
+    if pass in {passCmd2, passPP}:
+      conf.implicitIncludes.add findModule(conf, arg, toFullPath(conf, info)).string
   of "listcmd":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optListCmd)
+    processOnOffSwitchG(conf, {optListCmd}, arg, pass, info)
+  of "asm":
+    processOnOffSwitchG(conf, {optProduceAsm}, arg, pass, info)
   of "genmapping":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optGenMapping)
+    processOnOffSwitchG(conf, {optGenMapping}, arg, pass, info)
   of "os":
     expectArg(conf, switch, arg, pass, info)
-    if pass in {passCmd1, passPP}:
-      let theOS = platform.nameToOS(arg)
-      if theOS == osNone: localError(conf, info, "unknown OS: '$1'" % arg)
-      elif theOS != conf.target.hostOS:
-        setTarget(conf.target, theOS, conf.target.targetCPU)
+    let theOS = platform.nameToOS(arg)
+    if theOS == osNone:
+      let osList = platform.listOSnames().join(", ")
+      localError(conf, info, "unknown OS: '$1'. Available options are: $2" % [arg, $osList])
+    else:
+      setTarget(conf.target, theOS, conf.target.targetCPU)
   of "cpu":
     expectArg(conf, switch, arg, pass, info)
-    if pass in {passCmd1, passPP}:
-      let cpu = platform.nameToCPU(arg)
-      if cpu == cpuNone: localError(conf, info, "unknown CPU: '$1'" % arg)
-      elif cpu != conf.target.hostCPU:
-        setTarget(conf.target, conf.target.targetOS, cpu)
+    let cpu = platform.nameToCPU(arg)
+    if cpu == cpuNone:
+      let cpuList = platform.listCPUnames().join(", ")
+      localError(conf, info, "unknown CPU: '$1'. Available options are: $2" % [ arg, cpuList])
+    else:
+      setTarget(conf.target, conf.target.targetOS, cpu)
   of "run", "r":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optRun)
+    processOnOffSwitchG(conf, {optRun}, arg, pass, info)
+  of "maxloopiterationsvm":
+    expectArg(conf, switch, arg, pass, info)
+    conf.maxLoopIterationsVM = parseInt(arg)
+  of "errormax":
+    expectArg(conf, switch, arg, pass, info)
+    # Note: `nim check` (etc) can overwrite this.
+    # `0` is meaningless, give it a useful meaning as in clang's -ferror-limit
+    # If user doesn't set this flag and the code doesn't either, it'd
+    # have the same effect as errorMax = 1
+    let ret = parseInt(arg)
+    conf.errorMax = if ret == 0: high(int) else: ret
   of "verbosity":
     expectArg(conf, switch, arg, pass, info)
     let verbosity = parseInt(arg)
     if verbosity notin {0..3}:
       localError(conf, info, "invalid verbosity level: '$1'" % arg)
     conf.verbosity = verbosity
-    conf.notes = NotesVerbosity[conf.verbosity]
-    incl(conf.notes, conf.enableNotes)
-    excl(conf.notes, conf.disableNotes)
+    var verb = NotesVerbosity[conf.verbosity]
+    ## We override the default `verb` by explicitly modified (set/unset) notes.
+    conf.notes = (conf.modifiedyNotes * conf.notes + verb) -
+      (conf.modifiedyNotes * verb - conf.notes)
     conf.mainPackageNotes = conf.notes
   of "parallelbuild":
     expectArg(conf, switch, arg, pass, info)
@@ -641,30 +733,30 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "help", "h":
     expectNoArg(conf, switch, arg, pass, info)
     helpOnError(conf, pass)
-  of "symbolfiles", "incremental":
+  of "symbolfiles": discard "ignore for backwards compat"
+  of "incremental":
+    when not nimIncremental:
+      localError(conf, info, "the compiler was not built with " &
+        "incremental compilation features; bootstrap with " &
+        "-d:nimIncremental to enable")
     case arg.normalize
     of "on": conf.symbolFiles = v2Sf
     of "off": conf.symbolFiles = disabledSf
     of "writeonly": conf.symbolFiles = writeOnlySf
     of "readonly": conf.symbolFiles = readOnlySf
     of "v2": conf.symbolFiles = v2Sf
-    else: localError(conf, info, "invalid option for --symbolFiles: " & arg)
+    else: localError(conf, info, "invalid option for --incremental: " & arg)
   of "skipcfg":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optSkipSystemConfigFile)
+    processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info)
   of "skipprojcfg":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optSkipProjConfigFile)
+    processOnOffSwitchG(conf, {optSkipProjConfigFile}, arg, pass, info)
   of "skipusercfg":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optSkipUserConfigFile)
+    processOnOffSwitchG(conf, {optSkipUserConfigFile}, arg, pass, info)
   of "skipparentcfg":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optSkipParentConfigFiles)
+    processOnOffSwitchG(conf, {optSkipParentConfigFiles}, arg, pass, info)
   of "genscript", "gendeps":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optGenScript)
-    incl(conf.globalOptions, optCompileOnly)
+    processOnOffSwitchG(conf, {optGenScript}, arg, pass, info)
+    processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info)
   of "colors": processOnOffSwitchG(conf, {optUseColors}, arg, pass, info)
   of "lib":
     expectArg(conf, switch, arg, pass, info)
@@ -698,16 +790,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectNoArg(conf, switch, arg, pass, info)
     conf.ideCmd = ideUse
   of "stdout":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optStdout)
+    processOnOffSwitchG(conf, {optStdout}, arg, pass, info)
   of "listfullpaths":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl conf.globalOptions, optListFullPaths
+    processOnOffSwitchG(conf, {optListFullPaths}, arg, pass, info)
   of "dynliboverride":
     dynlibOverride(conf, switch, arg, pass, info)
   of "dynliboverrideall":
-    expectNoArg(conf, switch, arg, pass, info)
-    incl conf.globalOptions, optDynlibOverrideAll
+    processOnOffSwitchG(conf, {optDynlibOverrideAll}, arg, pass, info)
   of "cs":
     # only supported for compatibility. Does nothing.
     expectArg(conf, switch, arg, pass, info)
@@ -719,32 +808,99 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         conf.features.incl parseEnum[Feature](arg)
       except ValueError:
         localError(conf, info, "unknown experimental feature")
+  of "legacy":
+    try:
+      conf.legacyFeatures.incl parseEnum[LegacyFeature](arg)
+    except ValueError:
+      localError(conf, info, "unknown obsolete feature")
   of "nocppexceptions":
     expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optNoCppExceptions)
+    conf.exc = low(ExceptionSystem)
     defineSymbol(conf.symbols, "noCppExceptions")
+  of "exceptions":
+    case arg.normalize
+    of "cpp": conf.exc = excCpp
+    of "setjmp": conf.exc = excSetjmp
+    of "quirky": conf.exc = excQuirky
+    of "goto": conf.exc = excGoto
+    else: localError(conf, info, errInvalidExceptionSystem % arg)
   of "cppdefine":
     expectArg(conf, switch, arg, pass, info)
     if conf != nil:
       conf.cppDefine(arg)
   of "newruntime":
     expectNoArg(conf, switch, arg, pass, info)
-    doAssert(conf != nil)
-    incl(conf.features, destructor)
-    defineSymbol(conf.symbols, "nimNewRuntime")
-  of "nep1":
-    processOnOffSwitchG(conf, {optCheckNep1}, arg, pass, info)
+    if pass in {passCmd2, passPP}:
+      doAssert(conf != nil)
+      incl(conf.features, destructor)
+      incl(conf.globalOptions, optTinyRtti)
+      incl(conf.globalOptions, optOwnedRefs)
+      incl(conf.globalOptions, optSeqDestructors)
+      defineSymbol(conf.symbols, "nimV2")
+      conf.selectedGC = gcHooks
+      defineSymbol(conf.symbols, "gchooks")
+      defineSymbol(conf.symbols, "nimSeqsV2")
+      defineSymbol(conf.symbols, "nimOwnedEnabled")
+  of "seqsv2":
+    processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
+    if pass in {passCmd2, passPP}:
+      defineSymbol(conf.symbols, "nimSeqsV2")
+  of "stylecheck":
+    case arg.normalize
+    of "off": conf.globalOptions = conf.globalOptions - {optStyleHint, optStyleError}
+    of "hint": conf.globalOptions = conf.globalOptions + {optStyleHint} - {optStyleError}
+    of "error": conf.globalOptions = conf.globalOptions + {optStyleError}
+    else: localError(conf, info, errOffHintsError % arg)
+  of "showallmismatches":
+    processOnOffSwitchG(conf, {optShowAllMismatches}, arg, pass, info)
   of "cppcompiletonamespace":
     if arg.len > 0:
       conf.cppCustomNamespace = arg
     else:
       conf.cppCustomNamespace = "Nim"
     defineSymbol(conf.symbols, "cppCompileToNamespace", conf.cppCustomNamespace)
+  of "docinternal":
+    processOnOffSwitchG(conf, {optDocInternal}, arg, pass, info)
+  of "multimethods":
+    processOnOffSwitchG(conf, {optMultiMethods}, arg, pass, info)
+  of "expandmacro":
+    expectArg(conf, switch, arg, pass, info)
+    conf.macrosToExpand[arg] = "T"
+  of "oldgensym":
+    processOnOffSwitchG(conf, {optNimV019}, arg, pass, info)
+  of "useversion":
+    expectArg(conf, switch, arg, pass, info)
+    case arg
+    of "1.0":
+      defineSymbol(conf.symbols, "NimMajor", "1")
+      defineSymbol(conf.symbols, "NimMinor", "0")
+      # always be compatible with 1.0.100:
+      defineSymbol(conf.symbols, "NimPatch", "100")
+      # old behaviors go here:
+      defineSymbol(conf.symbols, "nimOldRelativePathBehavior")
+      ast.eqTypeFlags.excl {tfGcSafe, tfNoSideEffect}
+      conf.globalOptions.incl optNimV1Emulation
+    else:
+      localError(conf, info, "unknown Nim version; currently supported values are: {1.0}")
+  of "benchmarkvm":
+    processOnOffSwitchG(conf, {optBenchmarkVM}, arg, pass, info)
+  of "sinkinference":
+    processOnOffSwitch(conf, {optSinkInference}, arg, pass, info)
+  of "panics":
+    processOnOffSwitchG(conf, {optPanics}, arg, pass, info)
+    if optPanics in conf.globalOptions:
+      defineSymbol(conf.symbols, "nimPanics")
+  of "sourcemap":
+    conf.globalOptions.incl optSourcemap
+    conf.options.incl optLineDir
+    # processOnOffSwitchG(conf, {optSourcemap, opt}, arg, pass, info)
+  of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -)
+    handleStdinInput(conf)
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)
 
-template gCmdLineInfo*(): untyped = newLineInfo(config, AbsoluteFile"command line", 1, 1)
+template gCmdLineInfo*(): untyped = newLineInfo(commandLineIdx, 1, 1)
 
 proc processCommand*(switch: string, pass: TCmdLinePass; config: ConfigRef) =
   var cmd, arg: string
@@ -767,8 +923,9 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser;
                       argsCount: var int; config: ConfigRef): bool =
   if argsCount == 0:
     # nim filename.nims  is the same as "nim e filename.nims":
-    if p.key.endswith(".nims"):
+    if p.key.endsWith(".nims"):
       config.command = "e"
+      incl(config.globalOptions, optWasNimscript)
       config.projectName = unixToNativePath(p.key)
       config.arguments = cmdLineRest(p)
       result = true
@@ -778,7 +935,8 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser;
     if pass == passCmd1: config.commandArgs.add p.key
     if argsCount == 1:
       # support UNIX style filenames everywhere for portable build scripts:
-      config.projectName = unixToNativePath(p.key)
+      if config.projectName.len == 0:
+        config.projectName = unixToNativePath(p.key)
       config.arguments = cmdLineRest(p)
       result = true
   inc argsCount

compiler/condsyms.nim

@@ -10,31 +10,26 @@
 # This module handles the conditional symbols.
 
 import
-  strtabs, platform, strutils, idents
+  strtabs
 
 from options import Feature
 from lineinfos import HintsToStr, WarningsToStr
 
-const
-  catNone = "false"
-
 proc defineSymbol*(symbols: StringTableRef; symbol: string, value: string = "true") =
   symbols[symbol] = value
 
 proc undefSymbol*(symbols: StringTableRef; symbol: string) =
-  symbols[symbol] = catNone
+  symbols.del(symbol)
 
 #proc lookupSymbol*(symbols: StringTableRef; symbol: string): string =
 #  result = if isDefined(symbol): gSymbols[symbol] else: nil
 
 iterator definedSymbolNames*(symbols: StringTableRef): string =
   for key, val in pairs(symbols):
-    if val != catNone: yield key
+    yield key
 
 proc countDefinedSymbols*(symbols: StringTableRef): int =
-  result = 0
-  for key, val in pairs(symbols):
-    if val != catNone: inc(result)
+  symbols.len
 
 proc initDefines*(symbols: StringTableRef) =
   # for bootstrapping purposes and old code:
@@ -53,13 +48,11 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimalias")
   defineSymbol("nimlocks")
   defineSymbol("nimnode")
-  defineSymbol("nimnomagic64")
   defineSymbol("nimvarargstyped")
   defineSymbol("nimtypedescfixed")
   defineSymbol("nimKnowsNimvm")
   defineSymbol("nimArrIdx")
-  defineSymbol("nimImmediateDeprecated")
-  defineSymbol("nimNewShiftOps")
+  defineSymbol("nimHasalignOf")
   defineSymbol("nimDistros")
   defineSymbol("nimHasCppDefine")
   defineSymbol("nimGenericInOutFlags")
@@ -82,8 +75,17 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimNoNilSeqs")
   defineSymbol("nimNoNilSeqs2")
   defineSymbol("nimHasUserErrors")
-
+  defineSymbol("nimUncheckedArrayTyp")
+  defineSymbol("nimHasTypeof")
+  defineSymbol("nimErrorProcCanHaveBody")
+  defineSymbol("nimHasInstantiationOfInMacro")
+  defineSymbol("nimHasHotCodeReloading")
   defineSymbol("nimHasNilSeqs")
+  defineSymbol("nimHasSignatureHashInMacro")
+  defineSymbol("nimHasDefault")
+  defineSymbol("nimMacrosSizealignof")
+  defineSymbol("nimNoZeroExtendMagic")
+  defineSymbol("nimMacrosGetNodeId")
   for f in low(Feature)..high(Feature):
     defineSymbol("nimHas" & $f)
 
@@ -91,3 +93,26 @@ proc initDefines*(symbols: StringTableRef) =
     defineSymbol("nimHasWarning" & s)
   for s in HintsToStr:
     defineSymbol("nimHasHint" & s)
+
+  defineSymbol("nimFixedOwned")
+  defineSymbol("nimHasStyleChecks")
+  defineSymbol("nimToOpenArrayCString")
+  defineSymbol("nimHasUsed")
+  defineSymbol("nimFixedForwardGeneric")
+  defineSymbol("nimnomagic64")
+  defineSymbol("nimNewShiftOps")
+  defineSymbol("nimHasCursor")
+  defineSymbol("nimHasExceptionsQuery")
+  defineSymbol("nimHasIsNamedTuple")
+  defineSymbol("nimHashOrdinalFixed")
+
+  when defined(nimHasLibFFI):
+    # Renaming as we can't conflate input vs output define flags; e.g. this
+    # will report the right thing regardless of whether user adds
+    # `-d:nimHasLibFFI` in his user config.
+    defineSymbol("nimHasLibFFIEnabled")
+
+  defineSymbol("nimHasSinkInference")
+  defineSymbol("nimNewIntegerOps")
+  defineSymbol("nimHasInvariant")
+  defineSymbol("nimHasStacktraceMsgs")

compiler/configuration.nim

@@ -1,6 +0,0 @@
-## Use the module 'lineinfos' instead!
-
-{.deprecated.}
-
-import lineinfos
-export lineinfos

compiler/depends.nim

@@ -10,13 +10,12 @@
 # This module implements a dependency file generator.
 
 import
-  os, options, ast, astalgo, msgs, ropes, idents, passes, modulepaths,
-  pathutils
+  options, ast, ropes, idents, passes, modulepaths, pathutils
 
-from modulegraphs import ModuleGraph
+from modulegraphs import ModuleGraph, PPassContext
 
 type
-  TGen = object of TPassContext
+  TGen = object of PPassContext
     module: PSym
     config: ConfigRef
     graph: ModuleGraph
@@ -26,7 +25,7 @@ type
     dotGraph: Rope
 
 proc addDependencyAux(b: Backend; importing, imported: string) =
-  addf(b.dotGraph, "$1 -> \"$2\";$n", [rope(importing), rope(imported)])
+  b.dotGraph.addf("$1 -> \"$2\";$n", [rope(importing), rope(imported)])
   # s1 -> s2_4[label="[0-9]"];
 
 proc addDotDependency(c: PPassContext, n: PNode): PNode =
@@ -35,14 +34,14 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode =
   let b = Backend(g.graph.backend)
   case n.kind
   of nkImportStmt:
-    for i in countup(0, sonsLen(n) - 1):
-      var imported = getModuleName(g.config, n.sons[i])
+    for i in 0..<n.len:
+      var imported = getModuleName(g.config, n[i])
       addDependencyAux(b, g.module.name.s, imported)
   of nkFromStmt, nkImportExceptStmt:
-    var imported = getModuleName(g.config, n.sons[0])
+    var imported = getModuleName(g.config, n[0])
     addDependencyAux(b, g.module.name.s, imported)
   of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr:
-    for i in countup(0, sonsLen(n) - 1): discard addDotDependency(c, n.sons[i])
+    for i in 0..<n.len: discard addDotDependency(c, n[i])
   else:
     discard
 

compiler/destroyer.nim

@@ -1,483 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2017 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Injects destructor calls into Nim code as well as
-## an optimizer that optimizes copies to moves. This is implemented as an
-## AST to AST transformation so that every backend benefits from it.
-
-## Rules for destructor injections:
-##
-## foo(bar(X(), Y()))
-## X and Y get destroyed after bar completes:
-##
-## foo( (tmpX = X(); tmpY = Y(); tmpBar = bar(tmpX, tmpY);
-##       destroy(tmpX); destroy(tmpY);
-##       tmpBar))
-## destroy(tmpBar)
-##
-## var x = f()
-## body
-##
-## is the same as:
-##
-##  var x;
-##  try:
-##    move(x, f())
-##  finally:
-##    destroy(x)
-##
-## But this really just an optimization that tries to avoid to
-## introduce too many temporaries, the 'destroy' is caused by
-## the 'f()' call. No! That is not true for 'result = f()'!
-##
-## x = y where y is read only once
-## is the same as:  move(x, y)
-##
-## Actually the more general rule is: The *last* read of ``y``
-## can become a move if ``y`` is the result of a construction.
-##
-## We also need to keep in mind here that the number of reads is
-## control flow dependent:
-## let x = foo()
-## while true:
-##   y = x  # only one read, but the 2nd iteration will fail!
-## This also affects recursions! Only usages that do not cross
-## a loop boundary (scope) and are not used in function calls
-## are safe.
-##
-##
-## x = f() is the same as:  move(x, f())
-##
-## x = y
-## is the same as:  copy(x, y)
-##
-## Reassignment works under this scheme:
-## var x = f()
-## x = y
-##
-## is the same as:
-##
-##  var x;
-##  try:
-##    move(x, f())
-##    copy(x, y)
-##  finally:
-##    destroy(x)
-##
-##  result = f()  must not destroy 'result'!
-##
-## The produced temporaries clutter up the code and might lead to
-## inefficiencies. A better strategy is to collect all the temporaries
-## in a single object that we put into a single try-finally that
-## surrounds the proc body. This means the code stays quite efficient
-## when compiled to C. In fact, we do the same for variables, so
-## destructors are called when the proc returns, not at scope exit!
-## This makes certains idioms easier to support. (Taking the slice
-## of a temporary object.)
-##
-## foo(bar(X(), Y()))
-## X and Y get destroyed after bar completes:
-##
-## var tmp: object
-## foo( (move tmp.x, X(); move tmp.y, Y(); tmp.bar = bar(tmpX, tmpY);
-##       tmp.bar))
-## destroy(tmp.bar)
-## destroy(tmp.x); destroy(tmp.y)
-##
-
-##[
-From https://github.com/nim-lang/Nim/wiki/Destructors
-
-Rule      Pattern                 Transformed into
-----      -------                 ----------------
-1.1	      var x: T; stmts	        var x: T; try stmts
-                                  finally: `=destroy`(x)
-1.2       var x: sink T; stmts    var x: sink T; stmts; ensureEmpty(x)
-2         x = f()                 `=sink`(x, f())
-3         x = lastReadOf z        `=sink`(x, z); wasMoved(z)
-4.1       y = sinkParam           `=sink`(y, sinkParam)
-4.2       x = y                   `=`(x, y) # a copy
-5.1       f_sink(g())             f_sink(g())
-5.2       f_sink(y)               f_sink(copy y); # copy unless we can see it's the last read
-5.3       f_sink(move y)          f_sink(y); wasMoved(y) # explicit moves empties 'y'
-5.4       f_noSink(g())           var tmp = bitwiseCopy(g()); f(tmp); `=destroy`(tmp)
-
-Remarks: Rule 1.2 is not yet implemented because ``sink`` is currently
-  not allowed as a local variable.
-
-``move`` builtin needs to be implemented.
-]##
-
-import
-  intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
-  strutils, options, dfa, lowerings, tables, modulegraphs,
-  lineinfos
-
-const
-  InterestingSyms = {skVar, skResult, skLet}
-
-type
-  Con = object
-    owner: PSym
-    g: ControlFlowGraph
-    jumpTargets: IntSet
-    tmpObj: PType
-    tmp: PSym
-    destroys, topLevelVars: PNode
-    toDropBit: Table[int, PSym]
-    graph: ModuleGraph
-    emptyNode: PNode
-
-proc getTemp(c: var Con; typ: PType; info: TLineInfo): PNode =
-  # XXX why are temps fields in an object here?
-  let f = newSym(skField, getIdent(c.graph.cache, ":d" & $c.tmpObj.n.len), c.owner, info)
-  f.typ = typ
-  rawAddField c.tmpObj, f
-  result = rawDirectAccess(c.tmp, f)
-
-proc isHarmlessVar*(s: PSym; c: Con): bool =
-  # 's' is harmless if it used only once and its
-  # definition/usage are not split by any labels:
-  #
-  # let s = foo()
-  # while true:
-  #   a[i] = s
-  #
-  # produces:
-  #
-  # def s
-  # L1:
-  #   use s
-  # goto L1
-  #
-  # let s = foo()
-  # if cond:
-  #   a[i] = s
-  # else:
-  #   a[j] = s
-  #
-  # produces:
-  #
-  # def s
-  # fork L2
-  # use s
-  # goto L3
-  # L2:
-  # use s
-  # L3
-  #
-  # So this analysis is for now overly conservative, but correct.
-  var defsite = -1
-  var usages = 0
-  for i in 0..<c.g.len:
-    case c.g[i].kind
-    of def:
-      if c.g[i].sym == s:
-        if defsite < 0: defsite = i
-        else: return false
-    of use, useWithinCall:
-      if c.g[i].sym == s:
-        if defsite < 0: return false
-        for j in defsite .. i:
-          # not within the same basic block?
-          if j in c.jumpTargets: return false
-        # if we want to die after the first 'use':
-        if usages > 1: return false
-        inc usages
-    #of useWithinCall:
-    #  if c.g[i].sym == s: return false
-    of goto, fork:
-      discard "we do not perform an abstract interpretation yet"
-  result = usages <= 1
-
-template interestingSym(s: PSym): bool =
-  s.owner == c.owner and s.kind in InterestingSyms and hasDestructor(s.typ)
-
-proc patchHead(n: PNode) =
-  if n.kind in nkCallKinds and n[0].kind == nkSym and n.len > 1:
-    let s = n[0].sym
-    if s.name.s[0] == '=' and s.name.s in ["=sink", "=", "=destroy"]:
-      if sfFromGeneric in s.flags:
-        excl(s.flags, sfFromGeneric)
-        patchHead(s.getBody)
-      if n[1].typ.isNil:
-        # XXX toptree crashes without this workaround. Figure out why.
-        return
-      let t = n[1].typ.skipTypes({tyVar, tyLent, tyGenericInst, tyAlias, tySink, tyInferred})
-      template patch(op, field) =
-        if s.name.s == op and field != nil and field != s:
-          n.sons[0].sym = field
-      patch "=sink", t.sink
-      patch "=", t.assignment
-      patch "=destroy", t.destructor
-  for x in n:
-    patchHead(x)
-
-proc patchHead(s: PSym) =
-  if sfFromGeneric in s.flags:
-    patchHead(s.ast[bodyPos])
-
-proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
-  var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">"
-  if opname == "=" and ri != nil:
-    m.add "; requires a copy because it's not the last read of '"
-    m.add renderTree(ri)
-    m.add '\''
-  localError(c.graph.config, ri.info, errGenerated, m)
-
-template genOp(opr, opname, ri) =
-  let op = opr
-  if op == nil:
-    globalError(c.graph.config, dest.info, "internal error: '" & opname & "' operator not found for type " & typeToString(t))
-  elif op.ast[genericParamsPos].kind != nkEmpty:
-    globalError(c.graph.config, dest.info, "internal error: '" & opname & "' operator is generic")
-  patchHead op
-  if sfError in op.flags: checkForErrorPragma(c, t, ri, opname)
-  result = newTree(nkCall, newSymNode(op), newTree(nkHiddenAddr, dest))
-
-proc genSink(c: Con; t: PType; dest, ri: PNode): PNode =
-  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  genOp(if t.sink != nil: t.sink else: t.assignment, "=sink", ri)
-
-proc genCopy(c: Con; t: PType; dest, ri: PNode): PNode =
-  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  genOp(t.assignment, "=", ri)
-
-proc genDestroy(c: Con; t: PType; dest: PNode): PNode =
-  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  genOp(t.destructor, "=destroy", nil)
-
-proc addTopVar(c: var Con; v: PNode) =
-  c.topLevelVars.add newTree(nkIdentDefs, v, c.emptyNode, c.emptyNode)
-
-proc dropBit(c: var Con; s: PSym): PSym =
-  result = c.toDropBit.getOrDefault(s.id)
-  assert result != nil
-
-proc registerDropBit(c: var Con; s: PSym) =
-  let result = newSym(skTemp, getIdent(c.graph.cache, s.name.s & "_AliveBit"), c.owner, s.info)
-  result.typ = getSysType(c.graph, s.info, tyBool)
-  let trueVal = newIntTypeNode(nkIntLit, 1, result.typ)
-  c.topLevelVars.add newTree(nkIdentDefs, newSymNode result, c.emptyNode, trueVal)
-  c.toDropBit[s.id] = result
-  # generate:
-  #  if not sinkParam_AliveBit: `=destroy`(sinkParam)
-  let t = s.typ.skipTypes({tyGenericInst, tyAlias, tySink})
-  if t.destructor != nil:
-    c.destroys.add newTree(nkIfStmt,
-      newTree(nkElifBranch, newSymNode result, genDestroy(c, t, newSymNode s)))
-
-proc p(n: PNode; c: var Con): PNode
-
-template recurse(n, dest) =
-  for i in 0..<n.len:
-    dest.add p(n[i], c)
-
-proc isSinkParam(s: PSym): bool {.inline.} =
-  result = s.kind == skParam and s.typ.kind == tySink
-
-const constrExprs = nkCallKinds+{nkObjConstr}
-
-proc destructiveMoveSink(n: PNode; c: var Con): PNode =
-  # generate:  (chckMove(sinkParam_AliveBit); sinkParam_AliveBit = false; sinkParam)
-  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
-  let bit = newSymNode dropBit(c, n.sym)
-  if optMoveCheck in c.owner.options:
-    result.add callCodegenProc(c.graph, "chckMove", bit)
-  result.add newTree(nkAsgn, bit,
-    newIntTypeNode(nkIntLit, 0, getSysType(c.graph, n.info, tyBool)))
-  result.add n
-
-proc genMagicCall(n: PNode; c: var Con; magicname: string; m: TMagic): PNode =
-  result = newNodeI(nkCall, n.info)
-  result.add(newSymNode(createMagic(c.graph, magicname, m)))
-  result.add n
-
-proc moveOrCopy(dest, ri: PNode; c: var Con): PNode =
-  if ri.kind in constrExprs:
-    result = genSink(c, dest.typ, dest, ri)
-    # watch out and no not transform 'ri' twice if it's a call:
-    let ri2 = copyNode(ri)
-    recurse(ri, ri2)
-    result.add ri2
-  elif ri.kind == nkSym and ri.sym.kind != skParam and isHarmlessVar(ri.sym, c):
-    # Rule 3: `=sink`(x, z); wasMoved(z)
-    var snk = genSink(c, dest.typ, dest, ri)
-    snk.add p(ri, c)
-    result = newTree(nkStmtList, snk, genMagicCall(ri, c, "wasMoved", mWasMoved))
-  elif ri.kind == nkSym and isSinkParam(ri.sym):
-    result = genSink(c, dest.typ, dest, ri)
-    result.add destructiveMoveSink(ri, c)
-  else:
-    result = genCopy(c, dest.typ, dest, ri)
-    result.add p(ri, c)
-
-proc passCopyToSink(n: PNode; c: var Con): PNode =
-  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
-  let tmp = getTemp(c, n.typ, n.info)
-  if hasDestructor(n.typ):
-    var m = genCopy(c, n.typ, tmp, n)
-    m.add p(n, c)
-    result.add m
-    message(c.graph.config, n.info, hintPerformance,
-      ("passing '$1' to a sink parameter introduces an implicit copy; " &
-      "use 'move($1)' to prevent it") % $n)
-  else:
-    result.add newTree(nkAsgn, tmp, p(n, c))
-  result.add tmp
-
-proc genWasMoved(n: PNode; c: var Con): PNode =
-  # The mWasMoved builtin does not take the address.
-  result = genMagicCall(n, c, "wasMoved", mWasMoved)
-
-proc destructiveMoveVar(n: PNode; c: var Con): PNode =
-  # generate: (let tmp = v; reset(v); tmp)
-  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
-
-  var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), c.owner, n.info)
-  temp.typ = n.typ
-  var v = newNodeI(nkLetSection, n.info)
-  let tempAsNode = newSymNode(temp)
-
-  var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
-  vpart.sons[0] = tempAsNode
-  vpart.sons[1] = c.emptyNode
-  vpart.sons[2] = n
-  add(v, vpart)
-
-  result.add v
-  result.add genWasMoved(n, c)
-  result.add tempAsNode
-
-proc p(n: PNode; c: var Con): PNode =
-  case n.kind
-  of nkVarSection, nkLetSection:
-    discard "transform; var x = y to  var x; x op y  where op is a move or copy"
-    result = newNodeI(nkStmtList, n.info)
-
-    for i in 0..<n.len:
-      let it = n[i]
-      let L = it.len-1
-      let ri = it[L]
-      if it.kind == nkVarTuple and hasDestructor(ri.typ):
-        let x = lowerTupleUnpacking(c.graph, it, c.owner)
-        result.add p(x, c)
-      elif it.kind == nkIdentDefs and hasDestructor(it[0].typ):
-        for j in 0..L-2:
-          let v = it[j]
-          doAssert v.kind == nkSym
-          # move the variable declaration to the top of the frame:
-          c.addTopVar v
-          # make sure it's destroyed at the end of the proc:
-          c.destroys.add genDestroy(c, v.typ, v)
-          if ri.kind != nkEmpty:
-            let r = moveOrCopy(v, ri, c)
-            result.add r
-      else:
-        # keep it, but transform 'ri':
-        var varSection = copyNode(n)
-        var itCopy = copyNode(it)
-        for j in 0..L-1:
-          itCopy.add it[j]
-        itCopy.add p(ri, c)
-        varSection.add itCopy
-        result.add varSection
-  of nkCallKinds:
-    let parameters = n[0].typ
-    let L = if parameters != nil: parameters.len else: 0
-    for i in 1 ..< n.len:
-      let arg = n[i]
-      if i < L and parameters[i].kind == tySink:
-        if arg.kind in nkCallKinds:
-          # recurse but skip the call expression in order to prevent
-          # destructor injections: Rule 5.1 is different from rule 5.4!
-          let a = copyNode(arg)
-          recurse(arg, a)
-          n.sons[i] = a
-        elif arg.kind in {nkObjConstr, nkCharLit..nkFloat128Lit}:
-          discard "object construction to sink parameter: nothing to do"
-        elif arg.kind == nkSym and isHarmlessVar(arg.sym, c):
-          # if x is a variable and it its last read we eliminate its
-          # destructor invokation, but don't. We need to reset its memory
-          # to disable its destructor which we have not elided:
-          n.sons[i] = destructiveMoveVar(arg, c)
-        elif arg.kind == nkSym and isSinkParam(arg.sym):
-          # mark the sink parameter as used:
-          n.sons[i] = destructiveMoveSink(arg, c)
-        else:
-          # an object that is not temporary but passed to a 'sink' parameter
-          # results in a copy.
-          n.sons[i] = passCopyToSink(arg, c)
-      else:
-        n.sons[i] = p(arg, c)
-
-    if n.typ != nil and hasDestructor(n.typ):
-      discard "produce temp creation"
-      result = newNodeIT(nkStmtListExpr, n.info, n.typ)
-      let tmp = getTemp(c, n.typ, n.info)
-      var sinkExpr = genSink(c, n.typ, tmp, n)
-      sinkExpr.add n
-      result.add sinkExpr
-      result.add tmp
-      c.destroys.add genDestroy(c, n.typ, tmp)
-    else:
-      result = n
-  of nkAsgn, nkFastAsgn:
-    if hasDestructor(n[0].typ):
-      result = moveOrCopy(n[0], n[1], c)
-    else:
-      result = copyNode(n)
-      recurse(n, result)
-  of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef,
-      nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef:
-    result = n
-  else:
-    result = copyNode(n)
-    recurse(n, result)
-
-proc injectDestructorCalls*(g: ModuleGraph; owner: PSym; n: PNode): PNode =
-  when defined(nimDebugDestroys):
-    echo "injecting into ", n
-  var c: Con
-  c.owner = owner
-  c.tmp = newSym(skTemp, getIdent(g.cache, ":d"), owner, n.info)
-  c.tmpObj = createObj(g, owner, n.info)
-  c.tmp.typ = c.tmpObj
-  c.destroys = newNodeI(nkStmtList, n.info)
-  c.topLevelVars = newNodeI(nkVarSection, n.info)
-  c.toDropBit = initTable[int, PSym]()
-  c.graph = g
-  c.emptyNode = newNodeI(nkEmpty, n.info)
-  let cfg = constructCfg(owner, n)
-  shallowCopy(c.g, cfg)
-  c.jumpTargets = initIntSet()
-  for i in 0..<c.g.len:
-    if c.g[i].kind in {goto, fork}:
-      c.jumpTargets.incl(i+c.g[i].dest)
-  if owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter}:
-    let params = owner.typ.n
-    for i in 1 ..< params.len:
-      let param = params[i].sym
-      if param.typ.kind == tySink: registerDropBit(c, param)
-  let body = p(n, c)
-  if c.tmp.typ.n.len > 0:
-    c.addTopVar(newSymNode c.tmp)
-  result = newNodeI(nkStmtList, n.info)
-  if c.topLevelVars.len > 0:
-    result.add c.topLevelVars
-  if c.destroys.len > 0:
-    result.add newTryFinally(body, c.destroys)
-  else:
-    result.add body
-
-  when defined(nimDebugDestroys):
-    if owner.name.s == "main" or true:
-      echo "------------------------------------"
-      echo owner.name.s, " transformed to: "
-      echo result

compiler/dfa.nim

@@ -7,9 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-## Data flow analysis for Nim. For now the task is to prove that every
-## usage of a local variable 'v' is covered by an initialization to 'v'
-## first.
+## Data flow analysis for Nim.
 ## We transform the AST into a linear list of instructions first to
 ## make this easier to handle: There are only 2 different branching
 ## instructions: 'goto X' is an unconditional goto, 'fork X'
@@ -17,26 +15,32 @@
 ## taken). Exhaustive case statements are translated
 ## so that the last branch is transformed into an 'else' branch.
 ## ``return`` and ``break`` are all covered by 'goto'.
-## The case to detect is ``use v`` that is not dominated by
-## a ``def v``.
+##
+## Control flow through exception handling:
+## Contrary to popular belief, exception handling doesn't cause
+## many problems for this DFA representation, ``raise`` is a statement
+## that ``goes to`` the outer ``finally`` or ``except`` if there is one,
+## otherwise it is the same as ``return``. Every call is treated as
+## a call that can potentially ``raise``. However, without a surrounding
+## ``try`` we don't emit these ``fork ReturnLabel`` instructions in order
+## to speed up the dataflow analysis passes.
+##
 ## The data structures and algorithms used here are inspired by
 ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
 ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
 
-import ast, astalgo, types, intsets, tables, msgs, options, lineinfos
+import ast, types, intsets, lineinfos, renderer
+
+from patterns import sameTrees
 
 type
   InstrKind* = enum
-    goto, fork, def, use,
-    useWithinCall # this strange special case is used to get more
-                  # move optimizations out of regular code
-                  # XXX This is still overly pessimistic in
-                  # call(let x = foo; bar(x))
+    goto, fork, join, def, use
   Instr* = object
-    n*: PNode
+    n*: PNode # contains the def/use location.
     case kind*: InstrKind
-    of def, use, useWithinCall: sym*: PSym
-    of goto, fork: dest*: int
+    of goto, fork, join: dest*: int
+    else: discard
 
   ControlFlowGraph* = seq[Instr]
 
@@ -45,13 +49,13 @@ type
     label: PSym
     fixups: seq[TPosition]
 
-  ValueKind = enum
-    undef, value, valueOrUndef
-
   Con = object
     code: ControlFlowGraph
-    inCall: int
+    inCall, inTryStmt: int
     blocks: seq[TBlock]
+    tryStmtFixups: seq[TPosition]
+    forks: seq[TPosition]
+    owner: PSym
 
 proc debugInfo(info: TLineInfo): string =
   result = $info.line #info.toFilename & ":" & $info.line
@@ -62,55 +66,209 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
   var jumpTargets = initIntSet()
   let last = if last < 0: c.len-1 else: min(last, c.len-1)
   for i in start..last:
-    if c[i].kind in {goto, fork}:
+    if c[i].kind in {goto, fork, join}:
       jumpTargets.incl(i+c[i].dest)
   var i = start
   while i <= last:
     if i in jumpTargets: result.add("L" & $i & ":\n")
     result.add "\t"
-    result.add $c[i].kind
+    result.add ($i & " " & $c[i].kind)
     result.add "\t"
     case c[i].kind
-    of def, use, useWithinCall:
-      result.add c[i].sym.name.s
-    of goto, fork:
+    of def, use:
+      result.add renderTree(c[i].n)
+    of goto, fork, join:
       result.add "L"
-      result.add c[i].dest+i
+      result.addInt c[i].dest+i
     result.add("\t#")
     result.add(debugInfo(c[i].n.info))
     result.add("\n")
     inc i
   if i in jumpTargets: result.add("L" & $i & ": End\n")
-
+  # consider calling `asciitables.alignTable`
 
 proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
   ## echos the ControlFlowGraph for debugging purposes.
   var buf = ""
   codeListing(c, buf, start, last)
-  when declared(echo):
-    echo buf
+  echo buf
 
 proc forkI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: fork, dest: 0)
+  c.forks.add result
 
 proc gotoI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: goto, dest: 0)
 
+#[
+
+Design of join
+==============
+
+block:
+  if cond: break
+  def(x)
+
+use(x)
+
+Generates:
+
+L0: fork lab1
+  join L0  # patched.
+  goto Louter
+lab1:
+  def x
+  join L0
+Louter:
+  use x
+
+
+block outer:
+  while a:
+    while b:
+      if foo:
+        if bar:
+          break outer # --> we need to 'join' every pushed 'fork' here
+
+
+This works and then our abstract interpretation needs to deal with 'fork'
+differently. It really causes a split in execution. Two threads are
+"spawned" and both need to reach the 'join L' instruction. Afterwards
+the abstract interpretations are joined and execution resumes single
+threaded.
+
+
+Abstract Interpretation
+-----------------------
+
+proc interpret(pc, state, comesFrom): state =
+  result = state
+  # we need an explicit 'create' instruction (an explicit heap), in order
+  # to deal with 'var x = create(); var y = x; var z = y; destroy(z)'
+  while true:
+    case pc
+    of fork:
+      let a = interpret(pc+1, result, pc)
+      let b = interpret(forkTarget, result, pc)
+      result = a ++ b # ++ is a union operation
+      inc pc
+    of join:
+      if joinTarget == comesFrom: return result
+      else: inc pc
+    of use X:
+      if not result.contains(x):
+        error "variable not initialized " & x
+      inc pc
+    of def X:
+      if not result.contains(x):
+        result.incl X
+      else:
+        error "overwrite of variable causes memory leak " & x
+      inc pc
+    of destroy X:
+      result.excl X
+
+This is correct but still can lead to false positives:
+
+proc p(cond: bool) =
+  if cond:
+    new(x)
+  otherThings()
+  if cond:
+    destroy x
+
+Is not a leak. We should find a way to model *data* flow, not just
+control flow. One solution is to rewrite the 'if' without a fork
+instruction. The unstructured aspect can now be easily dealt with
+the 'goto' and 'join' instructions.
+
+proc p(cond: bool) =
+  L0: fork Lend
+    new(x)
+    # do not 'join' here!
+
+  Lend:
+    otherThings()
+    join L0  # SKIP THIS FOR new(x) SOMEHOW
+  destroy x
+  join L0 # but here.
+
+
+
+But if we follow 'goto Louter' we will never come to the join point.
+We restore the bindings after popping pc from the stack then there
+"no" problem?!
+
+
+while cond:
+  prelude()
+  if not condB: break
+  postlude()
+
+--->
+var setFlag = true
+while cond and not setFlag:
+  prelude()
+  if not condB:
+    setFlag = true   # BUT: Dependency
+  if not setFlag:    # HERE
+    postlude()
+
+--->
+var setFlag = true
+while cond and not setFlag:
+  prelude()
+  if not condB:
+    postlude()
+    setFlag = true
+
+
+-------------------------------------------------
+
+while cond:
+  prelude()
+  if more:
+    if not condB: break
+    stuffHere()
+  postlude()
+
+-->
+var setFlag = true
+while cond and not setFlag:
+  prelude()
+  if more:
+    if not condB:
+      setFlag = false
+    else:
+      stuffHere()
+      postlude()
+  else:
+    postlude()
+
+This is getting complicated. Instead we keep the whole 'join' idea but
+duplicate the 'join' instructions on breaks and return exits!
+
+]#
+
+proc joinI(c: var Con; fromFork: TPosition; n: PNode) =
+  let dist = fromFork.int - c.code.len
+  c.code.add Instr(n: n, kind: join, dest: dist)
+
 proc genLabel(c: Con): TPosition =
   result = TPosition(c.code.len)
 
 proc jmpBack(c: var Con, n: PNode, p = TPosition(0)) =
   let dist = p.int - c.code.len
-  doAssert(-0x7fff < dist and dist < 0x7fff)
+  doAssert(low(int) div 2 + 1 < dist and dist < high(int) div 2)
   c.code.add Instr(n: n, kind: goto, dest: dist)
 
 proc patch(c: var Con, p: TPosition) =
   # patch with current index
   let p = p.int
   let diff = c.code.len - p
-  doAssert(-0x7fff < diff and diff < 0x7fff)
+  doAssert(low(int) div 2 + 1 < diff and diff < high(int) div 2)
   c.code[p].dest = diff
 
 proc popBlock(c: var Con; oldLen: int) =
@@ -130,134 +288,402 @@ proc isTrue(n: PNode): bool =
 
 proc gen(c: var Con; n: PNode) # {.noSideEffect.}
 
-proc genWhile(c: var Con; n: PNode) =
-  # L1:
-  #   cond, tmp
-  #   fork tmp, L2
-  #   body
-  #   jmp L1
-  # L2:
-  let L1 = c.genLabel
-  withBlock(nil):
-    if isTrue(n.sons[0]):
-      c.gen(n.sons[1])
-      c.jmpBack(n, L1)
+when true:
+  proc genWhile(c: var Con; n: PNode) =
+    # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
+    # through the loop. We need to prove this is correct for our
+    # purposes. But Herb Sutter claims it is. (Proof by authority.)
+    #[
+    while cond:
+      body
+
+    Becomes:
+
+    if cond:
+      body
+      if cond:
+        body
+        if cond:
+          body
+
+    We still need to ensure 'break' resolves properly, so an AST to AST
+    translation is impossible.
+
+    So the code to generate is:
+
+      cond
+      fork L4  # F1
+      body
+      cond
+      fork L5  # F2
+      body
+      cond
+      fork L6  # F3
+      body
+    L6:
+      join F3
+    L5:
+      join F2
+    L4:
+      join F1
+    ]#
+    if isTrue(n[0]):
+      # 'while true' is an idiom in Nim and so we produce
+      # better code for it:
+      for i in 0..2:
+        withBlock(nil):
+          c.gen(n[1])
     else:
-      c.gen(n.sons[0])
-      let L2 = c.forkI(n)
-      c.gen(n.sons[1])
-      c.jmpBack(n, L1)
-      c.patch(L2)
+      let oldForksLen = c.forks.len
+      var endings: array[3, TPosition]
+      for i in 0..2:
+        withBlock(nil):
+          c.gen(n[0])
+          endings[i] = c.forkI(n)
+          c.gen(n[1])
+      for i in countdown(endings.high, 0):
+        let endPos = endings[i]
+        c.patch(endPos)
+        c.joinI(c.forks.pop(), n)
+      doAssert(c.forks.len == oldForksLen)
+
+else:
+
+  proc genWhile(c: var Con; n: PNode) =
+    # lab1:
+    #   cond, tmp
+    #   fork tmp, lab2
+    #   body
+    #   jmp lab1
+    # lab2:
+    let oldForksLen = c.forks.len
+    let lab1 = c.genLabel
+    withBlock(nil):
+      if isTrue(n[0]):
+        c.gen(n[1])
+        c.jmpBack(n, lab1)
+      else:
+        c.gen(n[0])
+        let lab2 = c.forkI(n)
+        c.gen(n[1])
+        c.jmpBack(n, lab1)
+        c.patch(lab2)
+    setLen(c.forks, oldForksLen)
 
 proc genBlock(c: var Con; n: PNode) =
-  withBlock(n.sons[0].sym):
-    c.gen(n.sons[1])
+  withBlock(n[0].sym):
+    c.gen(n[1])
+
+proc genJoins(c: var Con; n: PNode) =
+  for i in countdown(c.forks.high, 0): joinI(c, c.forks[i], n)
 
 proc genBreak(c: var Con; n: PNode) =
-  let L1 = c.gotoI(n)
-  if n.sons[0].kind == nkSym:
-    #echo cast[int](n.sons[0].sym)
+  genJoins(c, n)
+  let lab1 = c.gotoI(n)
+  if n[0].kind == nkSym:
+    #echo cast[int](n[0].sym)
     for i in countdown(c.blocks.len-1, 0):
-      if c.blocks[i].label == n.sons[0].sym:
-        c.blocks[i].fixups.add L1
+      if c.blocks[i].label == n[0].sym:
+        c.blocks[i].fixups.add lab1
         return
     #globalError(n.info, "VM problem: cannot find 'break' target")
   else:
-    c.blocks[c.blocks.high].fixups.add L1
+    c.blocks[c.blocks.high].fixups.add lab1
+
+template forkT(n, body) =
+  let oldLen = c.forks.len
+  let lab1 = c.forkI(n)
+  body
+  c.patch(lab1)
+  c.joinI(lab1, n)
+  setLen(c.forks, oldLen)
 
 proc genIf(c: var Con, n: PNode) =
+  #[
+
+  if cond:
+    A
+  elif condB:
+    B
+  elif condC:
+    C
+  else:
+    D
+
+  cond
+  fork lab1
+  A
+  goto Lend
+  lab1:
+    condB
+    fork lab2
+    B
+    goto Lend2
+  lab2:
+    condC
+    fork L3
+    C
+    goto Lend3
+  L3:
+    D
+    goto Lend3 # not eliminated to simplify the join generation
+  Lend3:
+    join F3
+  Lend2:
+    join F2
+  Lend:
+    join F1
+
+  ]#
+  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
-  for i in countup(0, len(n) - 1):
-    var it = n.sons[i]
-    c.gen(it.sons[0])
+  for i in 0..<n.len:
+    var it = n[i]
+    c.gen(it[0])
     if it.len == 2:
-      let elsePos = c.forkI(it.sons[1])
-      c.gen(it.sons[1])
-      if i < sonsLen(n)-1:
-        endings.add(c.gotoI(it.sons[1]))
+      let elsePos = forkI(c, it[1])
+      c.gen(it[1])
+      endings.add(c.gotoI(it[1]))
       c.patch(elsePos)
-  for endPos in endings: c.patch(endPos)
+  for i in countdown(endings.high, 0):
+    let endPos = endings[i]
+    c.patch(endPos)
+    c.joinI(c.forks.pop(), n)
+  doAssert(c.forks.len == oldLen)
 
 proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, a
-  #   fork L1
+  #   fork lab1
   #   asgn dest, b
-  # L1:
-  c.gen(n.sons[1])
-  let L1 = c.forkI(n)
-  c.gen(n.sons[2])
-  c.patch(L1)
+  # lab1:
+  #   join F1
+  c.gen(n[1])
+  forkT(n):
+    c.gen(n[2])
 
 proc genCase(c: var Con; n: PNode) =
-  #  if (!expr1) goto L1;
+  #  if (!expr1) goto lab1;
   #    thenPart
   #    goto LEnd
-  #  L1:
-  #  if (!expr2) goto L2;
+  #  lab1:
+  #  if (!expr2) goto lab2;
   #    thenPart2
   #    goto LEnd
-  #  L2:
+  #  lab2:
   #    elsePart
   #  Lend:
+  let isExhaustive = skipTypes(n[0].typ,
+    abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
+
   var endings: seq[TPosition] = @[]
-  c.gen(n.sons[0])
-  for i in 1 ..< n.len:
-    let it = n.sons[i]
+  let oldLen = c.forks.len
+  c.gen(n[0])
+  for i in 1..<n.len:
+    let it = n[i]
     if it.len == 1:
-      c.gen(it.sons[0])
+      c.gen(it[0])
+    elif i == n.len-1 and isExhaustive:
+      # treat the last branch as 'else' if this is an exhaustive case statement.
+      c.gen(it.lastSon)
     else:
       let elsePos = c.forkI(it.lastSon)
       c.gen(it.lastSon)
-      if i < sonsLen(n)-1:
-        endings.add(c.gotoI(it.lastSon))
+      endings.add(c.gotoI(it.lastSon))
       c.patch(elsePos)
-  for endPos in endings: c.patch(endPos)
+  for i in countdown(endings.high, 0):
+    let endPos = endings[i]
+    c.patch(endPos)
+    c.joinI(c.forks.pop(), n)
+  doAssert(c.forks.len == oldLen)
 
 proc genTry(c: var Con; n: PNode) =
+  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
-  let elsePos = c.forkI(n)
-  c.gen(n.sons[0])
-  c.patch(elsePos)
-  for i in 1 ..< n.len:
-    let it = n.sons[i]
+  inc c.inTryStmt
+  let oldFixups = c.tryStmtFixups.len
+
+  #let elsePos = c.forkI(n)
+  c.gen(n[0])
+  dec c.inTryStmt
+  for i in oldFixups..c.tryStmtFixups.high:
+    let f = c.tryStmtFixups[i]
+    c.patch(f)
+    # we also need to produce join instructions
+    # for the 'fork' that might precede the goto instruction
+    if f.int-1 >= 0 and c.code[f.int-1].kind == fork:
+      c.joinI(TPosition(f.int-1), n)
+
+  setLen(c.tryStmtFixups, oldFixups)
+
+  #c.patch(elsePos)
+  for i in 1..<n.len:
+    let it = n[i]
     if it.kind != nkFinally:
-      var blen = len(it)
       let endExcept = c.forkI(it)
       c.gen(it.lastSon)
-      if i < sonsLen(n)-1:
-        endings.add(c.gotoI(it))
+      endings.add(c.gotoI(it))
       c.patch(endExcept)
-  for endPos in endings: c.patch(endPos)
+  for i in countdown(endings.high, 0):
+    let endPos = endings[i]
+    c.patch(endPos)
+    c.joinI(c.forks.pop(), n)
+
+  # join the 'elsePos' forkI instruction:
+  #c.joinI(c.forks.pop(), n)
+
   let fin = lastSon(n)
   if fin.kind == nkFinally:
-    c.gen(fin.sons[0])
+    c.gen(fin[0])
+  doAssert(c.forks.len == oldLen)
+
+template genNoReturn(c: var Con; n: PNode) =
+  # leave the graph
+  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 proc genRaise(c: var Con; n: PNode) =
-  gen(c, n.sons[0])
-  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
+  genJoins(c, n)
+  gen(c, n[0])
+  if c.inTryStmt > 0:
+    c.tryStmtFixups.add c.gotoI(n)
+  else:
+    genNoReturn(c, n)
+
+proc genImplicitReturn(c: var Con) =
+  if c.owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter} and resultPos < c.owner.ast.len:
+    gen(c, c.owner.ast[resultPos])
 
 proc genReturn(c: var Con; n: PNode) =
-  if n.sons[0].kind != nkEmpty: gen(c, n.sons[0])
-  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
+  genJoins(c, n)
+  if n[0].kind != nkEmpty:
+    gen(c, n[0])
+  else:
+    genImplicitReturn(c)
+  genNoReturn(c, n)
 
 const
-  InterestingSyms = {skVar, skResult, skLet}
+  InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp}
+  PathKinds0 = {nkDotExpr, nkCheckedFieldExpr,
+                nkBracketExpr, nkDerefExpr, nkHiddenDeref,
+                nkAddr, nkHiddenAddr,
+                nkObjDownConv, nkObjUpConv}
+  PathKinds1 = {nkHiddenStdConv, nkHiddenSubConv}
 
-proc genUse(c: var Con; n: PNode) =
-  var n = n
-  while n.kind in {nkDotExpr, nkCheckedFieldExpr,
-                   nkBracketExpr, nkDerefExpr, nkHiddenDeref,
-                   nkAddr, nkHiddenAddr}:
-    n = n[0]
+proc skipConvDfa*(n: PNode): PNode =
+  result = n
+  while true:
+    case result.kind
+    of nkObjDownConv, nkObjUpConv:
+      result = result[0]
+    of PathKinds1:
+      result = result[1]
+    else: break
+
+proc genUse(c: var Con; orig: PNode) =
+  var n = orig
+  while true:
+    case n.kind
+    of PathKinds0 - {nkBracketExpr}:
+      n = n[0]
+    of nkBracketExpr:
+      gen(c, n[1])
+      n = n[0]
+    of PathKinds1:
+      n = n[1]
+    else: break
+    if n.kind in nkCallKinds:
+      gen(c, n)
   if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    if c.inCall > 0:
-      c.code.add Instr(n: n, kind: useWithinCall, sym: n.sym)
+    c.code.add Instr(n: orig, kind: use)
+
+proc aliases*(obj, field: PNode): bool =
+  var n = field
+  var obj = obj
+  while obj.kind in {nkHiddenSubConv, nkHiddenStdConv, nkObjDownConv, nkObjUpConv}:
+    obj = obj[0]
+  while true:
+    if sameTrees(obj, n): return true
+    case n.kind
+    of PathKinds0, PathKinds1:
+      n = n[0]
     else:
-      c.code.add Instr(n: n, kind: use, sym: n.sym)
+      break
+
+proc useInstrTargets*(ins: Instr; loc: PNode): bool =
+  assert ins.kind == use
+  result = sameTrees(ins.n, loc) or
+    ins.n.aliases(loc) or loc.aliases(ins.n)
+  # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
+  # use x.f;  question: does it affect the full 'x'? No.
+  # use x; question does it affect 'x.f'? Yes.
+
+proc defInstrTargets*(ins: Instr; loc: PNode): bool =
+  assert ins.kind == def
+  result = sameTrees(ins.n, loc) or ins.n.aliases(loc)
+  # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
+  # def x.f; question: does it affect the full 'x'? No.
+  # def x; question: does it affect the 'x.f'? Yes.
+
+proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
+  var n = orig
+  while true:
+    case n.kind
+    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
+       nkObjDownConv, nkObjUpConv, nkHiddenAddr, nkAddr:
+      n = n[0]
+    of nkBracketExpr:
+      # in a[i] the 'i' must be known
+      if n.len > 1 and n[1].kind in {nkCharLit..nkUInt64Lit}:
+        n = n[0]
+      else:
+        return false
+    of nkHiddenDeref, nkDerefExpr:
+      # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
+      # pointer indirection.
+      n = n[0]
+      return n.kind == nkSym and n.sym.owner == owner and (isSinkParam(n.sym) or
+          n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
+    else:
+      break
+  # XXX Allow closure deref operations here if we know
+  # the owner controlled the closure allocation?
+  result = n.kind == nkSym and n.sym.owner == owner and
+    owner.kind != skModule and
+    (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar)
+  # Note: There is a different move analyzer possible that checks for
+  # consume(param.key); param.key = newValue  for all paths. Then code like
+  #
+  #   let splited = split(move self.root, x)
+  #   self.root = merge(splited.lower, splited.greater)
+  #
+  # could be written without the ``move self.root``. However, this would be
+  # wrong! Then the write barrier for the ``self.root`` assignment would
+  # free the old data and all is lost! Lesson: Don't be too smart, trust the
+  # lower level C++ optimizer to specialize this code.
 
 proc genDef(c: var Con; n: PNode) =
+  var m = n
+  # XXX do something about this duplicated logic here.
+  while true:
+    case m.kind
+    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
+        nkObjDownConv, nkObjUpConv, nkHiddenAddr, nkAddr:
+      m = m[0]
+    of nkBracketExpr:
+      gen(c, m[1])
+      m = m[0]
+    of nkHiddenDeref, nkDerefExpr:
+      m = m[0]
+    else:
+      break
+
   if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: n, kind: def, sym: n.sym)
+    c.code.add Instr(n: n, kind: def)
+  elif isAnalysableFieldAccess(n, c.owner):
+    c.code.add Instr(n: n, kind: def)
+  else:
+    # bug #13314: An assignment to t5.w = -5 is a usage of 't5'
+    # we still need to gather the use information:
+    gen(c, n)
 
 proc genCall(c: var Con; n: PNode) =
   gen(c, n[0])
@@ -266,8 +692,23 @@ proc genCall(c: var Con; n: PNode) =
   inc c.inCall
   for i in 1..<n.len:
     gen(c, n[i])
-    if t != nil and i < t.len and t.sons[i].kind == tyVar:
-      genDef(c, n[i])
+    when false:
+      if t != nil and i < t.len and t[i].kind == tyVar:
+        # This is wrong! Pass by var is a 'might def', not a 'must def'
+        # like the other defs we emit. This is not good enough for a move
+        # optimizer.
+        genDef(c, n[i])
+  # every call can potentially raise:
+  if c.inTryStmt > 0 and canRaiseConservative(n[0]):
+    # we generate the instruction sequence:
+    # fork lab1
+    # goto exceptionHandler (except or finally)
+    # lab1:
+    # join F1
+    let endGoto = c.forkI(n)
+    c.tryStmtFixups.add c.gotoI(n)
+    c.patch(endGoto)
+    c.joinI(c.forks.pop(), n)
   dec c.inCall
 
 proc genMagic(c: var Con; n: PNode; m: TMagic) =
@@ -276,171 +717,71 @@ proc genMagic(c: var Con; n: PNode; m: TMagic) =
   of mNew, mNewFinalize:
     genDef(c, n[1])
     for i in 2..<n.len: gen(c, n[i])
-  of mExit:
-    genCall(c, n)
-    c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
   else:
     genCall(c, n)
 
 proc genVarSection(c: var Con; n: PNode) =
   for a in n:
-    if a.kind == nkCommentStmt: continue
-    if a.kind == nkVarTuple:
+    if a.kind == nkCommentStmt:
+      discard
+    elif a.kind == nkVarTuple:
       gen(c, a.lastSon)
-      for i in 0 .. a.len-3: genDef(c, a[i])
+      for i in 0..<a.len-2: genDef(c, a[i])
     else:
       gen(c, a.lastSon)
       if a.lastSon.kind != nkEmpty:
-        genDef(c, a.sons[0])
+        genDef(c, a[0])
 
 proc gen(c: var Con; n: PNode) =
   case n.kind
   of nkSym: genUse(c, n)
   of nkCallKinds:
-    if n.sons[0].kind == nkSym:
-      let s = n.sons[0].sym
+    if n[0].kind == nkSym:
+      let s = n[0].sym
       if s.magic != mNone:
         genMagic(c, n, s.magic)
       else:
         genCall(c, n)
+      if sfNoReturn in n[0].sym.flags:
+        genNoReturn(c, n)
     else:
       genCall(c, n)
   of nkCharLit..nkNilLit: discard
   of nkAsgn, nkFastAsgn:
     gen(c, n[1])
+    # watch out: 'obj[i].f2 = value' sets 'f2' but
+    # "uses" 'i'. But we are only talking about builtin array indexing so
+    # it doesn't matter and 'x = 34' is NOT a usage of 'x'.
     genDef(c, n[0])
-  of nkDotExpr, nkCheckedFieldExpr, nkBracketExpr,
-     nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr:
-    gen(c, n[0])
+  of PathKinds0 - {nkHiddenStdConv, nkHiddenSubConv, nkObjDownConv, nkObjUpConv}:
+    genUse(c, n)
   of nkIfStmt, nkIfExpr: genIf(c, n)
   of nkWhenStmt:
     # This is "when nimvm" node. Chose the first branch.
-    gen(c, n.sons[0].sons[1])
+    gen(c, n[0][1])
   of nkCaseStmt: genCase(c, n)
   of nkWhileStmt: genWhile(c, n)
   of nkBlockExpr, nkBlockStmt: genBlock(c, n)
   of nkReturnStmt: genReturn(c, n)
   of nkRaiseStmt: genRaise(c, n)
   of nkBreakStmt: genBreak(c, n)
-  of nkTryStmt: genTry(c, n)
+  of nkTryStmt, nkHiddenTryStmt: genTry(c, n)
   of nkStmtList, nkStmtListExpr, nkChckRangeF, nkChckRange64, nkChckRange,
-     nkBracket, nkCurly, nkPar, nkTupleConstr, nkClosure, nkObjConstr:
+     nkBracket, nkCurly, nkPar, nkTupleConstr, nkClosure, nkObjConstr, nkYieldStmt:
     for x in n: gen(c, x)
   of nkPragmaBlock: gen(c, n.lastSon)
-  of nkDiscardStmt: gen(c, n.sons[0])
-  of nkHiddenStdConv, nkHiddenSubConv, nkConv, nkExprColonExpr, nkExprEqExpr,
-     nkCast:
-    gen(c, n.sons[1])
-  of nkObjDownConv, nkStringToCString, nkCStringToString: gen(c, n.sons[0])
+  of nkDiscardStmt, nkObjDownConv, nkObjUpConv: gen(c, n[0])
+  of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast, nkHiddenSubConv, nkHiddenStdConv:
+    gen(c, n[1])
+  of nkStringToCString, nkCStringToString: gen(c, n[0])
   of nkVarSection, nkLetSection: genVarSection(c, n)
+  of nkDefer:
+    doAssert false, "dfa construction pass requires the elimination of 'defer'"
   else: discard
 
-proc dfa(code: seq[Instr]; conf: ConfigRef) =
-  var u = newSeq[IntSet](code.len) # usages
-  var d = newSeq[IntSet](code.len) # defs
-  var c = newSeq[IntSet](code.len) # consumed
-  var backrefs = initTable[int, int]()
-  for i in 0..<code.len:
-    u[i] = initIntSet()
-    d[i] = initIntSet()
-    c[i] = initIntSet()
-    case code[i].kind
-    of use, useWithinCall: u[i].incl(code[i].sym.id)
-    of def: d[i].incl(code[i].sym.id)
-    of fork, goto:
-      let d = i+code[i].dest
-      backrefs.add(d, i)
-
-  var w = @[0]
-  var maxIters = 50
-  var someChange = true
-  var takenGotos = initIntSet()
-  var consuming = -1
-  while w.len > 0 and maxIters > 0: # and someChange:
-    dec maxIters
-    var pc = w.pop() # w[^1]
-    var prevPc = -1
-    # this simulates a single linear control flow execution:
-    while pc < code.len:
-      if prevPc >= 0:
-        someChange = false
-        # merge step and test for changes (we compute the fixpoints here):
-        # 'u' needs to be the union of prevPc, pc
-        # 'd' needs to be the intersection of 'pc'
-        for id in u[prevPc]:
-          if not u[pc].containsOrIncl(id):
-            someChange = true
-        # in (a; b) if ``a`` sets ``v`` so does ``b``. The intersection
-        # is only interesting on merge points:
-        for id in d[prevPc]:
-          if not d[pc].containsOrIncl(id):
-            someChange = true
-        # if this is a merge point, we take the intersection of the 'd' sets:
-        if backrefs.hasKey(pc):
-          var intersect = initIntSet()
-          assign(intersect, d[pc])
-          var first = true
-          for prevPc in backrefs.allValues(pc):
-            for def in d[pc]:
-              if def notin d[prevPc]:
-                excl(intersect, def)
-                someChange = true
-                when defined(debugDfa):
-                  echo "Excluding ", pc, " prev ", prevPc
-          assign d[pc], intersect
-      if consuming >= 0:
-        if not c[pc].containsOrIncl(consuming):
-          someChange = true
-        consuming = -1
-
-      # our interpretation ![I!]:
-      prevPc = pc
-      case code[pc].kind
-      of goto:
-        # we must leave endless loops eventually:
-        if not takenGotos.containsOrIncl(pc) or someChange:
-          pc = pc + code[pc].dest
-        else:
-          inc pc
-      of fork:
-        # we follow the next instruction but push the dest onto our "work" stack:
-        #if someChange:
-        w.add pc + code[pc].dest
-        inc pc
-      of use, useWithinCall:
-        #if not d[prevPc].missingOrExcl():
-        # someChange = true
-        consuming = code[pc].sym.id
-        inc pc
-      of def:
-        if not d[pc].containsOrIncl(code[pc].sym.id):
-          someChange = true
-        inc pc
-
-  when defined(useDfa) and defined(debugDfa):
-    for i in 0..<code.len:
-      echo "PC ", i, ": defs: ", d[i], "; uses ", u[i], "; consumes ", c[i]
-
-  # now check the condition we're interested in:
-  for i in 0..<code.len:
-    case code[i].kind
-    of use, useWithinCall:
-      let s = code[i].sym
-      if s.id notin d[i]:
-        localError(conf, code[i].n.info, "usage of uninitialized variable: " & s.name.s)
-      if s.id in c[i]:
-        localError(conf, code[i].n.info, "usage of an already consumed variable: " & s.name.s)
-
-    else: discard
-
-proc dataflowAnalysis*(s: PSym; body: PNode; conf: ConfigRef) =
-  var c = Con(code: @[], blocks: @[])
-  gen(c, body)
-  when defined(useDfa) and defined(debugDfa): echoCfg(c.code)
-  dfa(c.code, conf)
-
 proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
   ## constructs a control flow graph for ``body``.
-  var c = Con(code: @[], blocks: @[])
+  var c = Con(code: @[], blocks: @[], owner: s)
   gen(c, body)
+  genImplicitReturn(c)
   shallowCopy(result, c.code)

compiler/docgen2.nim

@@ -11,25 +11,24 @@
 # semantic checking.
 
 import
-  os, options, ast, astalgo, msgs, ropes, idents, passes, docgen, lineinfos,
-  pathutils
+  options, ast, msgs, passes, docgen, lineinfos, pathutils
 
-from modulegraphs import ModuleGraph
+from modulegraphs import ModuleGraph, PPassContext
 
 type
-  TGen = object of TPassContext
+  TGen = object of PPassContext
     doc: PDoc
     module: PSym
+    config: ConfigRef
   PGen = ref TGen
 
-template shouldProcess(g): bool =
-  (g.module.owner.id == g.doc.conf.mainPackageId and optWholeProject in g.doc.conf.globalOptions) or
-      sfMainModule in g.module.flags
+proc shouldProcess(g: PGen): bool =
+  (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or
+      sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
 
 template closeImpl(body: untyped) {.dirty.} =
   var g = PGen(p)
   let useWarning = sfMainModule notin g.module.flags
-  #echo g.module.name.s, " ", g.module.owner.id, " ", gMainPackageId
   if shouldProcess(g):
     body
     try:
@@ -55,20 +54,27 @@ proc processNodeJson(c: PPassContext, n: PNode): PNode =
   result = n
   var g = PGen(c)
   if shouldProcess(g):
-    generateJson(g.doc, n)
+    generateJson(g.doc, n, false)
 
-proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
+template myOpenImpl(ext: untyped) {.dirty.} =
   var g: PGen
   new(g)
   g.module = module
+  g.config = graph.config
   var d = newDocumentor(AbsoluteFile toFullPath(graph.config, FileIndex module.position),
-      graph.cache, graph.config)
+      graph.cache, graph.config, ext, module)
   d.hasToc = true
   g.doc = d
   result = g
 
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
+  myOpenImpl(HtmlExt)
+
+proc myOpenJson(graph: ModuleGraph; module: PSym): PPassContext =
+  myOpenImpl(JsonExt)
+
 const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
-const docgen2JsonPass* = makePass(open = myOpen, process = processNodeJson,
+const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
                                   close = closeJson)
 
 proc finishDoc2Pass*(project: string) =

compiler/enumtostr.nim

@@ -0,0 +1,105 @@
+
+import ast, idents, lineinfos, modulegraphs, magicsys
+
+proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
+  result = newSym(skProc, getIdent(g.cache, "$"), t.owner, info)
+
+  let dest = newSym(skParam, getIdent(g.cache, "e"), result, info)
+  dest.typ = t
+
+  let res = newSym(skResult, getIdent(g.cache, "result"), result, info)
+  res.typ = getSysType(g, info, tyString)
+
+  result.typ = newType(tyProc, t.owner)
+  result.typ.n = newNodeI(nkFormalParams, info)
+  rawAddSon(result.typ, res.typ)
+  result.typ.n.add newNodeI(nkEffectList, info)
+
+  result.typ.addParam dest
+
+  var body = newNodeI(nkStmtList, info)
+  var caseStmt = newNodeI(nkCaseStmt, info)
+  caseStmt.add(newSymNode dest)
+
+  # copy the branches over, but replace the fields with the for loop body:
+  for i in 0..<t.n.len:
+    assert(t.n[i].kind == nkSym)
+    var field = t.n[i].sym
+    let val = if field.ast == nil: field.name.s else: field.ast.strVal
+    caseStmt.add newTree(nkOfBranch, newSymNode(field),
+      newTree(nkStmtList, newTree(nkFastAsgn, newSymNode(res), newStrNode(val, info))))
+    #newIntTypeNode(nkIntLit, field.position, t)
+
+  body.add(caseStmt)
+
+  var n = newNodeI(nkProcDef, info, bodyPos+2)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = body
+  n[resultPos] = newSymNode(res)
+  result.ast = n
+  incl result.flags, sfFromGeneric
+  incl result.flags, sfNeverRaises
+
+proc searchObjCaseImpl(obj: PNode; field: PSym): PNode =
+  case obj.kind
+  of nkSym:
+    result = nil
+  of nkElse, nkOfBranch:
+    result = searchObjCaseImpl(obj.lastSon, field)
+  else:
+    if obj.kind == nkRecCase and obj[0].kind == nkSym and obj[0].sym == field:
+      result = obj
+    else:
+      for x in obj:
+        result = searchObjCaseImpl(x, field)
+        if result != nil: break
+
+proc searchObjCase(t: PType; field: PSym): PNode =
+  result = searchObjCaseImpl(t.n, field)
+  if result == nil and t.len > 0:
+    result = searchObjCase(t[0].skipTypes({tyAlias, tyGenericInst, tyRef, tyPtr}), field)
+  doAssert result != nil
+
+proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph): PSym =
+  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), t.owner, info)
+
+  let dest = newSym(skParam, getIdent(g.cache, "e"), result, info)
+  dest.typ = field.typ
+
+  let res = newSym(skResult, getIdent(g.cache, "result"), result, info)
+  res.typ = getSysType(g, info, tyUInt8)
+
+  result.typ = newType(tyProc, t.owner)
+  result.typ.n = newNodeI(nkFormalParams, info)
+  rawAddSon(result.typ, res.typ)
+  result.typ.n.add newNodeI(nkEffectList, info)
+
+  result.typ.addParam dest
+
+  var body = newNodeI(nkStmtList, info)
+  var caseStmt = newNodeI(nkCaseStmt, info)
+  caseStmt.add(newSymNode dest)
+
+  let subObj = searchObjCase(t, field)
+  for i in 1..<subObj.len:
+    let ofBranch = subObj[i]
+    var newBranch = newNodeI(ofBranch.kind, ofBranch.info)
+    for j in 0..<ofBranch.len-1:
+      newBranch.add ofBranch[j]
+
+    newBranch.add newTree(nkStmtList, newTree(nkFastAsgn, newSymNode(res), newIntNode(nkInt8Lit, i)))
+    caseStmt.add newBranch
+
+  body.add(caseStmt)
+
+  var n = newNodeI(nkProcDef, info, bodyPos+2)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = body
+  n[resultPos] = newSymNode(res)
+  result.ast = n
+  incl result.flags, sfFromGeneric
+  incl result.flags, sfNeverRaises

compiler/evalffi.nim

@@ -9,76 +9,80 @@
 
 ## This file implements the FFI part of the evaluator for Nim code.
 
-import ast, astalgo, ropes, types, options, tables, dynlib, libffi, msgs, os
+import ast, types, options, tables, dynlib, msgs, lineinfos
+from os import getAppFilename
+import pkg/libffi
 
 when defined(windows):
   const libcDll = "msvcrt.dll"
-else:
+elif defined(linux):
   const libcDll = "libc.so(.6|.5|)"
+elif defined(bsd):
+  const libcDll = "/lib/libc.so.7"
+elif defined(osx):
+  const libcDll = "/usr/lib/libSystem.dylib"
+else:
+  {.error: "`libcDll` not implemented on this platform".}
 
 type
-  TDllCache = tables.TTable[string, TLibHandle]
+  TDllCache = tables.Table[string, LibHandle]
 var
-  gDllCache = initTable[string, TLibHandle]()
+  gDllCache = initTable[string, LibHandle]()
 
 when defined(windows):
-  var gExeHandle = loadLib(os.getAppFilename())
+  var gExeHandle = loadLib(getAppFilename())
 else:
   var gExeHandle = loadLib()
 
-proc getDll(cache: var TDllCache; dll: string; info: TLineInfo): pointer =
-  result = cache[dll]
+proc getDll(conf: ConfigRef, cache: var TDllCache; dll: string; info: TLineInfo): pointer =
+  if dll in cache:
+    return cache[dll]
+  var libs: seq[string]
+  libCandidates(dll, libs)
+  for c in libs:
+    result = loadLib(c)
+    if not result.isNil: break
   if result.isNil:
-    var libs: seq[string] = @[]
-    libCandidates(dll, libs)
-    for c in libs:
-      result = loadLib(c)
-      if not result.isNil: break
-    if result.isNil:
-      globalError(info, "cannot load: " & dll)
-    cache[dll] = result
+    globalError(conf, info, "cannot load: " & dll)
+  cache[dll] = result
 
 const
   nkPtrLit = nkIntLit # hopefully we can get rid of this hack soon
 
-var myerrno {.importc: "errno", header: "<errno.h>".}: cint ## error variable
-
-proc importcSymbol*(sym: PSym): PNode =
-  let name = ropeToStr(sym.loc.r)
-
+proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode =
+  let name = sym.cname # $sym.loc.r would point to internal name
   # the AST does not support untyped pointers directly, so we use an nkIntLit
   # that contains the address instead:
   result = newNodeIT(nkPtrLit, sym.info, sym.typ)
-  case name
-  of "stdin":  result.intVal = cast[ByteAddress](system.stdin)
-  of "stdout": result.intVal = cast[ByteAddress](system.stdout)
-  of "stderr": result.intVal = cast[ByteAddress](system.stderr)
-  of "vmErrnoWrapper": result.intVal = cast[ByteAddress](myerrno)
-  else:
+  when true:
+    var libPathMsg = ""
     let lib = sym.annex
     if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}:
-      globalError(sym.info, "dynlib needs to be a string lit for the REPL")
+      globalError(conf, sym.info, "dynlib needs to be a string lit")
     var theAddr: pointer
-    if lib.isNil and not gExehandle.isNil:
+    if (lib.isNil or lib.kind == libHeader) and not gExeHandle.isNil:
+      libPathMsg = "current exe: " & getAppFilename() & " nor libc: " & libcDll
       # first try this exe itself:
-      theAddr = gExehandle.symAddr(name)
+      theAddr = gExeHandle.symAddr(name)
       # then try libc:
       if theAddr.isNil:
-        let dllhandle = gDllCache.getDll(libcDll, sym.info)
+        let dllhandle = getDll(conf, gDllCache, libcDll, sym.info)
         theAddr = dllhandle.symAddr(name)
     elif not lib.isNil:
-      let dllhandle = gDllCache.getDll(if lib.kind == libHeader: libcDll
-                                       else: lib.path.strVal, sym.info)
+      let dll = if lib.kind == libHeader: libcDll else: lib.path.strVal
+      libPathMsg = dll
+      let dllhandle = getDll(conf, gDllCache, dll, sym.info)
       theAddr = dllhandle.symAddr(name)
-    if theAddr.isNil: globalError(sym.info, "cannot import: " & sym.name.s)
+    if theAddr.isNil: globalError(conf, sym.info,
+      "cannot import symbol: " & name & " from " & libPathMsg)
     result.intVal = cast[ByteAddress](theAddr)
 
-proc mapType(t: ast.PType): ptr libffi.TType =
+proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type =
   if t == nil: return addr libffi.type_void
 
   case t.kind
   of tyBool, tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64, tySet:
-    case t.getSize
+    case getSize(conf, t)
     of 1: result = addr libffi.type_uint8
     of 2: result = addr libffi.type_sint16
     of 4: result = addr libffi.type_sint32
@@ -86,91 +90,91 @@ proc mapType(t: ast.PType): ptr libffi.TType =
     else: result = nil
   of tyFloat, tyFloat64: result = addr libffi.type_double
   of tyFloat32: result = addr libffi.type_float
-  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr,
-     tyStmt, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
+  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyUntyped,
+     tyTyped, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
     result = addr libffi.type_pointer
   of tyDistinct, tyAlias, tySink:
-    result = mapType(t.sons[0])
+    result = mapType(conf, t[0])
   else:
     result = nil
   # too risky:
   #of tyFloat128: result = addr libffi.type_longdouble
 
-proc mapCallConv(cc: TCallingConvention, info: TLineInfo): TABI =
+proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI =
   case cc
   of ccDefault: result = DEFAULT_ABI
-  of ccStdCall: result = when defined(windows): STDCALL else: DEFAULT_ABI
+  of ccStdCall: result = when defined(windows) and defined(x86): STDCALL else: DEFAULT_ABI
   of ccCDecl: result = DEFAULT_ABI
   else:
-    globalError(info, "cannot map calling convention to FFI")
+    globalError(conf, info, "cannot map calling convention to FFI")
 
 template rd(T, p: untyped): untyped = (cast[ptr T](p))[]
 template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v
 template `+!`(x, y: untyped): untyped =
   cast[pointer](cast[ByteAddress](x) + y)
 
-proc packSize(v: PNode, typ: PType): int =
+proc packSize(conf: ConfigRef, v: PNode, typ: PType): int =
   ## computes the size of the blob
   case typ.kind
   of tyPtr, tyRef, tyVar, tyLent:
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     else:
-      result = sizeof(pointer) + packSize(v.sons[0], typ.lastSon)
+      result = sizeof(pointer) + packSize(conf, v[0], typ.lastSon)
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    result = packSize(v, typ.sons[0])
+    result = packSize(conf, v, typ[0])
   of tyArray:
     # consider: ptr array[0..1000_000, int] which is common for interfacing;
     # we use the real length here instead
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     elif v.len != 0:
-      result = v.len * packSize(v.sons[0], typ.sons[1])
+      result = v.len * packSize(conf, v[0], typ[1])
   else:
-    result = typ.getSize.int
+    result = getSize(conf, typ).int
 
-proc pack(v: PNode, typ: PType, res: pointer)
+proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer)
 
-proc getField(n: PNode; position: int): PSym =
+proc getField(conf: ConfigRef, n: PNode; position: int): PSym =
   case n.kind
   of nkRecList:
-    for i in countup(0, sonsLen(n) - 1):
-      result = getField(n.sons[i], position)
+    for i in 0..<n.len:
+      result = getField(conf, n[i], position)
       if result != nil: return
   of nkRecCase:
-    result = getField(n.sons[0], position)
+    result = getField(conf, n[0], position)
     if result != nil: return
-    for i in countup(1, sonsLen(n) - 1):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = getField(lastSon(n.sons[i]), position)
+        result = getField(conf, lastSon(n[i]), position)
         if result != nil: return
-      else: internalError(n.info, "getField(record case branch)")
+      else: internalError(conf, n.info, "getField(record case branch)")
   of nkSym:
     if n.sym.position == position: result = n.sym
   else: discard
 
-proc packObject(x: PNode, typ: PType, res: pointer) =
-  internalAssert x.kind in {nkObjConstr, nkPar, nkTupleConstr}
+proc packObject(conf: ConfigRef, x: PNode, typ: PType, res: pointer) =
+  internalAssert conf, x.kind in {nkObjConstr, nkPar, nkTupleConstr}
   # compute the field's offsets:
-  discard typ.getSize
-  for i in countup(ord(x.kind == nkObjConstr), sonsLen(x) - 1):
-    var it = x.sons[i]
+  discard getSize(conf, typ)
+  for i in ord(x.kind == nkObjConstr)..<x.len:
+    var it = x[i]
     if it.kind == nkExprColonExpr:
-      internalAssert it.sons[0].kind == nkSym
-      let field = it.sons[0].sym
-      pack(it.sons[1], field.typ, res +! field.offset)
+      internalAssert conf, it[0].kind == nkSym
+      let field = it[0].sym
+      pack(conf, it[1], field.typ, res +! field.offset)
     elif typ.n != nil:
-      let field = getField(typ.n, i)
-      pack(it, field.typ, res +! field.offset)
+      let field = getField(conf, typ.n, i)
+      pack(conf, it, field.typ, res +! field.offset)
     else:
       # XXX: todo
-      globalError(x.info, "cannot pack unnamed tuple")
+      globalError(conf, x.info, "cannot pack unnamed tuple")
 
 const maxPackDepth = 20
 var packRecCheck = 0
 
-proc pack(v: PNode, typ: PType, res: pointer) =
+proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
   template awr(T, v: untyped): untyped =
     wr(T, res, v)
 
@@ -188,13 +192,13 @@ proc pack(v: PNode, typ: PType, res: pointer) =
   of tyUInt32: awr(uint32, v.intVal.uint32)
   of tyUInt64: awr(uint64, v.intVal.uint64)
   of tyEnum, tySet:
-    case v.typ.getSize
+    case getSize(conf, v.typ)
     of 1: awr(uint8, v.intVal.uint8)
     of 2: awr(uint16, v.intVal.uint16)
     of 4: awr(int32, v.intVal.int32)
     of 8: awr(int64, v.intVal.int64)
     else:
-      globalError(v.info, "cannot map value to FFI (tyEnum, tySet)")
+      globalError(conf, v.info, "cannot map value to FFI (tyEnum, tySet)")
   of tyFloat: awr(float, v.floatVal)
   of tyFloat32: awr(float32, v.floatVal)
   of tyFloat64: awr(float64, v.floatVal)
@@ -208,7 +212,7 @@ proc pack(v: PNode, typ: PType, res: pointer) =
     elif v.kind in {nkStrLit..nkTripleStrLit}:
       awr(cstring, cstring(v.strVal))
     else:
-      globalError(v.info, "cannot map pointer/proc value to FFI")
+      globalError(conf, v.info, "cannot map pointer/proc value to FFI")
   of tyPtr, tyRef, tyVar, tyLent:
     if v.kind == nkNilLit:
       # nothing to do since the memory is 0 initialized anyway
@@ -218,44 +222,44 @@ proc pack(v: PNode, typ: PType, res: pointer) =
     else:
       if packRecCheck > maxPackDepth:
         packRecCheck = 0
-        globalError(v.info, "cannot map value to FFI " & typeToString(v.typ))
+        globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ))
       inc packRecCheck
-      pack(v.sons[0], typ.lastSon, res +! sizeof(pointer))
+      pack(conf, v[0], typ.lastSon, res +! sizeof(pointer))
       dec packRecCheck
       awr(pointer, res +! sizeof(pointer))
   of tyArray:
-    let baseSize = typ.sons[1].getSize
-    for i in 0 ..< v.len:
-      pack(v.sons[i], typ.sons[1], res +! i * baseSize)
+    let baseSize = getSize(conf, typ[1])
+    for i in 0..<v.len:
+      pack(conf, v[i], typ[1], res +! i * baseSize)
   of tyObject, tyTuple:
-    packObject(v, typ, res)
+    packObject(conf, v, typ, res)
   of tyNil:
     discard
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    pack(v, typ.sons[0], res)
+    pack(conf, v, typ[0], res)
   else:
-    globalError(v.info, "cannot map value to FFI " & typeToString(v.typ))
+    globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ))
 
-proc unpack(x: pointer, typ: PType, n: PNode): PNode
+proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode
 
-proc unpackObjectAdd(x: pointer, n, result: PNode) =
+proc unpackObjectAdd(conf: ConfigRef, x: pointer, n, result: PNode) =
   case n.kind
   of nkRecList:
-    for i in countup(0, sonsLen(n) - 1):
-      unpackObjectAdd(x, n.sons[i], result)
+    for i in 0..<n.len:
+      unpackObjectAdd(conf, x, n[i], result)
   of nkRecCase:
-    globalError(result.info, "case objects cannot be unpacked")
+    globalError(conf, result.info, "case objects cannot be unpacked")
   of nkSym:
     var pair = newNodeI(nkExprColonExpr, result.info, 2)
-    pair.sons[0] = n
-    pair.sons[1] = unpack(x +! n.sym.offset, n.sym.typ, nil)
+    pair[0] = n
+    pair[1] = unpack(conf, x +! n.sym.offset, n.sym.typ, nil)
     #echo "offset: ", n.sym.name.s, " ", n.sym.offset
     result.add pair
   else: discard
 
-proc unpackObject(x: pointer, typ: PType, n: PNode): PNode =
+proc unpackObject(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   # compute the field's offsets:
-  discard typ.getSize
+  discard getSize(conf, typ)
 
   # iterate over any actual field of 'n' ... if n is nil we need to create
   # the nkPar node:
@@ -263,36 +267,36 @@ proc unpackObject(x: pointer, typ: PType, n: PNode): PNode =
     result = newNode(nkTupleConstr)
     result.typ = typ
     if typ.n.isNil:
-      internalError("cannot unpack unnamed tuple")
-    unpackObjectAdd(x, typ.n, result)
+      internalError(conf, "cannot unpack unnamed tuple")
+    unpackObjectAdd(conf, x, typ.n, result)
   else:
     result = n
     if result.kind notin {nkObjConstr, nkPar, nkTupleConstr}:
-      globalError(n.info, "cannot map value from FFI")
+      globalError(conf, n.info, "cannot map value from FFI")
     if typ.n.isNil:
-      globalError(n.info, "cannot unpack unnamed tuple")
-    for i in countup(ord(n.kind == nkObjConstr), sonsLen(n) - 1):
-      var it = n.sons[i]
+      globalError(conf, n.info, "cannot unpack unnamed tuple")
+    for i in ord(n.kind == nkObjConstr)..<n.len:
+      var it = n[i]
       if it.kind == nkExprColonExpr:
-        internalAssert it.sons[0].kind == nkSym
-        let field = it.sons[0].sym
-        it.sons[1] = unpack(x +! field.offset, field.typ, it.sons[1])
+        internalAssert conf, it[0].kind == nkSym
+        let field = it[0].sym
+        it[1] = unpack(conf, x +! field.offset, field.typ, it[1])
       else:
-        let field = getField(typ.n, i)
-        n.sons[i] = unpack(x +! field.offset, field.typ, it)
+        let field = getField(conf, typ.n, i)
+        n[i] = unpack(conf, x +! field.offset, field.typ, it)
 
-proc unpackArray(x: pointer, typ: PType, n: PNode): PNode =
+proc unpackArray(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   if n.isNil:
     result = newNode(nkBracket)
     result.typ = typ
-    newSeq(result.sons, lengthOrd(typ).int)
+    newSeq(result.sons, lengthOrd(conf, typ).toInt)
   else:
     result = n
     if result.kind != nkBracket:
-      globalError(n.info, "cannot map value from FFI")
-  let baseSize = typ.sons[1].getSize
-  for i in 0 ..< result.len:
-    result.sons[i] = unpack(x +! i * baseSize, typ.sons[1], result.sons[i])
+      globalError(conf, n.info, "cannot map value from FFI")
+  let baseSize = getSize(conf, typ[1])
+  for i in 0..<result.len:
+    result[i] = unpack(conf, x +! i * baseSize, typ[1], result[i])
 
 proc canonNodeKind(k: TNodeKind): TNodeKind =
   case k
@@ -301,7 +305,7 @@ proc canonNodeKind(k: TNodeKind): TNodeKind =
   of nkStrLit..nkTripleStrLit: result = nkStrLit
   else: result = k
 
-proc unpack(x: pointer, typ: PType, n: PNode): PNode =
+proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   template aw(k, v, field: untyped): untyped =
     if n.isNil:
       result = newNode(k)
@@ -313,7 +317,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
         #echo "expected ", k, " but got ", result.kind
         #debug result
         return newNodeI(nkExceptBranch, n.info)
-        #globalError(n.info, "cannot map value from FFI")
+        #globalError(conf, n.info, "cannot map value from FFI")
     result.field = v
 
   template setNil() =
@@ -323,7 +327,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
     else:
       reset n[]
       result = n
-      result.kind = nkNilLit
+      result[] = TNode(kind: nkNilLit)
       result.typ = typ
 
   template awi(kind, v: untyped): untyped = aw(kind, v, intVal)
@@ -344,13 +348,13 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
   of tyUInt32: awi(nkUInt32Lit, rd(uint32, x).BiggestInt)
   of tyUInt64: awi(nkUInt64Lit, rd(uint64, x).BiggestInt)
   of tyEnum:
-    case typ.getSize
+    case getSize(conf, typ)
     of 1: awi(nkIntLit, rd(uint8, x).BiggestInt)
     of 2: awi(nkIntLit, rd(uint16, x).BiggestInt)
     of 4: awi(nkIntLit, rd(int32, x).BiggestInt)
     of 8: awi(nkIntLit, rd(int64, x).BiggestInt)
     else:
-      globalError(n.info, "cannot map value from FFI (tyEnum, tySet)")
+      globalError(conf, n.info, "cannot map value from FFI (tyEnum, tySet)")
   of tyFloat: awf(nkFloatLit, rd(float, x))
   of tyFloat32: awf(nkFloat32Lit, rd(float32, x))
   of tyFloat64: awf(nkFloat64Lit, rd(float64, x))
@@ -371,15 +375,15 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
     elif n == nil or n.kind == nkPtrLit:
       awi(nkPtrLit, cast[ByteAddress](p))
     elif n != nil and n.len == 1:
-      internalAssert n.kind == nkRefTy
-      n.sons[0] = unpack(p, typ.lastSon, n.sons[0])
+      internalAssert(conf, n.kind == nkRefTy)
+      n[0] = unpack(conf, p, typ.lastSon, n[0])
       result = n
     else:
-      globalError(n.info, "cannot map value from FFI " & typeToString(typ))
+      globalError(conf, n.info, "cannot map value from FFI " & typeToString(typ))
   of tyObject, tyTuple:
-    result = unpackObject(x, typ, n)
+    result = unpackObject(conf, x, typ, n)
   of tyArray:
-    result = unpackArray(x, typ, n)
+    result = unpackArray(conf, x, typ, n)
   of tyCString, tyString:
     let p = rd(cstring, x)
     if p.isNil:
@@ -389,12 +393,12 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
   of tyNil:
     setNil()
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    result = unpack(x, typ.lastSon, n)
+    result = unpack(conf, x, typ.lastSon, n)
   else:
     # XXX what to do with 'array' here?
-    globalError(n.info, "cannot map value from FFI " & typeToString(typ))
+    globalError(conf, n.info, "cannot map value from FFI " & typeToString(typ))
 
-proc fficast*(x: PNode, destTyp: PType): PNode =
+proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode =
   if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyLent, tyPointer,
                                            tyProc, tyCString, tyString,
                                            tySequence}:
@@ -404,93 +408,93 @@ proc fficast*(x: PNode, destTyp: PType): PNode =
     result = newNodeIT(x.kind, x.info, destTyp)
   else:
     # we play safe here and allocate the max possible size:
-    let size = max(packSize(x, x.typ), packSize(x, destTyp))
+    let size = max(packSize(conf, x, x.typ), packSize(conf, x, destTyp))
     var a = alloc0(size)
-    pack(x, x.typ, a)
+    pack(conf, x, x.typ, a)
     # cast through a pointer needs a new inner object:
     let y = if x.kind == nkRefTy: newNodeI(nkRefTy, x.info, 1)
             else: x.copyTree
     y.typ = x.typ
-    result = unpack(a, destTyp, y)
+    result = unpack(conf, a, destTyp, y)
     dealloc a
 
-proc callForeignFunction*(call: PNode): PNode =
-  internalAssert call.sons[0].kind == nkPtrLit
+proc callForeignFunction*(conf: ConfigRef, call: PNode): PNode =
+  internalAssert conf, call[0].kind == nkPtrLit
 
   var cif: TCif
-  var sig: TParamList
+  var sig: ParamList
   # use the arguments' types for varargs support:
-  for i in 1..call.len-1:
-    sig[i-1] = mapType(call.sons[i].typ)
+  for i in 1..<call.len:
+    sig[i-1] = mapType(conf, call[i].typ)
     if sig[i-1].isNil:
-      globalError(call.info, "cannot map FFI type")
+      globalError(conf, call.info, "cannot map FFI type")
 
-  let typ = call.sons[0].typ
-  if prep_cif(cif, mapCallConv(typ.callConv, call.info), cuint(call.len-1),
-              mapType(typ.sons[0]), sig) != OK:
-    globalError(call.info, "error in FFI call")
+  let typ = call[0].typ
+  if prep_cif(cif, mapCallConv(conf, typ.callConv, call.info), cuint(call.len-1),
+              mapType(conf, typ[0]), sig) != OK:
+    globalError(conf, call.info, "error in FFI call")
 
-  var args: TArgList
-  let fn = cast[pointer](call.sons[0].intVal)
-  for i in 1 .. call.len-1:
-    var t = call.sons[i].typ
-    args[i-1] = alloc0(packSize(call.sons[i], t))
-    pack(call.sons[i], t, args[i-1])
-  let retVal = if isEmptyType(typ.sons[0]): pointer(nil)
-               else: alloc(typ.sons[0].getSize.int)
+  var args: ArgList
+  let fn = cast[pointer](call[0].intVal)
+  for i in 1..<call.len:
+    var t = call[i].typ
+    args[i-1] = alloc0(packSize(conf, call[i], t))
+    pack(conf, call[i], t, args[i-1])
+  let retVal = if isEmptyType(typ[0]): pointer(nil)
+               else: alloc(getSize(conf, typ[0]).int)
 
   libffi.call(cif, fn, retVal, args)
 
   if retVal.isNil:
     result = newNode(nkEmpty)
   else:
-    result = unpack(retVal, typ.sons[0], nil)
+    result = unpack(conf, retVal, typ[0], nil)
     result.info = call.info
 
   if retVal != nil: dealloc retVal
-  for i in 1 .. call.len-1:
-    call.sons[i] = unpack(args[i-1], typ.sons[i], call[i])
+  for i in 1..<call.len:
+    call[i] = unpack(conf, args[i-1], typ[i], call[i])
     dealloc args[i-1]
 
-proc callForeignFunction*(fn: PNode, fntyp: PType,
+proc callForeignFunction*(conf: ConfigRef, fn: PNode, fntyp: PType,
                           args: var TNodeSeq, start, len: int,
                           info: TLineInfo): PNode =
-  internalAssert fn.kind == nkPtrLit
+  internalAssert conf, fn.kind == nkPtrLit
 
   var cif: TCif
-  var sig: TParamList
+  var sig: ParamList
   for i in 0..len-1:
     var aTyp = args[i+start].typ
     if aTyp.isNil:
-      internalAssert i+1 < fntyp.len
-      aTyp = fntyp.sons[i+1]
+      internalAssert conf, i+1 < fntyp.len
+      aTyp = fntyp[i+1]
       args[i+start].typ = aTyp
-    sig[i] = mapType(aTyp)
-    if sig[i].isNil: globalError(info, "cannot map FFI type")
+    sig[i] = mapType(conf, aTyp)
+    if sig[i].isNil: globalError(conf, info, "cannot map FFI type")
 
-  if prep_cif(cif, mapCallConv(fntyp.callConv, info), cuint(len),
-              mapType(fntyp.sons[0]), sig) != OK:
-    globalError(info, "error in FFI call")
+  if prep_cif(cif, mapCallConv(conf, fntyp.callConv, info), cuint(len),
+              mapType(conf, fntyp[0]), sig) != OK:
+    globalError(conf, info, "error in FFI call")
 
-  var cargs: TArgList
+  var cargs: ArgList
   let fn = cast[pointer](fn.intVal)
-  for i in 0 .. len-1:
+  for i in 0..len-1:
     let t = args[i+start].typ
-    cargs[i] = alloc0(packSize(args[i+start], t))
-    pack(args[i+start], t, cargs[i])
-  let retVal = if isEmptyType(fntyp.sons[0]): pointer(nil)
-               else: alloc(fntyp.sons[0].getSize.int)
+    cargs[i] = alloc0(packSize(conf, args[i+start], t))
+    pack(conf, args[i+start], t, cargs[i])
+  let retVal = if isEmptyType(fntyp[0]): pointer(nil)
+               else: alloc(getSize(conf, fntyp[0]).int)
 
   libffi.call(cif, fn, retVal, cargs)
 
   if retVal.isNil:
     result = newNode(nkEmpty)
   else:
-    result = unpack(retVal, fntyp.sons[0], nil)
+    result = unpack(conf, retVal, fntyp[0], nil)
     result.info = info
 
   if retVal != nil: dealloc retVal
-  for i in 0 .. len-1:
+  for i in 0..len-1:
     let t = args[i+start].typ
-    args[i+start] = unpack(cargs[i], t, args[i+start])
+    args[i+start] = unpack(conf, cargs[i], t, args[i+start])
     dealloc cargs[i]

compiler/evaltempl.nim

@@ -10,16 +10,17 @@
 ## Template evaluation engine. Now hygienic.
 
 import
-  strutils, options, ast, astalgo, msgs, os, idents, wordrecg, renderer,
-  lineinfos
+  strutils, options, ast, astalgo, msgs, renderer, lineinfos, idents
 
 type
   TemplCtx = object
     owner, genSymOwner: PSym
     instLines: bool   # use the instantiation lines numbers
+    isDeclarative: bool
     mapping: TIdTable # every gensym'ed symbol needs to be mapped to some
                       # new symbol
     config: ConfigRef
+    ic: IdentCache
 
 proc copyNode(ctx: TemplCtx, a, b: PNode): PNode =
   result = copyNode(a)
@@ -36,29 +37,55 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
   case templ.kind
   of nkSym:
     var s = templ.sym
-    if s.owner.id == c.owner.id:
-      if s.kind == skParam and sfGenSym notin s.flags:
-        handleParam actual.sons[s.position]
-      elif s.kind == skGenericParam or
-           s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam:
-        handleParam actual.sons[s.owner.typ.len + s.position - 1]
+    if (s.owner == nil and s.kind == skParam) or s.owner == c.owner:
+      if s.kind == skParam and {sfGenSym, sfTemplateParam} * s.flags == {sfTemplateParam}:
+        handleParam actual[s.position]
+      elif (s.owner != nil) and (s.kind == skGenericParam or
+           s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam):
+        handleParam actual[s.owner.typ.len + s.position - 1]
       else:
         internalAssert c.config, sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
         if x == nil:
-          x = copySym(s, false)
-          x.owner = c.genSymOwner
+          x = copySym(s)
+          # sem'check needs to set the owner properly later, see bug #9476
+          x.owner = nil # c.genSymOwner
+          #if x.kind == skParam and x.owner.kind == skModule:
+          #  internalAssert c.config, false
           idTablePut(c.mapping, s, x)
-        result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
+        if sfGenSym in s.flags and optNimV019 notin c.config.globalOptions:
+          result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $x.id),
+            if c.instLines: actual.info else: templ.info)
+        else:
+          result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
     else:
       result.add copyNode(c, templ, actual)
   of nkNone..nkIdent, nkType..nkNilLit: # atom
     result.add copyNode(c, templ, actual)
+  of nkCommentStmt:
+    # for the documentation generator we don't keep documentation comments
+    # in the AST that would confuse it (bug #9432), but only if we are not in a
+    # "declarative" context (bug #9235).
+    if c.isDeclarative:
+      var res = copyNode(c, templ, actual)
+      for i in 0..<templ.len:
+        evalTemplateAux(templ[i], actual, c, res)
+      result.add res
+    else:
+      result.add newNodeI(nkEmpty, templ.info)
   else:
+    var isDeclarative = false
+    if templ.kind in {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef,
+                      nkMacroDef, nkTemplateDef, nkConverterDef, nkTypeSection,
+                      nkVarSection, nkLetSection, nkConstSection} and
+        not c.isDeclarative:
+      c.isDeclarative = true
+      isDeclarative = true
     var res = copyNode(c, templ, actual)
-    for i in countup(0, sonsLen(templ) - 1):
-      evalTemplateAux(templ.sons[i], actual, c, res)
+    for i in 0..<templ.len:
+      evalTemplateAux(templ[i], actual, c, res)
     result.add res
+    if isDeclarative: c.isDeclarative = false
 
 const
   errWrongNumberOfArguments = "wrong number of arguments"
@@ -81,7 +108,7 @@ proc evalTemplateArgs(n: PNode, s: PSym; conf: ConfigRef; fromHlo: bool): PNode
     # their bodies. We could try to fix this, but it may be
     # wiser to just deprecate immediate templates and macros
     # now that we have working untyped parameters.
-    genericParams = if sfImmediate in s.flags or fromHlo: 0
+    genericParams = if fromHlo: 0
                     else: s.ast[genericParamsPos].len
     expectedRegularParams = s.typ.len-1
     givenRegularParams = totalParams - genericParams
@@ -95,22 +122,22 @@ proc evalTemplateArgs(n: PNode, s: PSym; conf: ConfigRef; fromHlo: bool): PNode
                 n.renderTree)
 
   result = newNodeI(nkArgList, n.info)
-  for i in 1 .. givenRegularParams:
-    result.addSon n[i]
+  for i in 1..givenRegularParams:
+    result.add n[i]
 
   # handle parameters with default values, which were
   # not supplied by the user
-  for i in givenRegularParams+1 .. expectedRegularParams:
-    let default = s.typ.n.sons[i].sym.ast
+  for i in givenRegularParams+1..expectedRegularParams:
+    let default = s.typ.n[i].sym.ast
     if default.isNil or default.kind == nkEmpty:
       localError(conf, n.info, errWrongNumberOfArguments)
-      addSon(result, newNodeI(nkEmpty, n.info))
+      result.add newNodeI(nkEmpty, n.info)
     else:
-      addSon(result, default.copyTree)
+      result.add default.copyTree
 
-  # add any generic paramaters
-  for i in 1 .. genericParams:
-    result.addSon n.sons[givenRegularParams + i]
+  # add any generic parameters
+  for i in 1..genericParams:
+    result.add n[givenRegularParams + i]
 
 # to prevent endless recursion in template instantiation
 const evalTemplateLimit* = 1000
@@ -128,7 +155,7 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
       if x.kind in nkCallKinds:
         for i in 1..<x.len:
           if x[i].kind in nkCallKinds:
-            x.sons[i].info = info
+            x[i].info = info
   else:
     result = newNodeI(nkStmtListExpr, info)
     var d = newNodeI(nkComesFrom, info)
@@ -138,7 +165,9 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
     result.typ = res.typ
 
 proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
-                   conf: ConfigRef; fromHlo=false): PNode =
+                   conf: ConfigRef;
+                   ic: IdentCache;
+                   fromHlo=false): PNode =
   inc(conf.evalTemplateCounter)
   if conf.evalTemplateCounter > evalTemplateLimit:
     globalError(conf, n.info, errTemplateInstantiationTooNested)
@@ -150,24 +179,27 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   ctx.owner = tmpl
   ctx.genSymOwner = genSymOwner
   ctx.config = conf
+  ctx.ic = ic
   initIdTable(ctx.mapping)
 
   let body = tmpl.getBody
+  #echo "instantion of ", renderTree(body, {renderIds})
   if isAtom(body):
     result = newNodeI(nkPar, body.info)
     evalTemplateAux(body, args, ctx, result)
-    if result.len == 1: result = result.sons[0]
+    if result.len == 1: result = result[0]
     else:
       localError(conf, result.info, "illformed AST: " &
                   renderTree(result, {renderNoComments}))
   else:
     result = copyNode(body)
-    #ctx.instLines = body.kind notin {nkStmtList, nkStmtListExpr,
-    #                                 nkBlockStmt, nkBlockExpr}
-    #if ctx.instLines: result.info = n.info
-    for i in countup(0, safeLen(body) - 1):
-      evalTemplateAux(body.sons[i], args, ctx, result)
+    ctx.instLines = sfCallsite in tmpl.flags
+    if ctx.instLines:
+      result.info = n.info
+    for i in 0..<body.safeLen:
+      evalTemplateAux(body[i], args, ctx, result)
   result.flags.incl nfFromTemplate
   result = wrapInComesFrom(n.info, tmpl, result)
+  #if ctx.debugActive:
+  #  echo "instantion of ", renderTree(result, {renderIds})
   dec(conf.evalTemplateCounter)
-

compiler/filter_tmpl.nim

@@ -10,8 +10,8 @@
 # This module implements Nim's standard template filter.
 
 import
-  llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
-  renderer, filters, lineinfos, pathutils
+  llstream, strutils, ast, msgs, options,
+  filters, lineinfos, pathutils
 
 type
   TParseState = enum
@@ -22,7 +22,7 @@ type
     info: TLineInfo
     indent, emitPar: int
     x: string                # the current input line
-    outp: PLLStream          # the ouput will be parsed by pnimsyn
+    outp: PLLStream          # the output will be parsed by pnimsyn
     subsChar, nimDirective: char
     emit, conc, toStr: string
     curly, bracket, par: int
@@ -75,7 +75,7 @@ proc parseLine(p: var TTmplParser) =
     let d = j
     var keyw = ""
     while j < len and p.x[j] in PatternChars:
-      add(keyw, p.x[j])
+      keyw.add(p.x[j])
       inc(j)
 
     scanPar(p, j)

compiler/filters.nim

@@ -10,7 +10,7 @@
 # This module implements Nim's simple filters and helpers for filters.
 
 import
-  llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
+  llstream, idents, strutils, ast, msgs, options,
   renderer, pathutils
 
 proc invalidPragma(conf: ConfigRef; n: PNode) =
@@ -20,13 +20,13 @@ proc invalidPragma(conf: ConfigRef; n: PNode) =
 proc getArg(conf: ConfigRef; n: PNode, name: string, pos: int): PNode =
   result = nil
   if n.kind in {nkEmpty..nkNilLit}: return
-  for i in countup(1, sonsLen(n) - 1):
-    if n.sons[i].kind == nkExprEqExpr:
-      if n.sons[i].sons[0].kind != nkIdent: invalidPragma(conf, n)
-      if cmpIgnoreStyle(n.sons[i].sons[0].ident.s, name) == 0:
-        return n.sons[i].sons[1]
+  for i in 1..<n.len:
+    if n[i].kind == nkExprEqExpr:
+      if n[i][0].kind != nkIdent: invalidPragma(conf, n)
+      if cmpIgnoreStyle(n[i][0].ident.s, name) == 0:
+        return n[i][1]
     elif i == pos:
-      return n.sons[i]
+      return n[i]
 
 proc charArg*(conf: ConfigRef; n: PNode, name: string, pos: int, default: char): char =
   var x = getArg(conf, n, name, pos)
@@ -55,7 +55,7 @@ proc filterStrip*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, cal
   var line = newStringOfCap(80)
   while llStreamReadLine(stdin, line):
     var stripped = strip(line, leading, trailing)
-    if len(pattern) == 0 or startsWith(stripped, pattern):
+    if pattern.len == 0 or startsWith(stripped, pattern):
       llStreamWriteln(result, stripped)
     else:
       llStreamWriteln(result, line)
@@ -63,7 +63,7 @@ proc filterStrip*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, cal
 
 proc filterReplace*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, call: PNode): PLLStream =
   var sub = strArg(conf, call, "sub", 1, "")
-  if len(sub) == 0: invalidPragma(conf, call)
+  if sub.len == 0: invalidPragma(conf, call)
   var by = strArg(conf, call, "by", 2, "")
   result = llStreamOpen("")
   var line = newStringOfCap(80)

compiler/forloops.nim

@@ -13,8 +13,8 @@ import ast, astalgo
 
 const
   someCmp = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
-    mEqUntracedRef, mLeI, mLeF64, mLeU, mLeU64, mLeEnum,
-    mLeCh, mLeB, mLePtr, mLtI, mLtF64, mLtU, mLtU64, mLtEnum,
+    mLeI, mLeF64, mLeU, mLeEnum,
+    mLeCh, mLeB, mLePtr, mLtI, mLtF64, mLtU, mLtEnum,
     mLtCh, mLtB, mLtPtr}
 
 proc isCounter(s: PSym): bool {.inline.} =
@@ -45,13 +45,13 @@ proc counterInTree(n, loop: PNode; counter: PSym): bool =
     for it in n:
       if counterInTree(it.lastSon): return true
   else:
-    for i in 0 ..< safeLen(n):
+    for i in 0..<n.safeLen:
       if counterInTree(n[i], loop, counter): return true
 
 proc copyExcept(n: PNode, x, dest: PNode) =
   if x == n: return
   if n.kind in {nkStmtList, nkStmtListExpr}:
-    for i in 0 ..< n.len: copyExcept(n[i], x, dest)
+    for i in 0..<n.len: copyExcept(n[i], x, dest)
   else:
     dest.add n
 

compiler/gorgeimpl.nim

@@ -9,7 +9,7 @@
 
 ## Module that implements ``gorge`` for the compiler.
 
-import msgs, std / sha1, os, osproc, streams, strutils, options,
+import msgs, std / sha1, os, osproc, streams, options,
   lineinfos, pathutils
 
 proc readOutput(p: Process): (string, int) =
@@ -24,22 +24,23 @@ proc readOutput(p: Process): (string, int) =
 
 proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (string, int) =
   let workingDir = parentDir(toFullPath(conf, info))
-  if cache.len > 0:# and optForceFullMake notin gGlobalOptions:
+  if cache.len > 0:
     let h = secureHash(cmd & "\t" & input & "\t" & cache)
     let filename = toGeneratedFile(conf, AbsoluteFile("gorge_" & $h), "txt").string
     var f: File
-    if open(f, filename):
+    if optForceFullMake notin conf.globalOptions and open(f, filename):
       result = (f.readAll, 0)
       f.close
       return
     var readSuccessful = false
     try:
       var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStderrToStdout})
+                           options={poEvalCommand, poStdErrToStdOut})
       if input.len != 0:
         p.inputStream.write(input)
         p.inputStream.close()
       result = p.readOutput
+      p.close()
       readSuccessful = true
       # only cache successful runs:
       if result[1] == 0:
@@ -49,10 +50,11 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
   else:
     try:
       var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStderrToStdout})
+                           options={poEvalCommand, poStdErrToStdOut})
       if input.len != 0:
         p.inputStream.write(input)
         p.inputStream.close()
       result = p.readOutput
+      p.close()
     except IOError, OSError:
       result = ("", -1)

compiler/guards.nim

@@ -10,22 +10,21 @@
 ## This module implements the 'implies' relation for guards.
 
 import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents,
-  saturate, modulegraphs, options, lineinfos
+  saturate, modulegraphs, options, lineinfos, int128
 
 const
   someEq = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
-    mEqUntracedRef, mEqStr, mEqSet, mEqCString}
+    mEqStr, mEqSet, mEqCString}
 
   # set excluded here as the semantics are vastly different:
-  someLe = {mLeI, mLeF64, mLeU, mLeU64, mLeEnum,
+  someLe = {mLeI, mLeF64, mLeU, mLeEnum,
             mLeCh, mLeB, mLePtr, mLeStr}
-  someLt = {mLtI, mLtF64, mLtU, mLtU64, mLtEnum,
+  someLt = {mLtI, mLtF64, mLtU, mLtEnum,
             mLtCh, mLtB, mLtPtr, mLtStr}
 
-  someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq,
-             mXLenStr, mXLenSeq}
+  someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq}
 
-  someIn = {mInRange, mInSet}
+  someIn = {mInSet}
 
   someHigh = {mHigh}
   # we don't list unsigned here because wrap around semantics suck for
@@ -35,8 +34,8 @@ const
   someMul = {mMulI, mMulF64}
   someDiv = {mDivI, mDivF64}
   someMod = {mModI}
-  someMax = {mMaxI, mMaxF64}
-  someMin = {mMinI, mMinF64}
+  someMax = {mMaxI}
+  someMin = {mMinI}
   someBinaryOp = someAdd+someSub+someMul+someMax+someMin
 
 proc isValue(n: PNode): bool = n.kind in {nkCharLit..nkNilLit}
@@ -65,16 +64,16 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
   while true:
     case n.kind
     of nkDotExpr, nkCheckedFieldExpr, nkObjUpConv, nkObjDownConv:
-      n = n.sons[0]
+      n = n[0]
     of nkDerefExpr, nkHiddenDeref:
-      n = n.sons[0]
+      n = n[0]
       inc derefs
     of nkBracketExpr:
-      if isConstExpr(n.sons[1]) or isLet(n.sons[1]):
-        n = n.sons[0]
+      if isConstExpr(n[1]) or isLet(n[1]) or isConstExpr(n[1].skipConv):
+        n = n[0]
       else: return
     of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-      n = n.sons[1]
+      n = n[1]
     else:
       break
   result = n.isLet and derefs <= ord(isApprox)
@@ -105,34 +104,34 @@ proc initOperators*(g: ModuleGraph): Operators =
 
 proc swapArgs(fact: PNode, newOp: PSym): PNode =
   result = newNodeI(nkCall, fact.info, 3)
-  result.sons[0] = newSymNode(newOp)
-  result.sons[1] = fact.sons[2]
-  result.sons[2] = fact.sons[1]
+  result[0] = newSymNode(newOp)
+  result[1] = fact[2]
+  result[2] = fact[1]
 
 proc neg(n: PNode; o: Operators): PNode =
   if n == nil: return nil
   case n.getMagic
   of mNot:
-    result = n.sons[1]
+    result = n[1]
   of someLt:
     # not (a < b)  ==  a >= b  ==  b <= a
     result = swapArgs(n, o.opLe)
   of someLe:
     result = swapArgs(n, o.opLt)
   of mInSet:
-    if n.sons[1].kind != nkCurly: return nil
-    let t = n.sons[2].typ.skipTypes(abstractInst)
+    if n[1].kind != nkCurly: return nil
+    let t = n[2].typ.skipTypes(abstractInst)
     result = newNodeI(nkCall, n.info, 3)
-    result.sons[0] = n.sons[0]
-    result.sons[2] = n.sons[2]
+    result[0] = n[0]
+    result[2] = n[2]
     if t.kind == tyEnum:
-      var s = newNodeIT(nkCurly, n.info, n.sons[1].typ)
+      var s = newNodeIT(nkCurly, n.info, n[1].typ)
       for e in t.n:
         let eAsNode = newIntNode(nkIntLit, e.sym.position)
-        if not inSet(n.sons[1], eAsNode): s.add eAsNode
-      result.sons[1] = s
+        if not inSet(n[1], eAsNode): s.add eAsNode
+      result[1] = s
     #elif t.kind notin {tyString, tySequence} and lengthOrd(t) < 1000:
-    #  result.sons[1] = complement(n.sons[1])
+    #  result[1] = complement(n[1])
     else:
       # not ({2, 3, 4}.contains(x))   x != 2 and x != 3 and x != 4
       # XXX todo
@@ -140,13 +139,13 @@ proc neg(n: PNode; o: Operators): PNode =
   of mOr:
     # not (a or b) --> not a and not b
     let
-      a = n.sons[1].neg(o)
-      b = n.sons[2].neg(o)
+      a = n[1].neg(o)
+      b = n[2].neg(o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
     elif a != nil:
       result = a
     elif b != nil:
@@ -154,19 +153,19 @@ proc neg(n: PNode; o: Operators): PNode =
   else:
     # leave  not (a == 4)  as it is
     result = newNodeI(nkCall, n.info, 2)
-    result.sons[0] = newSymNode(o.opNot)
-    result.sons[1] = n
+    result[0] = newSymNode(o.opNot)
+    result[1] = n
 
 proc buildCall(op: PSym; a: PNode): PNode =
   result = newNodeI(nkCall, a.info, 2)
-  result.sons[0] = newSymNode(op)
-  result.sons[1] = a
+  result[0] = newSymNode(op)
+  result[1] = a
 
 proc buildCall(op: PSym; a, b: PNode): PNode =
   result = newNodeI(nkInfix, a.info, 3)
-  result.sons[0] = newSymNode(op)
-  result.sons[1] = a
-  result.sons[2] = b
+  result[0] = newSymNode(op)
+  result[1] = a
+  result[2] = b
 
 proc `|+|`(a, b: PNode): PNode =
   result = copyNode(a)
@@ -250,35 +249,35 @@ proc pred(n: PNode): PNode =
   else:
     result = n
 
+proc buildLe*(o: Operators; a, b: PNode): PNode =
+  result = o.opLe.buildCall(a, b)
+
 proc canon*(n: PNode; o: Operators): PNode =
-  # XXX for now only the new code in 'semparallel' uses this
   if n.safeLen >= 1:
     result = shallowCopy(n)
-    for i in 0 ..< n.len:
-      result.sons[i] = canon(n.sons[i], o)
+    for i in 0..<n.len:
+      result[i] = canon(n[i], o)
   elif n.kind == nkSym and n.sym.kind == skLet and
-      n.sym.ast.getMagic in (someEq + someAdd + someMul + someMin +
-      someMax + someHigh + {mUnaryLt} + someSub + someLen + someDiv):
-    result = n.sym.ast.copyTree
+      n.sym.astdef.getMagic in (someEq + someAdd + someMul + someMin +
+      someMax + someHigh + someSub + someLen + someDiv):
+    result = n.sym.astdef.copyTree
   else:
     result = n
   case result.getMagic
   of someEq, someAdd, someMul, someMin, someMax:
     # these are symmetric; put value as last:
-    if result.sons[1].isValue and not result.sons[2].isValue:
-      result = swapArgs(result, result.sons[0].sym)
+    if result[1].isValue and not result[2].isValue:
+      result = swapArgs(result, result[0].sym)
       # (4 + foo) + 2 --> (foo + 4) + 2
   of someHigh:
     # high == len+(-1)
     result = o.opAdd.buildCall(o.opLen.buildCall(result[1]), minusOne())
-  of mUnaryLt:
-    result = buildCall(o.opAdd, result[1], minusOne())
   of someSub:
     # x - 4  -->  x + (-4)
     result = negate(result[1], result[2], result, o)
   of someLen:
-    result.sons[0] = o.opLen.newSymNode
-  of someLt:
+    result[0] = o.opLen.newSymNode
+  of someLt - {mLtF64}:
     # x < y  same as x <= y-1:
     let y = n[2].canon(o)
     let p = pred(y)
@@ -318,16 +317,16 @@ proc canon*(n: PNode; o: Operators): PNode =
         if plus != nil and not isLetLocation(x, true):
           result = buildCall(result[0].sym, plus, y[1])
       else: discard
-    elif x.isValue and y.getMagic in someAdd and y[2].isValue:
+    elif x.isValue and y.getMagic in someAdd and y[2].kind == x.kind:
       # 0 <= a.len + 3
       # -3 <= a.len
-      result.sons[1] = x |-| y[2]
-      result.sons[2] = y[1]
-    elif x.isValue and y.getMagic in someSub and y[2].isValue:
+      result[1] = x |-| y[2]
+      result[2] = y[1]
+    elif x.isValue and y.getMagic in someSub and y[2].kind == x.kind:
       # 0 <= a.len - 3
       # 3 <= a.len
-      result.sons[1] = x |+| y[2]
-      result.sons[2] = y[1]
+      result[1] = x |+| y[2]
+      result[2] = y[1]
   else: discard
 
 proc buildAdd*(a: PNode; b: BiggestInt; o: Operators): PNode =
@@ -336,41 +335,43 @@ proc buildAdd*(a: PNode; b: BiggestInt; o: Operators): PNode =
 proc usefulFact(n: PNode; o: Operators): PNode =
   case n.getMagic
   of someEq:
-    if skipConv(n.sons[2]).kind == nkNilLit and (
-        isLetLocation(n.sons[1], false) or isVar(n.sons[1])):
-      result = o.opIsNil.buildCall(n.sons[1])
+    if skipConv(n[2]).kind == nkNilLit and (
+        isLetLocation(n[1], false) or isVar(n[1])):
+      result = o.opIsNil.buildCall(n[1])
     else:
-      if isLetLocation(n.sons[1], true) or isLetLocation(n.sons[2], true):
+      if isLetLocation(n[1], true) or isLetLocation(n[2], true):
         # XXX algebraic simplifications!  'i-1 < a.len' --> 'i < a.len+1'
         result = n
+      elif n[1].getMagic in someLen or n[2].getMagic in someLen:
+        result = n
   of someLe+someLt:
-    if isLetLocation(n.sons[1], true) or isLetLocation(n.sons[2], true):
+    if isLetLocation(n[1], true) or isLetLocation(n[2], true):
       # XXX algebraic simplifications!  'i-1 < a.len' --> 'i < a.len+1'
       result = n
     elif n[1].getMagic in someLen or n[2].getMagic in someLen:
       # XXX Rethink this whole idea of 'usefulFact' for semparallel
       result = n
   of mIsNil:
-    if isLetLocation(n.sons[1], false) or isVar(n.sons[1]):
+    if isLetLocation(n[1], false) or isVar(n[1]):
       result = n
   of someIn:
-    if isLetLocation(n.sons[1], true):
+    if isLetLocation(n[1], true):
       result = n
   of mAnd:
     let
-      a = usefulFact(n.sons[1], o)
-      b = usefulFact(n.sons[2], o)
+      a = usefulFact(n[1], o)
+      b = usefulFact(n[2], o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
     elif a != nil:
       result = a
     elif b != nil:
       result = b
   of mNot:
-    let a = usefulFact(n.sons[1], o)
+    let a = usefulFact(n[1], o)
     if a != nil:
       result = a.neg(o)
   of mOr:
@@ -381,13 +382,13 @@ proc usefulFact(n: PNode; o: Operators): PNode =
     #  (x == 3) or (y == 2)  ---> not ( not (x==3) and not (y == 2))
     #  not (x != 3 and y != 2)
     let
-      a = usefulFact(n.sons[1], o).neg(o)
-      b = usefulFact(n.sons[2], o).neg(o)
+      a = usefulFact(n[1], o).neg(o)
+      b = usefulFact(n[2], o).neg(o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
       result = result.neg(o)
   elif n.kind == nkSym and n.sym.kind == skLet:
     # consider:
@@ -395,8 +396,8 @@ proc usefulFact(n: PNode; o: Operators): PNode =
     #   if a:
     #     ...
     # We make can easily replace 'a' by '2 < x' here:
-    if n.sym.ast != nil:
-      result = usefulFact(n.sym.ast, o)
+    if n.sym.astdef != nil:
+      result = usefulFact(n.sym.astdef, o)
   elif n.kind == nkStmtListExpr:
     result = usefulFact(n.lastSon, o)
 
@@ -404,10 +405,20 @@ type
   TModel* = object
     s*: seq[PNode] # the "knowledge base"
     o*: Operators
+    beSmart*: bool
 
 proc addFact*(m: var TModel, nn: PNode) =
   let n = usefulFact(nn, m.o)
-  if n != nil: m.s.add n
+  if n != nil:
+    if not m.beSmart:
+      m.s.add n
+    else:
+      let c = canon(n, m.o)
+      if c.getMagic == mAnd:
+        addFact(m, c[1])
+        addFact(m, c[2])
+      else:
+        m.s.add c
 
 proc addFactNeg*(m: var TModel, n: PNode) =
   let n = n.neg(m.o)
@@ -436,22 +447,22 @@ proc sameTree*(a, b: PNode): bool =
       if not result and a.sym.magic != mNone:
         result = a.sym.magic == b.sym.magic or sameOpr(a.sym, b.sym)
     of nkIdent: result = a.ident.id == b.ident.id
-    of nkCharLit..nkInt64Lit: result = a.intVal == b.intVal
+    of nkCharLit..nkUInt64Lit: result = a.intVal == b.intVal
     of nkFloatLit..nkFloat64Lit: result = a.floatVal == b.floatVal
     of nkStrLit..nkTripleStrLit: result = a.strVal == b.strVal
     of nkType: result = a.typ == b.typ
     of nkEmpty, nkNilLit: result = true
     else:
-      if sonsLen(a) == sonsLen(b):
-        for i in countup(0, sonsLen(a) - 1):
-          if not sameTree(a.sons[i], b.sons[i]): return
+      if a.len == b.len:
+        for i in 0..<a.len:
+          if not sameTree(a[i], b[i]): return
         result = true
 
 proc hasSubTree(n, x: PNode): bool =
   if n.sameTree(x): result = true
   else:
-    for i in 0..safeLen(n)-1:
-      if hasSubTree(n.sons[i], x): return true
+    for i in 0..n.safeLen-1:
+      if hasSubTree(n[i], x): return true
 
 proc invalidateFacts*(m: var TModel, n: PNode) =
   # We are able to guard local vars (as opposed to 'let' variables)!
@@ -479,21 +490,21 @@ proc valuesUnequal(a, b: PNode): bool =
     result = not sameValue(a, b)
 
 proc impliesEq(fact, eq: PNode): TImplication =
-  let (loc, val) = if isLocation(eq.sons[1]): (1, 2) else: (2, 1)
+  let (loc, val) = if isLocation(eq[1]): (1, 2) else: (2, 1)
 
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], eq.sons[loc]):
+    if sameTree(fact[1], eq[loc]):
       # this is not correct; consider:  a == b;  a == 1 --> unknown!
-      if sameTree(fact.sons[2], eq.sons[val]): result = impYes
-      elif valuesUnequal(fact.sons[2], eq.sons[val]): result = impNo
-    elif sameTree(fact.sons[2], eq.sons[loc]):
-      if sameTree(fact.sons[1], eq.sons[val]): result = impYes
-      elif valuesUnequal(fact.sons[1], eq.sons[val]): result = impNo
+      if sameTree(fact[2], eq[val]): result = impYes
+      elif valuesUnequal(fact[2], eq[val]): result = impNo
+    elif sameTree(fact[2], eq[loc]):
+      if sameTree(fact[1], eq[val]): result = impYes
+      elif valuesUnequal(fact[1], eq[val]): result = impNo
   of mInSet:
     # remember: mInSet is 'contains' so the set comes first!
-    if sameTree(fact.sons[2], eq.sons[loc]) and isValue(eq.sons[val]):
-      if inSet(fact.sons[1], eq.sons[val]): result = impYes
+    if sameTree(fact[2], eq[loc]) and isValue(eq[val]):
+      if inSet(fact[1], eq[val]): result = impYes
       else: result = impNo
   of mNot, mOr, mAnd: assert(false, "impliesEq")
   else: discard
@@ -522,7 +533,7 @@ proc geImpliesIn(x, c, aSet: PNode): TImplication =
     var value = newIntNode(c.kind, c.intVal)
     let max = lastOrd(nil, x.typ)
     # don't iterate too often:
-    if max - value.intVal < 1000:
+    if max - getInt(value) < toInt128(1000):
       var i, pos, neg: int
       while value.intVal <= max:
         if inSet(aSet, value): inc pos
@@ -536,28 +547,28 @@ proc compareSets(a, b: PNode): TImplication =
   elif intersectSets(nil, a, b).len == 0: result = impNo
 
 proc impliesIn(fact, loc, aSet: PNode): TImplication =
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], loc):
-      if inSet(aSet, fact.sons[2]): result = impYes
+    if sameTree(fact[1], loc):
+      if inSet(aSet, fact[2]): result = impYes
       else: result = impNo
-    elif sameTree(fact.sons[2], loc):
-      if inSet(aSet, fact.sons[1]): result = impYes
+    elif sameTree(fact[2], loc):
+      if inSet(aSet, fact[1]): result = impYes
       else: result = impNo
   of mInSet:
-    if sameTree(fact.sons[2], loc):
-      result = compareSets(fact.sons[1], aSet)
+    if sameTree(fact[2], loc):
+      result = compareSets(fact[1], aSet)
   of someLe:
-    if sameTree(fact.sons[1], loc):
-      result = leImpliesIn(fact.sons[1], fact.sons[2], aSet)
-    elif sameTree(fact.sons[2], loc):
-      result = geImpliesIn(fact.sons[2], fact.sons[1], aSet)
+    if sameTree(fact[1], loc):
+      result = leImpliesIn(fact[1], fact[2], aSet)
+    elif sameTree(fact[2], loc):
+      result = geImpliesIn(fact[2], fact[1], aSet)
   of someLt:
-    if sameTree(fact.sons[1], loc):
-      result = leImpliesIn(fact.sons[1], fact.sons[2].pred, aSet)
-    elif sameTree(fact.sons[2], loc):
+    if sameTree(fact[1], loc):
+      result = leImpliesIn(fact[1], fact[2].pred, aSet)
+    elif sameTree(fact[2], loc):
       # 4 < x  -->  3 <= x
-      result = geImpliesIn(fact.sons[2], fact.sons[1].pred, aSet)
+      result = geImpliesIn(fact[2], fact[1].pred, aSet)
   of mNot, mOr, mAnd: assert(false, "impliesIn")
   else: discard
 
@@ -567,90 +578,93 @@ proc valueIsNil(n: PNode): TImplication =
   else: impUnknown
 
 proc impliesIsNil(fact, eq: PNode): TImplication =
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of mIsNil:
-    if sameTree(fact.sons[1], eq.sons[1]):
+    if sameTree(fact[1], eq[1]):
       result = impYes
   of someEq:
-    if sameTree(fact.sons[1], eq.sons[1]):
-      result = valueIsNil(fact.sons[2].skipConv)
-    elif sameTree(fact.sons[2], eq.sons[1]):
-      result = valueIsNil(fact.sons[1].skipConv)
+    if sameTree(fact[1], eq[1]):
+      result = valueIsNil(fact[2].skipConv)
+    elif sameTree(fact[2], eq[1]):
+      result = valueIsNil(fact[1].skipConv)
   of mNot, mOr, mAnd: assert(false, "impliesIsNil")
   else: discard
 
 proc impliesGe(fact, x, c: PNode): TImplication =
   assert isLocation(x)
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x = 4;  question x >= 56? --> true iff 4 >= 56
-        if leValue(c, fact.sons[2]): result = impYes
+        if leValue(c, fact[2]): result = impYes
         else: result = impNo
-    elif sameTree(fact.sons[2], x):
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impYes
+    elif sameTree(fact[2], x):
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impYes
         else: result = impNo
   of someLt:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x < 4;  question N <= x? --> false iff N <= 4
-        if leValue(fact.sons[2], c): result = impNo
+        if leValue(fact[2], c): result = impNo
         # fact:  x < 4;  question 2 <= x? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 < x; question: N-1 < x ?  --> true iff N-1 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c.pred, fact.sons[1]): result = impYes
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c.pred, fact[1]): result = impYes
   of someLe:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x <= 4;  question x >= 56? --> false iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impNo
+        if leValue(fact[2], c): result = impNo
         # fact:  x <= 4;  question x >= 2? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 <= x; question: x >= 2 ?  --> true iff 2 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impYes
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impYes
   of mNot, mOr, mAnd: assert(false, "impliesGe")
   else: discard
 
 proc impliesLe(fact, x, c: PNode): TImplication =
   if not isLocation(x):
+    if c.isValue:
+      if leValue(x, x): return impYes
+      else: return impNo
     return impliesGe(fact, c, x)
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x = 4;  question x <= 56? --> true iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impYes
+        if leValue(fact[2], c): result = impYes
         else: result = impNo
-    elif sameTree(fact.sons[2], x):
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(fact.sons[1], c): result = impYes
+    elif sameTree(fact[2], x):
+      if isValue(fact[1]) and isValue(c):
+        if leValue(fact[1], c): result = impYes
         else: result = impNo
   of someLt:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x < 4;  question x <= N? --> true iff N-1 <= 4
-        if leValue(fact.sons[2], c.pred): result = impYes
+        if leValue(fact[2], c.pred): result = impYes
         # fact:  x < 4;  question x <= 2? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 < x; question: x <= 1 ?  --> false iff 1 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impNo
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impNo
 
   of someLe:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x <= 4;  question x <= 56? --> true iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impYes
+        if leValue(fact[2], c): result = impYes
         # fact:  x <= 4;  question x <= 2? --> we don't know
 
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 <= x; question: x <= 2 ?  --> false iff 2 < 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1].pred): result = impNo
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1].pred): result = impNo
 
   of mNot, mOr, mAnd: assert(false, "impliesLe")
   else: discard
@@ -686,32 +700,32 @@ proc factImplies(fact, prop: PNode): TImplication =
 
     #  (not a) -> b  compute as  not (a -> b) ???
     #  == not a or not b == not (a and b)
-    let arg = fact.sons[1]
+    let arg = fact[1]
     case arg.getMagic
     of mIsNil, mEqRef:
       return ~factImplies(arg, prop)
     of mAnd:
       # not (a and b)  means  not a or not b:
       # a or b --> both need to imply 'prop'
-      let a = factImplies(arg.sons[1], prop)
-      let b = factImplies(arg.sons[2], prop)
+      let a = factImplies(arg[1], prop)
+      let b = factImplies(arg[2], prop)
       if a == b: return ~a
       return impUnknown
     else:
       return impUnknown
   of mAnd:
-    result = factImplies(fact.sons[1], prop)
+    result = factImplies(fact[1], prop)
     if result != impUnknown: return result
-    return factImplies(fact.sons[2], prop)
+    return factImplies(fact[2], prop)
   else: discard
 
-  case prop.sons[0].sym.magic
-  of mNot: result = ~fact.factImplies(prop.sons[1])
+  case prop[0].sym.magic
+  of mNot: result = ~fact.factImplies(prop[1])
   of mIsNil: result = impliesIsNil(fact, prop)
   of someEq: result = impliesEq(fact, prop)
-  of someLe: result = impliesLe(fact, prop.sons[1], prop.sons[2])
-  of someLt: result = impliesLt(fact, prop.sons[1], prop.sons[2])
-  of mInSet: result = impliesIn(fact, prop.sons[2], prop.sons[1])
+  of someLe: result = impliesLe(fact, prop[1], prop[2])
+  of someLt: result = impliesLt(fact, prop[1], prop[2])
+  of mInSet: result = impliesIn(fact, prop[2], prop[1])
   else: result = impUnknown
 
 proc doesImply*(facts: TModel, prop: PNode): TImplication =
@@ -802,7 +816,7 @@ proc ple(m: TModel; a, b: PNode): TImplication =
     if lastOrd(nil, a.typ) <= b.intVal: return impYes
   # 3 <= x   iff  low(x) <= 3
   if a.isValue and b.typ != nil and b.typ.isOrdinalType:
-    if firstOrd(nil, b.typ) <= a.intVal: return impYes
+    if a.intVal <= firstOrd(nil, b.typ): return impYes
 
   # x <= x
   if sameTree(a, b): return impYes
@@ -813,7 +827,11 @@ proc ple(m: TModel; a, b: PNode): TImplication =
 
   #   x <= y+c  if 0 <= c and x <= y
   #   x <= y+(-c)  if c <= 0  and y >= x
-  if b.getMagic in someAdd and zero() <=? b[2] and a <=? b[1]: return impYes
+  if b.getMagic in someAdd:
+    if zero() <=? b[2] and a <=? b[1]: return impYes
+    # x <= y-c  if x+c <= y
+    if b[2] <=? zero() and (canon(m.o.opSub.buildCall(a, b[2]), m.o) <=? b[1]):
+      return impYes
 
   #   x+c <= y  if c <= 0 and x <= y
   if a.getMagic in someAdd and a[2] <=? zero() and a[1] <=? b: return impYes
@@ -881,8 +899,8 @@ proc replaceSubTree(n, x, by: PNode): PNode =
     result = by
   elif hasSubTree(n, x):
     result = shallowCopy(n)
-    for i in 0 .. safeLen(n)-1:
-      result.sons[i] = replaceSubTree(n.sons[i], x, by)
+    for i in 0..n.safeLen-1:
+      result[i] = replaceSubTree(n[i], x, by)
   else:
     result = n
 
@@ -902,10 +920,13 @@ proc pleViaModelRec(m: var TModel; a, b: PNode): TImplication =
       # --> true  if  (len-100) <= (len-1)
       let x = fact[1]
       let y = fact[2]
-      if sameTree(x, a) and y.getMagic in someAdd and b.getMagic in someAdd and
-         sameTree(y[1], b[1]):
-        if ple(m, b[2], y[2]) == impYes:
-          return impYes
+      # x <= y.
+      # Question: x <= b? True iff y <= b.
+      if sameTree(x, a):
+        if ple(m, y, b) == impYes: return impYes
+        if y.getMagic in someAdd and b.getMagic in someAdd and sameTree(y[1], b[1]):
+          if ple(m, b[2], y[2]) == impYes:
+            return impYes
 
       # x <= y implies a <= b  if  a <= x and y <= b
       if ple(m, a, x) == impYes:
@@ -964,6 +985,10 @@ proc proveLe*(m: TModel; a, b: PNode): TImplication =
 proc addFactLe*(m: var TModel; a, b: PNode) =
   m.s.add canon(m.o.opLe.buildCall(a, b), m.o)
 
+proc addFactLt*(m: var TModel; a, b: PNode) =
+  let bb = m.o.opAdd.buildCall(b, minusOne())
+  addFactLe(m, a, bb)
+
 proc settype(n: PNode): PType =
   result = newType(tySet, n.typ.owner)
   addSonSkipIntLit(result, n.typ)
@@ -971,36 +996,37 @@ proc settype(n: PNode): PType =
 proc buildOf(it, loc: PNode; o: Operators): PNode =
   var s = newNodeI(nkCurly, it.info, it.len-1)
   s.typ = settype(loc)
-  for i in 0..it.len-2: s.sons[i] = it.sons[i]
+  for i in 0..<it.len-1: s[i] = it[i]
   result = newNodeI(nkCall, it.info, 3)
-  result.sons[0] = newSymNode(o.opContains)
-  result.sons[1] = s
-  result.sons[2] = loc
+  result[0] = newSymNode(o.opContains)
+  result[1] = s
+  result[2] = loc
 
 proc buildElse(n: PNode; o: Operators): PNode =
-  var s = newNodeIT(nkCurly, n.info, settype(n.sons[0]))
-  for i in 1..n.len-2:
-    let branch = n.sons[i]
-    assert branch.kind == nkOfBranch
-    for j in 0..branch.len-2:
-      s.add(branch.sons[j])
+  var s = newNodeIT(nkCurly, n.info, settype(n[0]))
+  for i in 1..<n.len-1:
+    let branch = n[i]
+    assert branch.kind != nkElse
+    if branch.kind == nkOfBranch:
+      for j in 0..<branch.len-1:
+        s.add(branch[j])
   result = newNodeI(nkCall, n.info, 3)
-  result.sons[0] = newSymNode(o.opContains)
-  result.sons[1] = s
-  result.sons[2] = n.sons[0]
+  result[0] = newSymNode(o.opContains)
+  result[1] = s
+  result[2] = n[0]
 
 proc addDiscriminantFact*(m: var TModel, n: PNode) =
   var fact = newNodeI(nkCall, n.info, 3)
-  fact.sons[0] = newSymNode(m.o.opEq)
-  fact.sons[1] = n.sons[0]
-  fact.sons[2] = n.sons[1]
+  fact[0] = newSymNode(m.o.opEq)
+  fact[1] = n[0]
+  fact[2] = n[1]
   m.s.add fact
 
 proc addAsgnFact*(m: var TModel, key, value: PNode) =
   var fact = newNodeI(nkCall, key.info, 3)
-  fact.sons[0] = newSymNode(m.o.opEq)
-  fact.sons[1] = key
-  fact.sons[2] = value
+  fact[0] = newSymNode(m.o.opEq)
+  fact[1] = key
+  fact[2] = value
   m.s.add fact
 
 proc sameSubexprs*(m: TModel; a, b: PNode): bool =
@@ -1014,34 +1040,34 @@ proc sameSubexprs*(m: TModel; a, b: PNode): bool =
   # However, nil checking requires exactly the same mechanism! But for now
   # we simply use sameTree and live with the unsoundness of the analysis.
   var check = newNodeI(nkCall, a.info, 3)
-  check.sons[0] = newSymNode(m.o.opEq)
-  check.sons[1] = a
-  check.sons[2] = b
+  check[0] = newSymNode(m.o.opEq)
+  check[1] = a
+  check[2] = b
   result = m.doesImply(check) == impYes
 
 proc addCaseBranchFacts*(m: var TModel, n: PNode, i: int) =
-  let branch = n.sons[i]
+  let branch = n[i]
   if branch.kind == nkOfBranch:
-    m.s.add buildOf(branch, n.sons[0], m.o)
+    m.s.add buildOf(branch, n[0], m.o)
   else:
     m.s.add n.buildElse(m.o).neg(m.o)
 
 proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
-  if check.sons[1].kind == nkCurly:
+  if check[1].kind == nkCurly:
     result = copyTree(check)
     if access.kind == nkDotExpr:
       var a = copyTree(access)
-      a.sons[1] = check.sons[2]
-      result.sons[2] = a
+      a[1] = check[2]
+      result[2] = a
       # 'access.kind != nkDotExpr' can happen for object constructors
       # which we don't check yet
   else:
     # it is some 'not'
     assert check.getMagic == mNot
-    result = buildProperFieldCheck(access, check.sons[1], o).neg(o)
+    result = buildProperFieldCheck(access, check[1], o).neg(o)
 
 proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef) =
-  for i in 1..n.len-1:
-    let check = buildProperFieldCheck(n.sons[0], n.sons[i], m.o)
+  for i in 1..<n.len:
+    let check = buildProperFieldCheck(n[0], n[i], m.o)
     if check != nil and m.doesImply(check) != impYes:
-      message(conf, n.info, warnProveField, renderTree(n.sons[0])); break
+      message(conf, n.info, warnProveField, renderTree(n[0])); break

compiler/hlo.nim

@@ -12,14 +12,14 @@
 proc hlo(c: PContext, n: PNode): PNode
 
 proc evalPattern(c: PContext, n, orig: PNode): PNode =
-  internalAssert c.config, n.kind == nkCall and n.sons[0].kind == nkSym
+  internalAssert c.config, n.kind == nkCall and n[0].kind == nkSym
   # we need to ensure that the resulting AST is semchecked. However, it's
-  # aweful to semcheck before macro invocation, so we don't and treat
+  # awful to semcheck before macro invocation, so we don't and treat
   # templates and macros as immediate in this context.
   var rule: string
-  if optHints in c.config.options and hintPattern in c.config.notes:
+  if c.config.hasHint(hintPattern):
     rule = renderTree(n, {renderNoComments})
-  let s = n.sons[0].sym
+  let s = n[0].sym
   case s.kind
   of skMacro:
     result = semMacroExpr(c, n, orig, s)
@@ -27,7 +27,7 @@ proc evalPattern(c: PContext, n, orig: PNode): PNode =
     result = semTemplateExpr(c, n, s, {efFromHlo})
   else:
     result = semDirectOp(c, n, {})
-  if optHints in c.config.options and hintPattern in c.config.notes:
+  if c.config.hasHint(hintPattern):
     message(c.config, orig.info, hintPattern, rule & " --> '" &
       renderTree(result, {renderNoComments}) & "'")
 
@@ -50,7 +50,7 @@ proc applyPatterns(c: PContext, n: PNode): PNode =
         c.patterns[i] = nil
         if x.kind == nkStmtList:
           assert x.len == 3
-          x.sons[1] = evalPattern(c, x.sons[1], result)
+          x[1] = evalPattern(c, x[1], result)
           result = flattenStmts(x)
         else:
           result = evalPattern(c, x, result)
@@ -68,17 +68,17 @@ proc hlo(c: PContext, n: PNode): PNode =
     result = n
   else:
     if n.kind in {nkFastAsgn, nkAsgn, nkIdentDefs, nkVarTuple} and
-        n.sons[0].kind == nkSym and
-        {sfGlobal, sfPure} * n.sons[0].sym.flags == {sfGlobal, sfPure}:
+        n[0].kind == nkSym and
+        {sfGlobal, sfPure} * n[0].sym.flags == {sfGlobal, sfPure}:
       # do not optimize 'var g {.global} = re(...)' again!
       return n
     result = applyPatterns(c, n)
     if result == n:
       # no optimization applied, try subtrees:
-      for i in 0 ..< safeLen(result):
-        let a = result.sons[i]
+      for i in 0..<result.safeLen:
+        let a = result[i]
         let h = hlo(c, a)
-        if h != a: result.sons[i] = h
+        if h != a: result[i] = h
     else:
       # perform type checking, so that the replacement still fits:
       if isEmptyType(n.typ) and isEmptyType(result.typ):
@@ -92,12 +92,12 @@ proc hlo(c: PContext, n: PNode): PNode =
 
 proc hloBody(c: PContext, n: PNode): PNode =
   # fast exit:
-  if c.patterns.len == 0 or optPatterns notin c.config.options: return n
+  if c.patterns.len == 0 or optTrMacros notin c.config.options: return n
   c.hloLoopDetector = 0
   result = hlo(c, n)
 
 proc hloStmt(c: PContext, n: PNode): PNode =
   # fast exit:
-  if c.patterns.len == 0 or optPatterns notin c.config.options: return n
+  if c.patterns.len == 0 or optTrMacros notin c.config.options: return n
   c.hloLoopDetector = 0
   result = hlo(c, n)

compiler/idents.nim

@@ -12,7 +12,7 @@
 # id. This module is essential for the compiler's performance.
 
 import
-  hashes, strutils, wordrecg
+  hashes, wordrecg
 
 type
   TIdObj* = object of RootObj
@@ -87,7 +87,7 @@ proc getIdent*(ic: IdentCache; identifier: cstring, length: int, h: Hash): PIden
   new(result)
   result.h = h
   result.s = newString(length)
-  for i in countup(0, length - 1): result.s[i] = identifier[i]
+  for i in 0..<length: result.s[i] = identifier[i]
   result.next = ic.buckets[idx]
   ic.buckets[idx] = result
   if id == 0:
@@ -97,11 +97,11 @@ proc getIdent*(ic: IdentCache; identifier: cstring, length: int, h: Hash): PIden
     result.id = id
 
 proc getIdent*(ic: IdentCache; identifier: string): PIdent =
-  result = getIdent(ic, cstring(identifier), len(identifier),
+  result = getIdent(ic, cstring(identifier), identifier.len,
                     hashIgnoreStyle(identifier))
 
 proc getIdent*(ic: IdentCache; identifier: string, h: Hash): PIdent =
-  result = getIdent(ic, cstring(identifier), len(identifier), h)
+  result = getIdent(ic, cstring(identifier), identifier.len, h)
 
 proc newIdentCache*(): IdentCache =
   result = IdentCache()
@@ -110,7 +110,7 @@ proc newIdentCache*(): IdentCache =
   result.idDelegator = result.getIdent":delegator"
   result.emptyIdent = result.getIdent("")
   # initialize the keywords:
-  for s in countup(succ(low(specialWords)), high(specialWords)):
+  for s in succ(low(specialWords))..high(specialWords):
     result.getIdent(specialWords[s], hashIgnoreStyle(specialWords[s])).id = ord(s)
 
 proc whichKeyword*(id: PIdent): TSpecialWord =

compiler/idgen.nim

@@ -9,7 +9,7 @@
 
 ## This module contains a simple persistent id generator.
 
-import idents, strutils, os, options, pathutils
+import idents, strutils, options, pathutils
 
 var gFrontEndId*: int
 

compiler/importer.nim

@@ -10,13 +10,13 @@
 ## This module implements the symbol importing mechanism.
 
 import
-  intsets, strutils, os, ast, astalgo, msgs, options, idents, lookups,
-  semdata, passes, renderer, modulepaths, sigmatch, lineinfos
+  intsets, ast, astalgo, msgs, options, idents, lookups,
+  semdata, modulepaths, sigmatch, lineinfos, sets
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
   result = initIntSet()
-  for i in 1 ..< n.len:
+  for i in 1..<n.len:
     let ident = lookups.considerQuotedIdent(c, n[i])
     result.incl(ident.id)
 
@@ -27,11 +27,11 @@ proc importPureEnumField*(c: PContext; s: PSym) =
     if checkB == nil:
       strTableAdd(c.pureEnumFields, s)
     else:
-      # mark as ambigous:
+      # mark as ambiguous:
       incl(c.ambiguousSymbols, checkB.id)
       incl(c.ambiguousSymbols, s.id)
 
-proc rawImportSymbol(c: PContext, s: PSym) =
+proc rawImportSymbol(c: PContext, s, origin: PSym) =
   # This does not handle stubs, because otherwise loading on demand would be
   # pointless in practice. So importing stubs is fine here!
   # check if we have already a symbol of the same name:
@@ -47,8 +47,8 @@ proc rawImportSymbol(c: PContext, s: PSym) =
   if s.kind == skType:
     var etyp = s.typ
     if etyp.kind in {tyBool, tyEnum}:
-      for j in countup(0, sonsLen(etyp.n) - 1):
-        var e = etyp.n.sons[j].sym
+      for j in 0..<etyp.n.len:
+        var e = etyp.n[j].sym
         if e.kind != skEnumField:
           internalError(c.config, s.info, "rawImportSymbol")
           # BUGFIX: because of aliases for enums the symbol may already
@@ -63,13 +63,14 @@ proc rawImportSymbol(c: PContext, s: PSym) =
           check = nextIdentIter(it, c.importTable.symbols)
         if e != nil:
           if sfPure notin s.flags:
-            rawImportSymbol(c, e)
+            rawImportSymbol(c, e, origin)
           else:
             importPureEnumField(c, e)
   else:
-    # rodgen assures that converters and patterns are no stubs
     if s.kind == skConverter: addConverter(c, s)
     if hasPattern(s): addPattern(c, s)
+  if s.owner != origin:
+    c.exportIndirections.incl((origin.id, s.id))
 
 proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
   let ident = lookups.considerQuotedIdent(c, n)
@@ -79,19 +80,20 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
   else:
     when false:
       if s.kind == skStub: loadStub(s)
-    if s.kind notin ExportableSymKinds:
-      internalError(c.config, n.info, "importSymbol: 2")
+    let multiImport = s.kind notin ExportableSymKinds or s.kind in skProcKinds
     # for an enumeration we have to add all identifiers
-    case s.kind
-    of skProcKinds:
+    if multiImport:
       # for a overloadable syms add all overloaded routines
       var it: TIdentIter
       var e = initIdentIter(it, fromMod.tab, s.name)
       while e != nil:
         if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3")
-        rawImportSymbol(c, e)
+        if s.kind in ExportableSymKinds:
+          rawImportSymbol(c, e, fromMod)
         e = nextIdentIter(it, fromMod.tab)
-    else: rawImportSymbol(c, s)
+    else:
+      rawImportSymbol(c, s, fromMod)
+    suggestSym(c.config, n.info, s, c.graph.usageSym, false)
 
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
   var i: TTabIter
@@ -100,16 +102,16 @@ proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
     if s.kind != skModule:
       if s.kind != skEnumField:
         if s.kind notin ExportableSymKinds:
-          internalError(c.config, s.info, "importAllSymbols: " & $s.kind)
+          internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s)
         if exceptSet.isNil or s.name.id notin exceptSet:
-          rawImportSymbol(c, s)
+          rawImportSymbol(c, s, fromMod)
     s = nextIter(i, fromMod.tab)
 
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
   var exceptSet: IntSet
   importAllSymbolsExcept(c, fromMod, exceptSet)
 
-proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet) =
+proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym) =
   if n.isNil: return
   case n.kind
   of nkExportStmt:
@@ -119,38 +121,43 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet) =
       if s.kind == skModule:
         importAllSymbolsExcept(c, s, exceptSet)
       elif exceptSet.isNil or s.name.id notin exceptSet:
-        rawImportSymbol(c, s)
+        rawImportSymbol(c, s, fromMod)
   of nkExportExceptStmt:
     localError(c.config, n.info, "'export except' not implemented")
   else:
-    for i in 0..safeLen(n)-1:
-      importForwarded(c, n.sons[i], exceptSet)
+    for i in 0..n.safeLen-1:
+      importForwarded(c, n[i], exceptSet, fromMod)
 
 proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
   result = realModule
+  c.unusedImports.add((realModule, n.info))
   if n.kind != nkImportAs: discard
-  elif n.len != 2 or n.sons[1].kind != nkIdent:
+  elif n.len != 2 or n[1].kind != nkIdent:
     localError(c.config, n.info, "module alias must be an identifier")
-  elif n.sons[1].ident.id != realModule.name.id:
+  elif n[1].ident.id != realModule.name.id:
     # some misguided guy will write 'import abc.foo as foo' ...
-    result = createModuleAlias(realModule, n.sons[1].ident, realModule.info,
+    result = createModuleAlias(realModule, n[1].ident, realModule.info,
                                c.config.options)
 
 proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
   let f = checkModuleName(c.config, n)
-  if f != InvalidFileIDX:
+  if f != InvalidFileIdx:
     let L = c.graph.importStack.len
     let recursion = c.graph.importStack.find(f)
     c.graph.importStack.add f
     #echo "adding ", toFullPath(f), " at ", L+1
     if recursion >= 0:
       var err = ""
-      for i in countup(recursion, L-1):
+      for i in recursion..<L:
         if i > recursion: err.add "\n"
         err.add toFullPath(c.config, c.graph.importStack[i]) & " imports " &
                 toFullPath(c.config, c.graph.importStack[i+1])
       c.recursiveDep = err
+
+    discard pushOptionEntry(c)
     result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f))
+    popOptionEntry(c)
+
     #echo "set back to ", L
     c.graph.importStack.setLen(L)
     # we cannot perform this check reliably because of
@@ -161,9 +168,9 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
         localError(c.config, n.info, "A module cannot import itself")
     if sfDeprecated in result.flags:
       if result.constraint != nil:
-        message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s)
+        message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated")
       else:
-        message(c.config, n.info, warnDeprecated, result.name.s)
+        message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated")
     suggestSym(c.config, n.info, result, c.graph.usageSym, false)
     importStmtResult.add newSymNode(result, n.info)
     #newStrNode(toFullPath(c.config, f), n.info)
@@ -171,8 +178,8 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
 proc transformImportAs(c: PContext; n: PNode): PNode =
   if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
     result = newNodeI(nkImportAs, n.info)
-    result.add n.sons[1]
-    result.add n.sons[2]
+    result.add n[1]
+    result.add n[2]
   else:
     result = n
 
@@ -184,12 +191,12 @@ proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
     # ``addDecl`` needs to be done before ``importAllSymbols``!
     addDecl(c, m, it.info) # add symbol to symbol table of module
     importAllSymbolsExcept(c, m, emptySet)
-    #importForwarded(c, m.ast, emptySet)
+    #importForwarded(c, m.ast, emptySet, m)
 
 proc evalImport*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
-  for i in countup(0, sonsLen(n) - 1):
-    let it = n.sons[i]
+  for i in 0..<n.len:
+    let it = n[i]
     if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
       let sep = it[0]
       let dir = it[1]
@@ -199,13 +206,13 @@ proc evalImport*(c: PContext, n: PNode): PNode =
       imp.add sep # dummy entry, replaced in the loop
       for x in it[2]:
         # transform `a/b/[c as d]` to `/a/b/c as d`
-        if x.kind == nkInfix and x.sons[0].ident.s == "as":
+        if x.kind == nkInfix and x[0].ident.s == "as":
           let impAs = copyTree(x)
-          imp.sons[2] = x.sons[1]
-          impAs.sons[1] = imp
+          imp[2] = x[1]
+          impAs[1] = imp
           impMod(c, imp, result)
         else:
-          imp.sons[2] = x
+          imp[2] = x
           impMod(c, imp, result)
     else:
       impMod(c, it, result)
@@ -213,22 +220,22 @@ proc evalImport*(c: PContext, n: PNode): PNode =
 proc evalFrom*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n.sons[0] = transformImportAs(c, n.sons[0])
-  var m = myImportModule(c, n.sons[0], result)
+  n[0] = transformImportAs(c, n[0])
+  var m = myImportModule(c, n[0], result)
   if m != nil:
-    n.sons[0] = newSymNode(m)
+    n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
-    for i in countup(1, sonsLen(n) - 1):
-      if n.sons[i].kind != nkNilLit:
-        importSymbol(c, n.sons[i], m)
+    for i in 1..<n.len:
+      if n[i].kind != nkNilLit:
+        importSymbol(c, n[i], m)
 
 proc evalImportExcept*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n.sons[0] = transformImportAs(c, n.sons[0])
-  var m = myImportModule(c, n.sons[0], result)
+  n[0] = transformImportAs(c, n[0])
+  var m = myImportModule(c, n[0], result)
   if m != nil:
-    n.sons[0] = newSymNode(m)
+    n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
     importAllSymbolsExcept(c, m, readExceptSet(c, n))
-    #importForwarded(c, m.ast, exceptSet)
+    #importForwarded(c, m.ast, exceptSet, m)

compiler/incremental.nim

@@ -12,11 +12,12 @@
 
 const nimIncremental* = defined(nimIncremental)
 
-import options, lineinfos, pathutils
+import options, lineinfos
 
 when nimIncremental:
-  import ast, msgs, intsets, btrees, db_sqlite, std / sha1
+  import ast, msgs, intsets, btrees, db_sqlite, std / sha1, pathutils
   from strutils import parseInt
+  from os import isAbsolute
 
   type
     Writer* = object
@@ -47,7 +48,7 @@ when nimIncremental:
 
   proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: AbsoluteFile): string =
     result = msgs.getHash(conf, fileIdx)
-    if result.len == 0:
+    if result.len == 0 and isAbsolute(string fullpath):
       result = $secureHashFile(string fullpath)
       msgs.setHash(conf, fileIdx, result)
 
@@ -59,8 +60,8 @@ when nimIncremental:
     let id = row[0]
     let fullhash = hashFileCached(conf, fileIdx, AbsoluteFile fullpath)
     if id.len == 0:
-      result = int incr.db.insertID(sql"insert into filenames(fullpath, fullhash) values (?, ?)",
-        fullpath, fullhash)
+      result = int incr.db.insertID(sql"insert into filenames(nimid, fullpath, fullhash) values (?, ?, ?)",
+        int(fileIdx), fullpath, fullhash)
     else:
       if row[1] != fullhash:
         incr.db.exec(sql"update filenames set fullhash = ? where fullpath = ?", fullhash, fullpath)
@@ -102,6 +103,7 @@ when nimIncremental:
     db.exec(sql"""
       create table if not exists filenames(
         id integer primary key,
+        nimid integer not null,
         fullpath varchar(8000) not null,
         fullHash varchar(256) not null
       );

compiler/installer.ini

@@ -6,10 +6,10 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;amd64;powerpc64;arm;sparc;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv64
+  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv64
   macosx: i386;amd64;powerpc64
   solaris: i386;amd64;sparc;sparc64
-  freebsd: i386;amd64
+  freebsd: i386;amd64;powerpc64;arm;arm64
   netbsd: i386;amd64
   openbsd: i386;amd64
   dragonfly: i386;amd64
@@ -49,11 +49,8 @@ Start: "doc/html/overview.html"
 
 
 [Other]
-Files: "readme.txt;copying.txt;install.txt"
-Files: "makefile"
+Files: "copying.txt"
 Files: "koch.nim"
-Files: "install_nimble.nims"
-Files: "install_tools.nims"
 
 Files: "icons/nim.ico"
 Files: "icons/nim.rc"
@@ -68,17 +65,11 @@ Files: "compiler"
 Files: "doc"
 Files: "doc/html"
 Files: "tools"
+Files: "tools/nim-gdb.py"
 Files: "nimpretty"
+Files: "testament"
 Files: "nimsuggest"
 Files: "nimsuggest/tests/*.nim"
-Files: "web/website.ini"
-Files: "web/ticker.html"
-Files: "web/*.nim"
-Files: "web/*.rst"
-Files: "web/*.csv"
-Files: "web/news/*.rst"
-Files: "bin/nimblepkg/*.nim"
-Files: "bin/nimblepkg/*.cfg"
 
 [Lib]
 Files: "lib"
@@ -86,18 +77,18 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
-Files: "dist/nimsuggest"
 
 Files: "tests"
 
 [Windows]
 Files: "bin/nim.exe"
-Files: "bin/c2nim.exe"
 Files: "bin/nimgrep.exe"
 Files: "bin/nimsuggest.exe"
 Files: "bin/nimble.exe"
 Files: "bin/vccexe.exe"
 Files: "bin/nimgrab.exe"
+Files: "bin/nimpretty.exe"
+Files: "bin/testament.exe"
 
 Files: "koch.exe"
 Files: "finish.exe"
@@ -148,4 +139,4 @@ licenses: "bin/nim,MIT;lib/*,MIT;"
 
 [nimble]
 pkgName: "compiler"
-pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt"
+pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt;doc/nimdoc.css"

compiler/int128.nim

@@ -0,0 +1,781 @@
+## This module is for compiler internal use only. For reliable error
+## messages and range checks, the compiler needs a data type that can
+## hold all from ``low(BiggestInt)`` to ``high(BiggestUInt)``, This
+## type is for that purpose.
+
+from math import trunc
+
+type
+  Int128* = object
+    udata: array[4,uint32]
+
+template sdata(arg: Int128, idx: int): int32 =
+  # udata and sdata was supposed to be in a union, but unions are
+  # handled incorrectly in the VM.
+  cast[ptr int32](arg.udata[idx].unsafeAddr)[]
+
+# encoding least significant int first (like LittleEndian)
+
+const
+  Zero* = Int128(udata: [0'u32,0,0,0])
+  One* = Int128(udata: [1'u32,0,0,0])
+  Ten* = Int128(udata: [10'u32,0,0,0])
+  Min = Int128(udata: [0'u32,0,0,0x80000000'u32])
+  Max = Int128(udata: [high(uint32),high(uint32),high(uint32),uint32(high(int32))])
+  NegOne* = Int128(udata: [0xffffffff'u32,0xffffffff'u32,0xffffffff'u32,0xffffffff'u32])
+
+template low*(t: typedesc[Int128]): Int128 = Min
+template high*(t: typedesc[Int128]): Int128 = Max
+
+proc `$`*(a: Int128): string
+
+proc toInt128*[T: SomeInteger | bool](arg: T): Int128 =
+  when T is bool: result.sdata(0) = int32(arg)
+  elif T is SomeUnsignedInt:
+    when sizeof(arg) <= 4:
+      result.udata[0] = uint32(arg)
+    else:
+      result.udata[0] = uint32(arg and T(0xffffffff))
+      result.udata[1] = uint32(arg shr 32)
+  elif sizeof(arg) <= 4:
+    result.sdata(0) = int32(arg)
+    if arg < 0: # sign extend
+      result.sdata(1) = -1
+      result.sdata(2) = -1
+      result.sdata(3) = -1
+  else:
+    let tmp = int64(arg)
+    result.udata[0] = uint32(tmp and 0xffffffff)
+    result.sdata(1) = int32(tmp shr 32)
+    if arg < 0: # sign extend
+      result.sdata(2) = -1
+      result.sdata(3) = -1
+
+template isNegative(arg: Int128): bool =
+  arg.sdata(3) < 0
+
+template isNegative(arg: int32): bool =
+  arg < 0
+
+proc bitconcat(a,b: uint32): uint64 =
+  (uint64(a) shl 32) or uint64(b)
+
+proc bitsplit(a: uint64): (uint32,uint32) =
+  (cast[uint32](a shr 32), cast[uint32](a))
+
+proc toInt64*(arg: Int128): int64 =
+  if isNegative(arg):
+    assert(arg.sdata(3) == -1, "out of range")
+    assert(arg.sdata(2) == -1, "out of range")
+  else:
+    assert(arg.sdata(3) == 0, "out of range")
+    assert(arg.sdata(2) == 0, "out of range")
+
+  cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
+
+proc toInt64Checked*(arg: Int128; onError: int64): int64 =
+  if isNegative(arg):
+    if arg.sdata(3) != -1 or arg.sdata(2) != -1:
+      return onError
+  else:
+    if arg.sdata(3) != 0 or arg.sdata(2) != 0:
+      return onError
+  return cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
+
+proc toInt32*(arg: Int128): int32 =
+  if isNegative(arg):
+    assert(arg.sdata(3) == -1, "out of range")
+    assert(arg.sdata(2) == -1, "out of range")
+    assert(arg.sdata(1) == -1, "out of range")
+  else:
+    assert(arg.sdata(3) == 0, "out of range")
+    assert(arg.sdata(2) == 0, "out of range")
+    assert(arg.sdata(1) == 0, "out of range")
+
+  arg.sdata(0)
+
+proc toInt16*(arg: Int128): int16 =
+  if isNegative(arg):
+    assert(arg.sdata(3) == -1, "out of range")
+    assert(arg.sdata(2) == -1, "out of range")
+    assert(arg.sdata(1) == -1, "out of range")
+  else:
+    assert(arg.sdata(3) == 0, "out of range")
+    assert(arg.sdata(2) == 0, "out of range")
+    assert(arg.sdata(1) == 0, "out of range")
+
+  int16(arg.sdata(0))
+
+proc toInt8*(arg: Int128): int8 =
+  if isNegative(arg):
+    assert(arg.sdata(3) == -1, "out of range")
+    assert(arg.sdata(2) == -1, "out of range")
+    assert(arg.sdata(1) == -1, "out of range")
+  else:
+    assert(arg.sdata(3) == 0, "out of range")
+    assert(arg.sdata(2) == 0, "out of range")
+    assert(arg.sdata(1) == 0, "out of range")
+
+  int8(arg.sdata(0))
+
+proc toInt*(arg: Int128): int =
+  when sizeof(int) == 4:
+    cast[int](toInt32(arg))
+  else:
+    cast[int](toInt64(arg))
+
+proc toUInt64*(arg: Int128): uint64 =
+  assert(arg.udata[3] == 0)
+  assert(arg.udata[2] == 0)
+  bitconcat(arg.udata[1], arg.udata[0])
+
+proc toUInt32*(arg: Int128): uint32 =
+  assert(arg.udata[3] == 0)
+  assert(arg.udata[2] == 0)
+  assert(arg.udata[1] == 0)
+  arg.udata[0]
+
+proc toUInt16*(arg: Int128): uint16 =
+  assert(arg.udata[3] == 0)
+  assert(arg.udata[2] == 0)
+  assert(arg.udata[1] == 0)
+  uint16(arg.udata[0])
+
+proc toUInt8*(arg: Int128): uint8 =
+  assert(arg.udata[3] == 0)
+  assert(arg.udata[2] == 0)
+  assert(arg.udata[1] == 0)
+  uint8(arg.udata[0])
+
+proc toUInt*(arg: Int128): uint =
+  when sizeof(int) == 4:
+    cast[uint](toInt32(arg))
+  else:
+    cast[uint](toInt64(arg))
+
+proc castToInt64*(arg: Int128): int64 =
+  ## Conversion to int64 without range check.
+  cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
+
+proc castToUInt64*(arg: Int128): uint64 =
+  ## Conversion to uint64 without range check.
+  cast[uint64](bitconcat(arg.udata[1], arg.udata[0]))
+
+proc addToHex(result: var string; arg: uint32) =
+  for i in 0..<8:
+    let idx = (arg shr ((7-i) * 4)) and 0xf
+    result.add "0123456789abcdef"[idx]
+
+proc addToHex*(result: var string; arg: Int128) =
+  var i = 3
+  while i >= 0:
+    result.addToHex(arg.udata[i])
+    i -= 1
+
+proc toHex*(arg: Int128): string =
+  result.addToHex(arg)
+
+proc inc*(a: var Int128, y: uint32 = 1) =
+  let input = a
+  a.udata[0] += y
+  if unlikely(a.udata[0] < y):
+    a.udata[1].inc
+    if unlikely(a.udata[1] == 0):
+      a.udata[2].inc
+      if unlikely(a.udata[2] == 0):
+        a.udata[3].inc
+        doAssert(a.sdata(3) != low(int32), "overflow")
+
+proc cmp*(a,b: Int128): int =
+  let tmp1 = cmp(a.sdata(3), b.sdata(3))
+  if tmp1 != 0: return tmp1
+  let tmp2 = cmp(a.udata[2], b.udata[2])
+  if tmp2 != 0: return tmp2
+  let tmp3 = cmp(a.udata[1], b.udata[1])
+  if tmp3 != 0: return tmp3
+  let tmp4 = cmp(a.udata[0], b.udata[0])
+  return tmp4
+
+proc `<`*(a,b: Int128): bool =
+  cmp(a,b) < 0
+
+proc `<=`*(a,b: Int128): bool =
+  cmp(a,b) <= 0
+
+proc `==`*(a,b: Int128): bool =
+  if a.udata[0] != b.udata[0]: return false
+  if a.udata[1] != b.udata[1]: return false
+  if a.udata[2] != b.udata[2]: return false
+  if a.udata[3] != b.udata[3]: return false
+  return true
+
+proc inplaceBitnot(a: var Int128) =
+  a.udata[0] = not a.udata[0]
+  a.udata[1] = not a.udata[1]
+  a.udata[2] = not a.udata[2]
+  a.udata[3] = not a.udata[3]
+
+proc bitnot*(a: Int128): Int128 =
+  result.udata[0] = not a.udata[0]
+  result.udata[1] = not a.udata[1]
+  result.udata[2] = not a.udata[2]
+  result.udata[3] = not a.udata[3]
+
+proc bitand*(a,b: Int128): Int128 =
+  result.udata[0] = a.udata[0] and b.udata[0]
+  result.udata[1] = a.udata[1] and b.udata[1]
+  result.udata[2] = a.udata[2] and b.udata[2]
+  result.udata[3] = a.udata[3] and b.udata[3]
+
+proc bitor*(a,b: Int128): Int128 =
+  result.udata[0] = a.udata[0] or b.udata[0]
+  result.udata[1] = a.udata[1] or b.udata[1]
+  result.udata[2] = a.udata[2] or b.udata[2]
+  result.udata[3] = a.udata[3] or b.udata[3]
+
+proc bitxor*(a,b: Int128): Int128 =
+  result.udata[0] = a.udata[0] xor b.udata[0]
+  result.udata[1] = a.udata[1] xor b.udata[1]
+  result.udata[2] = a.udata[2] xor b.udata[2]
+  result.udata[3] = a.udata[3] xor b.udata[3]
+
+proc `shr`*(a: Int128, b: int): Int128 =
+  let b = b and 127
+  if b < 32:
+    result.sdata(3) = a.sdata(3) shr b
+    result.udata[2] = cast[uint32](bitconcat(a.udata[3], a.udata[2]) shr b)
+    result.udata[1] = cast[uint32](bitconcat(a.udata[2], a.udata[1]) shr b)
+    result.udata[0] = cast[uint32](bitconcat(a.udata[1], a.udata[0]) shr b)
+  elif b < 64:
+    if isNegative(a):
+      result.sdata(3) = -1
+    result.sdata(2) = a.sdata(3) shr (b and 31)
+    result.udata[1] = cast[uint32](bitconcat(a.udata[3], a.udata[2]) shr (b and 31))
+    result.udata[0] = cast[uint32](bitconcat(a.udata[2], a.udata[1]) shr (b and 31))
+  elif b < 96:
+    if isNegative(a):
+      result.sdata(3) = -1
+      result.sdata(2) = -1
+    result.sdata(1) = a.sdata(3) shr (b and 31)
+    result.udata[0] = cast[uint32](bitconcat(a.udata[3], a.udata[2]) shr (b and 31))
+  else: # b < 128
+    if isNegative(a):
+      result.sdata(3) = -1
+      result.sdata(2) = -1
+      result.sdata(1) = -1
+    result.sdata(0) = a.sdata(3) shr (b and 31)
+
+proc `shl`*(a: Int128, b: int): Int128 =
+  let b = b and 127
+  if b < 32:
+    result.udata[0] = a.udata[0] shl b
+    result.udata[1] = cast[uint32]((bitconcat(a.udata[1], a.udata[0]) shl b) shr 32)
+    result.udata[2] = cast[uint32]((bitconcat(a.udata[2], a.udata[1]) shl b) shr 32)
+    result.udata[3] = cast[uint32]((bitconcat(a.udata[3], a.udata[2]) shl b) shr 32)
+  elif b < 64:
+    result.udata[0] = 0
+    result.udata[1] = a.udata[0] shl (b and 31)
+    result.udata[2] = cast[uint32]((bitconcat(a.udata[1], a.udata[0]) shl (b and 31)) shr 32)
+    result.udata[3] = cast[uint32]((bitconcat(a.udata[2], a.udata[1]) shl (b and 31)) shr 32)
+  elif b < 96:
+    result.udata[0] = 0
+    result.udata[1] = 0
+    result.udata[2] = a.udata[0] shl (b and 31)
+    result.udata[3] = cast[uint32]((bitconcat(a.udata[1], a.udata[0]) shl (b and 31)) shr 32)
+  else:
+    result.udata[0] = 0
+    result.udata[1] = 0
+    result.udata[2] = 0
+    result.udata[3] = a.udata[0] shl (b and 31)
+
+proc `+`*(a,b: Int128): Int128 =
+  let tmp0 = uint64(a.udata[0]) + uint64(b.udata[0])
+  result.udata[0] = cast[uint32](tmp0)
+  let tmp1 = uint64(a.udata[1]) + uint64(b.udata[1]) + (tmp0 shr 32)
+  result.udata[1] = cast[uint32](tmp1)
+  let tmp2 = uint64(a.udata[2]) + uint64(b.udata[2]) + (tmp1 shr 32)
+  result.udata[2] = cast[uint32](tmp2)
+  let tmp3 = uint64(a.udata[3]) + uint64(b.udata[3]) + (tmp2 shr 32)
+  result.udata[3] = cast[uint32](tmp3)
+
+proc `+=`*(a: var Int128, b: Int128) =
+  a = a + b
+
+proc `-`*(a: Int128): Int128 =
+  result = bitnot(a)
+  result.inc
+
+proc `-`*(a,b: Int128): Int128 =
+  a + (-b)
+
+proc `-=`*(a: var Int128, b: Int128) =
+  a = a - b
+
+proc abs*(a: Int128): Int128 =
+  if isNegative(a):
+    -a
+  else:
+    a
+
+proc abs(a: int32): int =
+  if a < 0: -a else: a
+
+proc `*`(a: Int128, b: uint32): Int128 =
+  let tmp0 = uint64(a.udata[0]) * uint64(b)
+  let tmp1 = uint64(a.udata[1]) * uint64(b)
+  let tmp2 = uint64(a.udata[2]) * uint64(b)
+  let tmp3 = uint64(a.udata[3]) * uint64(b)
+
+  if unlikely(tmp3 > uint64(high(int32))):
+    assert(false, "overflow")
+
+  result.udata[0] = cast[uint32](tmp0)
+  result.udata[1] = cast[uint32](tmp1) + cast[uint32](tmp0 shr 32)
+  result.udata[2] = cast[uint32](tmp2) + cast[uint32](tmp1 shr 32)
+  result.udata[3] = cast[uint32](tmp3) + cast[uint32](tmp2 shr 32)
+
+proc `*`*(a: Int128, b: int32): Int128 =
+  result = a * cast[uint32](abs(b))
+  if b < 0:
+    result = -result
+
+proc `*=`*(a: var Int128, b: int32): Int128 =
+  result = result * b
+
+proc makeInt128(high,low: uint64): Int128 =
+  result.udata[0] = cast[uint32](low)
+  result.udata[1] = cast[uint32](low shr 32)
+  result.udata[2] = cast[uint32](high)
+  result.udata[3] = cast[uint32](high shr 32)
+
+proc high64(a: Int128): uint64 =
+  bitconcat(a.udata[3], a.udata[2])
+
+proc low64(a: Int128): uint64 =
+  bitconcat(a.udata[1], a.udata[0])
+
+proc `*`*(lhs,rhs: Int128): Int128 =
+  let
+    a = cast[uint64](lhs.udata[0])
+    b = cast[uint64](lhs.udata[1])
+    c = cast[uint64](lhs.udata[2])
+    d = cast[uint64](lhs.udata[3])
+
+    e = cast[uint64](rhs.udata[0])
+    f = cast[uint64](rhs.udata[1])
+    g = cast[uint64](rhs.udata[2])
+    h = cast[uint64](rhs.udata[3])
+
+
+  let a32 = cast[uint64](lhs.udata[1])
+  let a00 = cast[uint64](lhs.udata[0])
+  let b32 = cast[uint64](rhs.udata[1])
+  let b00 = cast[uint64](rhs.udata[0])
+
+  result = makeInt128(high64(lhs) * low64(rhs) + low64(lhs) * high64(rhs) + a32 * b32, a00 * b00)
+  result = result + toInt128(a32 * b00) shl 32
+  result = result + toInt128(a00 * b32) shl 32
+
+proc `*=`*(a: var Int128, b: Int128) =
+  a = a * b
+
+import bitops
+
+proc fastLog2*(a: Int128): int =
+  if a.udata[3] != 0:
+    return 96 + fastLog2(a.udata[3])
+  if a.udata[2] != 0:
+    return 64 + fastLog2(a.udata[2])
+  if a.udata[1] != 0:
+    return 32 + fastLog2(a.udata[1])
+  if a.udata[0] != 0:
+    return      fastLog2(a.udata[0])
+
+proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
+  assert(divisor != Zero)
+  let isNegativeA = isNegative(dividend)
+  let isNegativeB = isNegative(divisor)
+
+  var dividend = abs(dividend)
+  let divisor = abs(divisor)
+
+  if divisor > dividend:
+    result.quotient = Zero
+    if isNegativeA:
+      result.remainder = -dividend
+    else:
+      result.remainder = dividend
+    return
+
+  if divisor == dividend:
+    if isNegativeA xor isNegativeB:
+      result.quotient = NegOne
+    else:
+      result.quotient = One
+    result.remainder = Zero
+    return
+
+  var denominator = divisor
+  var quotient = Zero
+
+  # Left aligns the MSB of the denominator and the dividend.
+  let shift = fastLog2(dividend) - fastLog2(denominator)
+  denominator = denominator shl shift
+
+  # Uses shift-subtract algorithm to divide dividend by denominator. The
+  # remainder will be left in dividend.
+  for i in 0..shift:
+    quotient = quotient shl 1
+    if dividend >= denominator:
+      dividend = dividend - denominator
+      quotient = bitor(quotient, One)
+
+    denominator = denominator shr 1
+
+  if isNegativeA xor isNegativeB:
+    result.quotient = -quotient
+  else:
+    result.quotient = quotient
+  if isNegativeA:
+    result.remainder = -dividend
+  else:
+    result.remainder = dividend
+
+proc `div`*(a,b: Int128): Int128 =
+  let (a,b) = divMod(a,b)
+  return a
+
+proc `mod`*(a,b: Int128): Int128 =
+  let (a,b) = divMod(a,b)
+  return b
+
+proc addInt128*(result: var string; value: Int128) =
+  let initialSize = result.len
+  if value == Zero:
+    result.add "0"
+  elif value == low(Int128):
+    result.add "-170141183460469231731687303715884105728"
+  else:
+    let isNegative = isNegative(value)
+    var value = abs(value)
+    while value > Zero:
+      let (quot, rem) = divMod(value, Ten)
+      result.add "0123456789"[rem.toInt64]
+      value = quot
+    if isNegative:
+      result.add '-'
+
+    var i = initialSize
+    var j = high(result)
+    while i < j:
+      swap(result[i], result[j])
+      i += 1
+      j -= 1
+
+proc `$`*(a: Int128): string =
+  result.addInt128(a)
+
+proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =
+  assert(pos < arg.len)
+  assert(arg[pos] in {'-','0'..'9'})
+
+  var isNegative = false
+  var pos = pos
+  if arg[pos] == '-':
+    isNegative = true
+    pos += 1
+
+  result = Zero
+  while pos < arg.len and arg[pos] in '0'..'9':
+    result = result * Ten
+    result.inc(uint32(arg[pos]) - uint32('0'))
+    pos += 1
+
+  if isNegative:
+    result = -result
+
+# fluff
+
+proc `<`*(a: Int128, b: BiggestInt): bool =
+  cmp(a,toInt128(b)) < 0
+
+proc `<`*(a: BiggestInt, b: Int128): bool =
+  cmp(toInt128(a), b) < 0
+
+proc `<=`*(a: Int128, b: BiggestInt): bool =
+  cmp(a,toInt128(b)) <= 0
+
+proc `<=`*(a: BiggestInt, b: Int128): bool =
+  cmp(toInt128(a), b) <= 0
+
+proc `==`*(a: Int128, b: BiggestInt): bool =
+  a == toInt128(b)
+
+proc `==`*(a: BiggestInt, b: Int128): bool =
+  toInt128(a) == b
+
+proc `-`*(a: BiggestInt, b: Int128): Int128 =
+  toInt128(a) - b
+
+proc `-`*(a: Int128, b: BiggestInt): Int128 =
+  a - toInt128(b)
+
+proc `+`*(a: BiggestInt, b: Int128): Int128 =
+  toInt128(a) + b
+
+proc `+`*(a: Int128, b: BiggestInt): Int128 =
+  a + toInt128(b)
+
+proc toFloat64*(arg: Int128): float64 =
+  let isNegative = isNegative(arg)
+  let arg = abs(arg)
+
+  let a = float64(bitconcat(arg.udata[1], arg.udata[0]))
+  let b = float64(bitconcat(arg.udata[3], arg.udata[2]))
+
+  result = a + 18446744073709551616'f64 * b # a + 2^64 * b
+  if isNegative:
+    result = -result
+
+proc ldexp(x: float64, exp: cint): float64 {.importc: "ldexp", header: "<math.h>".}
+
+template bitor(a,b,c: Int128): Int128 = bitor(bitor(a,b), c)
+
+proc toInt128*(arg: float64): Int128 =
+  let isNegative = arg < 0
+  let v0 = ldexp(abs(arg), -100)
+  let w0 = uint64(trunc(v0))
+  let v1 = ldexp(v0 - float64(w0), 50)
+  let w1 = uint64(trunc(v1))
+  let v2 = ldexp(v1 - float64(w1), 50)
+  let w2 = uint64(trunc(v2))
+
+  let res = bitor(toInt128(w0) shl 100, toInt128(w1) shl 50, toInt128(w2))
+  if isNegative:
+    return -res
+  else:
+    return res
+
+proc maskUInt64*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0]
+  result.udata[1] = arg.udata[1]
+  result.udata[2] = 0
+  result.udata[3] = 0
+
+proc maskUInt32*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0]
+  result.udata[1] = 0
+  result.udata[2] = 0
+  result.udata[3] = 0
+
+proc maskUInt16*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0] and 0xffff
+  result.udata[1] = 0
+  result.udata[2] = 0
+  result.udata[3] = 0
+
+proc maskUInt8*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0] and 0xff
+  result.udata[1] = 0
+  result.udata[2] = 0
+  result.udata[3] = 0
+
+proc maskBytes*(arg: Int128, numbytes: int): Int128 {.noinit.} =
+  case numbytes
+  of 1:
+    return maskUInt8(arg)
+  of 2:
+    return maskUInt16(arg)
+  of 4:
+    return maskUInt32(arg)
+  of 8:
+    return maskUInt64(arg)
+  else:
+    assert(false, "masking only implemented for 1, 2, 4 and 8 bytes")
+
+
+
+
+when isMainModule:
+  let (a,b) = divMod(Ten,Ten)
+
+  doAssert $One == "1"
+  doAssert $Ten == "10"
+  doAssert $Zero == "0"
+  let c = parseDecimalInt128("12345678989876543210123456789")
+  doAssert $c == "12345678989876543210123456789"
+
+  var d : array[39, Int128]
+  d[0] =  parseDecimalInt128("1")
+  d[1] =  parseDecimalInt128("10")
+  d[2] =  parseDecimalInt128("100")
+  d[3] =  parseDecimalInt128("1000")
+  d[4] =  parseDecimalInt128("10000")
+  d[5] =  parseDecimalInt128("100000")
+  d[6] =  parseDecimalInt128("1000000")
+  d[7] =  parseDecimalInt128("10000000")
+  d[8] =  parseDecimalInt128("100000000")
+  d[9] =  parseDecimalInt128("1000000000")
+  d[10] = parseDecimalInt128("10000000000")
+  d[11] = parseDecimalInt128("100000000000")
+  d[12] = parseDecimalInt128("1000000000000")
+  d[13] = parseDecimalInt128("10000000000000")
+  d[14] = parseDecimalInt128("100000000000000")
+  d[15] = parseDecimalInt128("1000000000000000")
+  d[16] = parseDecimalInt128("10000000000000000")
+  d[17] = parseDecimalInt128("100000000000000000")
+  d[18] = parseDecimalInt128("1000000000000000000")
+  d[19] = parseDecimalInt128("10000000000000000000")
+  d[20] = parseDecimalInt128("100000000000000000000")
+  d[21] = parseDecimalInt128("1000000000000000000000")
+  d[22] = parseDecimalInt128("10000000000000000000000")
+  d[23] = parseDecimalInt128("100000000000000000000000")
+  d[24] = parseDecimalInt128("1000000000000000000000000")
+  d[25] = parseDecimalInt128("10000000000000000000000000")
+  d[26] = parseDecimalInt128("100000000000000000000000000")
+  d[27] = parseDecimalInt128("1000000000000000000000000000")
+  d[28] = parseDecimalInt128("10000000000000000000000000000")
+  d[29] = parseDecimalInt128("100000000000000000000000000000")
+  d[30] = parseDecimalInt128("1000000000000000000000000000000")
+  d[31] = parseDecimalInt128("10000000000000000000000000000000")
+  d[32] = parseDecimalInt128("100000000000000000000000000000000")
+  d[33] = parseDecimalInt128("1000000000000000000000000000000000")
+  d[34] = parseDecimalInt128("10000000000000000000000000000000000")
+  d[35] = parseDecimalInt128("100000000000000000000000000000000000")
+  d[36] = parseDecimalInt128("1000000000000000000000000000000000000")
+  d[37] = parseDecimalInt128("10000000000000000000000000000000000000")
+  d[38] = parseDecimalInt128("100000000000000000000000000000000000000")
+
+  for i in 0..<d.len:
+    for j in 0..<d.len:
+      doAssert(cmp(d[i], d[j]) == cmp(i,j))
+      if i + j < d.len:
+        doAssert d[i] * d[j] == d[i+j]
+      if i - j >= 0:
+        doAssert d[i] div d[j] == d[i-j]
+
+  var sum: Int128
+
+  for it in d:
+    sum += it
+
+  doAssert $sum == "111111111111111111111111111111111111111"
+
+  for it in d.mitems:
+    it = -it
+
+  for i in 0..<d.len:
+    for j in 0..<d.len:
+      doAssert(cmp(d[i], d[j]) == -cmp(i,j))
+      if i + j < d.len:
+        doAssert d[i] * d[j] == -d[i+j]
+      if i - j >= 0:
+        doAssert d[i] div d[j] == -d[i-j]
+
+  doAssert $high(Int128) == "170141183460469231731687303715884105727"
+  doAssert $low(Int128) == "-170141183460469231731687303715884105728"
+
+  var ma = 100'i64
+  var mb = 13
+
+  doAssert toInt128(ma) * toInt128(0) == toInt128(0)
+  doAssert toInt128(-ma) * toInt128(0) == toInt128(0)
+
+  # sign correctness
+  doAssert divMod(toInt128( ma),toInt128( mb)) == (toInt128( ma div  mb), toInt128( ma mod  mb))
+  doAssert divMod(toInt128(-ma),toInt128( mb)) == (toInt128(-ma div  mb), toInt128(-ma mod  mb))
+  doAssert divMod(toInt128( ma),toInt128(-mb)) == (toInt128( ma div -mb), toInt128( ma mod -mb))
+  doAssert divMod(toInt128(-ma),toInt128(-mb)) == (toInt128(-ma div -mb), toInt128(-ma mod -mb))
+
+  doAssert divMod(toInt128( mb),toInt128( mb)) == (toInt128( mb div  mb), toInt128( mb mod  mb))
+  doAssert divMod(toInt128(-mb),toInt128( mb)) == (toInt128(-mb div  mb), toInt128(-mb mod  mb))
+  doAssert divMod(toInt128( mb),toInt128(-mb)) == (toInt128( mb div -mb), toInt128( mb mod -mb))
+  doAssert divMod(toInt128(-mb),toInt128(-mb)) == (toInt128(-mb div -mb), toInt128(-mb mod -mb))
+
+  doAssert divMod(toInt128( mb),toInt128( ma)) == (toInt128( mb div  ma), toInt128( mb mod  ma))
+  doAssert divMod(toInt128(-mb),toInt128( ma)) == (toInt128(-mb div  ma), toInt128(-mb mod  ma))
+  doAssert divMod(toInt128( mb),toInt128(-ma)) == (toInt128( mb div -ma), toInt128( mb mod -ma))
+  doAssert divMod(toInt128(-mb),toInt128(-ma)) == (toInt128(-mb div -ma), toInt128(-mb mod -ma))
+
+  let e = parseDecimalInt128("70997106675279150998592376708984375")
+
+  let strArray = [
+    # toHex(e shr 0), toHex(e shr 1), toHex(e shr 2), toHex(e shr 3)
+    "000dac6d782d266a37300c32591eee37", "0006d636bc1693351b9806192c8f771b", "00036b1b5e0b499a8dcc030c9647bb8d", "0001b58daf05a4cd46e601864b23ddc6",
+    "0000dac6d782d266a37300c32591eee3", "00006d636bc1693351b9806192c8f771", "000036b1b5e0b499a8dcc030c9647bb8", "00001b58daf05a4cd46e601864b23ddc",
+    "00000dac6d782d266a37300c32591eee", "000006d636bc1693351b9806192c8f77", "0000036b1b5e0b499a8dcc030c9647bb", "000001b58daf05a4cd46e601864b23dd",
+    "000000dac6d782d266a37300c32591ee", "0000006d636bc1693351b9806192c8f7", "00000036b1b5e0b499a8dcc030c9647b", "0000001b58daf05a4cd46e601864b23d",
+    "0000000dac6d782d266a37300c32591e", "00000006d636bc1693351b9806192c8f", "000000036b1b5e0b499a8dcc030c9647", "00000001b58daf05a4cd46e601864b23",
+    "00000000dac6d782d266a37300c32591", "000000006d636bc1693351b9806192c8", "0000000036b1b5e0b499a8dcc030c964", "000000001b58daf05a4cd46e601864b2",
+    "000000000dac6d782d266a37300c3259", "0000000006d636bc1693351b9806192c", "00000000036b1b5e0b499a8dcc030c96", "0000000001b58daf05a4cd46e601864b",
+    "0000000000dac6d782d266a37300c325", "00000000006d636bc1693351b9806192", "000000000036b1b5e0b499a8dcc030c9", "00000000001b58daf05a4cd46e601864",
+    "00000000000dac6d782d266a37300c32", "000000000006d636bc1693351b980619", "0000000000036b1b5e0b499a8dcc030c", "000000000001b58daf05a4cd46e60186",
+    "000000000000dac6d782d266a37300c3", "0000000000006d636bc1693351b98061", "00000000000036b1b5e0b499a8dcc030", "0000000000001b58daf05a4cd46e6018",
+    "0000000000000dac6d782d266a37300c", "00000000000006d636bc1693351b9806", "000000000000036b1b5e0b499a8dcc03", "00000000000001b58daf05a4cd46e601",
+    "00000000000000dac6d782d266a37300", "000000000000006d636bc1693351b980", "0000000000000036b1b5e0b499a8dcc0", "000000000000001b58daf05a4cd46e60",
+    "000000000000000dac6d782d266a3730", "0000000000000006d636bc1693351b98", "00000000000000036b1b5e0b499a8dcc", "0000000000000001b58daf05a4cd46e6",
+    "0000000000000000dac6d782d266a373", "00000000000000006d636bc1693351b9", "000000000000000036b1b5e0b499a8dc", "00000000000000001b58daf05a4cd46e",
+    "00000000000000000dac6d782d266a37", "000000000000000006d636bc1693351b", "0000000000000000036b1b5e0b499a8d", "000000000000000001b58daf05a4cd46",
+    "000000000000000000dac6d782d266a3", "0000000000000000006d636bc1693351", "00000000000000000036b1b5e0b499a8", "0000000000000000001b58daf05a4cd4",
+    "0000000000000000000dac6d782d266a", "00000000000000000006d636bc169335", "000000000000000000036b1b5e0b499a", "00000000000000000001b58daf05a4cd",
+    "00000000000000000000dac6d782d266", "000000000000000000006d636bc16933", "0000000000000000000036b1b5e0b499", "000000000000000000001b58daf05a4c",
+    "000000000000000000000dac6d782d26", "0000000000000000000006d636bc1693", "00000000000000000000036b1b5e0b49", "0000000000000000000001b58daf05a4",
+    "0000000000000000000000dac6d782d2", "00000000000000000000006d636bc169", "000000000000000000000036b1b5e0b4", "00000000000000000000001b58daf05a",
+    "00000000000000000000000dac6d782d", "000000000000000000000006d636bc16", "0000000000000000000000036b1b5e0b", "000000000000000000000001b58daf05",
+    "000000000000000000000000dac6d782", "0000000000000000000000006d636bc1", "00000000000000000000000036b1b5e0", "0000000000000000000000001b58daf0",
+    "0000000000000000000000000dac6d78", "00000000000000000000000006d636bc", "000000000000000000000000036b1b5e", "00000000000000000000000001b58daf",
+    "00000000000000000000000000dac6d7", "000000000000000000000000006d636b", "0000000000000000000000000036b1b5", "000000000000000000000000001b58da",
+    "000000000000000000000000000dac6d", "0000000000000000000000000006d636", "00000000000000000000000000036b1b", "0000000000000000000000000001b58d",
+    "0000000000000000000000000000dac6", "00000000000000000000000000006d63", "000000000000000000000000000036b1", "00000000000000000000000000001b58",
+    "00000000000000000000000000000dac", "000000000000000000000000000006d6", "0000000000000000000000000000036b", "000000000000000000000000000001b5",
+    "000000000000000000000000000000da", "0000000000000000000000000000006d", "00000000000000000000000000000036", "0000000000000000000000000000001b",
+    "0000000000000000000000000000000d", "00000000000000000000000000000006", "00000000000000000000000000000003", "00000000000000000000000000000001",
+    "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000",
+    "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000",
+    "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000",
+  ]
+
+  for i in 0 ..< 128:
+    let str1 = toHex(e shr i)
+    let str2 = strArray[i]
+    doAssert str1 == str2
+
+  let strArray2 = [
+    "000dac6d782d266a37300c32591eee37", "001b58daf05a4cd46e601864b23ddc6e", "0036b1b5e0b499a8dcc030c9647bb8dc", "006d636bc1693351b9806192c8f771b8",
+    "00dac6d782d266a37300c32591eee370", "01b58daf05a4cd46e601864b23ddc6e0", "036b1b5e0b499a8dcc030c9647bb8dc0", "06d636bc1693351b9806192c8f771b80",
+    "0dac6d782d266a37300c32591eee3700", "1b58daf05a4cd46e601864b23ddc6e00", "36b1b5e0b499a8dcc030c9647bb8dc00", "6d636bc1693351b9806192c8f771b800",
+    "dac6d782d266a37300c32591eee37000", "b58daf05a4cd46e601864b23ddc6e000", "6b1b5e0b499a8dcc030c9647bb8dc000", "d636bc1693351b9806192c8f771b8000",
+    "ac6d782d266a37300c32591eee370000", "58daf05a4cd46e601864b23ddc6e0000", "b1b5e0b499a8dcc030c9647bb8dc0000", "636bc1693351b9806192c8f771b80000",
+    "c6d782d266a37300c32591eee3700000", "8daf05a4cd46e601864b23ddc6e00000", "1b5e0b499a8dcc030c9647bb8dc00000", "36bc1693351b9806192c8f771b800000",
+    "6d782d266a37300c32591eee37000000", "daf05a4cd46e601864b23ddc6e000000", "b5e0b499a8dcc030c9647bb8dc000000", "6bc1693351b9806192c8f771b8000000",
+    "d782d266a37300c32591eee370000000", "af05a4cd46e601864b23ddc6e0000000", "5e0b499a8dcc030c9647bb8dc0000000", "bc1693351b9806192c8f771b80000000",
+    "782d266a37300c32591eee3700000000", "f05a4cd46e601864b23ddc6e00000000", "e0b499a8dcc030c9647bb8dc00000000", "c1693351b9806192c8f771b800000000",
+    "82d266a37300c32591eee37000000000", "05a4cd46e601864b23ddc6e000000000", "0b499a8dcc030c9647bb8dc000000000", "1693351b9806192c8f771b8000000000",
+    "2d266a37300c32591eee370000000000", "5a4cd46e601864b23ddc6e0000000000", "b499a8dcc030c9647bb8dc0000000000", "693351b9806192c8f771b80000000000",
+    "d266a37300c32591eee3700000000000", "a4cd46e601864b23ddc6e00000000000", "499a8dcc030c9647bb8dc00000000000", "93351b9806192c8f771b800000000000",
+    "266a37300c32591eee37000000000000", "4cd46e601864b23ddc6e000000000000", "99a8dcc030c9647bb8dc000000000000", "3351b9806192c8f771b8000000000000",
+    "66a37300c32591eee370000000000000", "cd46e601864b23ddc6e0000000000000", "9a8dcc030c9647bb8dc0000000000000", "351b9806192c8f771b80000000000000",
+    "6a37300c32591eee3700000000000000", "d46e601864b23ddc6e00000000000000", "a8dcc030c9647bb8dc00000000000000", "51b9806192c8f771b800000000000000",
+    "a37300c32591eee37000000000000000", "46e601864b23ddc6e000000000000000", "8dcc030c9647bb8dc000000000000000", "1b9806192c8f771b8000000000000000",
+    "37300c32591eee370000000000000000", "6e601864b23ddc6e0000000000000000", "dcc030c9647bb8dc0000000000000000", "b9806192c8f771b80000000000000000",
+    "7300c32591eee3700000000000000000", "e601864b23ddc6e00000000000000000", "cc030c9647bb8dc00000000000000000", "9806192c8f771b800000000000000000",
+    "300c32591eee37000000000000000000", "601864b23ddc6e000000000000000000", "c030c9647bb8dc000000000000000000", "806192c8f771b8000000000000000000",
+    "00c32591eee370000000000000000000", "01864b23ddc6e0000000000000000000", "030c9647bb8dc0000000000000000000", "06192c8f771b80000000000000000000",
+    "0c32591eee3700000000000000000000", "1864b23ddc6e00000000000000000000", "30c9647bb8dc00000000000000000000", "6192c8f771b800000000000000000000",
+    "c32591eee37000000000000000000000", "864b23ddc6e000000000000000000000", "0c9647bb8dc000000000000000000000", "192c8f771b8000000000000000000000",
+    "32591eee370000000000000000000000", "64b23ddc6e0000000000000000000000", "c9647bb8dc0000000000000000000000", "92c8f771b80000000000000000000000",
+    "2591eee3700000000000000000000000", "4b23ddc6e00000000000000000000000", "9647bb8dc00000000000000000000000", "2c8f771b800000000000000000000000",
+    "591eee37000000000000000000000000", "b23ddc6e000000000000000000000000", "647bb8dc000000000000000000000000", "c8f771b8000000000000000000000000",
+    "91eee370000000000000000000000000", "23ddc6e0000000000000000000000000", "47bb8dc0000000000000000000000000", "8f771b80000000000000000000000000",
+    "1eee3700000000000000000000000000", "3ddc6e00000000000000000000000000", "7bb8dc00000000000000000000000000", "f771b800000000000000000000000000",
+    "eee37000000000000000000000000000", "ddc6e000000000000000000000000000", "bb8dc000000000000000000000000000", "771b8000000000000000000000000000",
+    "ee370000000000000000000000000000", "dc6e0000000000000000000000000000", "b8dc0000000000000000000000000000", "71b80000000000000000000000000000",
+    "e3700000000000000000000000000000", "c6e00000000000000000000000000000", "8dc00000000000000000000000000000", "1b800000000000000000000000000000",
+    "37000000000000000000000000000000", "6e000000000000000000000000000000", "dc000000000000000000000000000000", "b8000000000000000000000000000000",
+    "70000000000000000000000000000000", "e0000000000000000000000000000000", "c0000000000000000000000000000000", "80000000000000000000000000000000",
+  ]
+
+  for i in 0 ..< 128:
+    let str1 = toHex(e shl i)
+    let str2 = strArray2[i]
+    doAssert str1 == str2

compiler/jstypes.nim

@@ -7,28 +7,30 @@
 #    distribution, for details about the copyright.
 #
 
+# included from jsgen.nim
+
 ## Type info generation for the JS backend.
 
+proc rope(arg: Int128): Rope = rope($arg)
+
 proc genTypeInfo(p: PProc, typ: PType): Rope
 proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
   var
     s, u: Rope
-    length: int
     field: PSym
     b: PNode
   result = nil
   case n.kind
   of nkRecList:
-    length = sonsLen(n)
-    if length == 1:
-      result = genObjectFields(p, typ, n.sons[0])
+    if n.len == 1:
+      result = genObjectFields(p, typ, n[0])
     else:
       s = nil
-      for i in countup(0, length - 1):
-        if i > 0: add(s, ", \L")
-        add(s, genObjectFields(p, typ, n.sons[i]))
+      for i in 0..<n.len:
+        if i > 0: s.add(", \L")
+        s.add(genObjectFields(p, typ, n[i]))
       result = ("{kind: 2, len: $1, offset: 0, " &
-          "typ: null, name: null, sons: [$2]}") % [rope(length), s]
+          "typ: null, name: null, sons: [$2]}") % [rope(n.len), s]
   of nkSym:
     field = n.sym
     s = genTypeInfo(p, field.typ)
@@ -37,29 +39,28 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
                    [mangleName(p.module, field), s,
                     makeJSString(field.name.s)]
   of nkRecCase:
-    length = sonsLen(n)
-    if (n.sons[0].kind != nkSym): internalError(p.config, n.info, "genObjectFields")
-    field = n.sons[0].sym
+    if (n[0].kind != nkSym): internalError(p.config, n.info, "genObjectFields")
+    field = n[0].sym
     s = genTypeInfo(p, field.typ)
-    for i in countup(1, length - 1):
-      b = n.sons[i]           # branch
+    for i in 1..<n.len:
+      b = n[i]           # branch
       u = nil
       case b.kind
       of nkOfBranch:
-        if sonsLen(b) < 2:
+        if b.len < 2:
           internalError(p.config, b.info, "genObjectFields; nkOfBranch broken")
-        for j in countup(0, sonsLen(b) - 2):
-          if u != nil: add(u, ", ")
-          if b.sons[j].kind == nkRange:
-            addf(u, "[$1, $2]", [rope(getOrdValue(b.sons[j].sons[0])),
-                                 rope(getOrdValue(b.sons[j].sons[1]))])
+        for j in 0..<b.len - 1:
+          if u != nil: u.add(", ")
+          if b[j].kind == nkRange:
+            u.addf("[$1, $2]", [rope(getOrdValue(b[j][0])),
+                                 rope(getOrdValue(b[j][1]))])
           else:
-            add(u, rope(getOrdValue(b.sons[j])))
+            u.add(rope(getOrdValue(b[j])))
       of nkElse:
         u = rope(lengthOrd(p.config, field.typ))
       else: internalError(p.config, n.info, "genObjectFields(nkRecCase)")
-      if result != nil: add(result, ", \L")
-      addf(result, "[setConstr($1), $2]",
+      if result != nil: result.add(", \L")
+      result.addf("[setConstr($1), $2]",
            [u, genObjectFields(p, typ, lastSon(b))])
     result = ("{kind: 3, offset: \"$1\", len: $3, " &
         "typ: $2, name: $4, sons: [$5]}") % [
@@ -68,27 +69,27 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
   else: internalError(p.config, n.info, "genObjectFields")
 
 proc objHasTypeField(t: PType): bool {.inline.} =
-  tfInheritable in t.flags or t.sons[0] != nil
+  tfInheritable in t.flags or t[0] != nil
 
 proc genObjectInfo(p: PProc, typ: PType, name: Rope) =
   let kind = if objHasTypeField(typ): tyObject else: tyTuple
   var s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
            "finalizer: null};$n") % [name, rope(ord(kind))]
   prepend(p.g.typeInfo, s)
-  addf(p.g.typeInfo, "var NNI$1 = $2;$n",
+  p.g.typeInfo.addf("var NNI$1 = $2;$n",
        [rope(typ.id), genObjectFields(p, typ, typ.n)])
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
-  if (typ.kind == tyObject) and (typ.sons[0] != nil):
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [name, genTypeInfo(p, typ.sons[0].skipTypes(skipPtrs))])
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
+  if (typ.kind == tyObject) and (typ[0] != nil):
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [name, genTypeInfo(p, typ[0].skipTypes(skipPtrs))])
 
 proc genTupleFields(p: PProc, typ: PType): Rope =
   var s: Rope = nil
-  for i in 0 ..< typ.len:
-    if i > 0: add(s, ", \L")
+  for i in 0..<typ.len:
+    if i > 0: s.add(", \L")
     s.addf("{kind: 1, offset: \"Field$1\", len: 0, " &
            "typ: $2, name: \"Field$1\", sons: null}",
-           [i.rope, genTypeInfo(p, typ.sons[i])])
+           [i.rope, genTypeInfo(p, typ[i])])
   result = ("{kind: 2, len: $1, offset: 0, " &
             "typ: null, name: null, sons: [$2]}") % [rope(typ.len), s]
 
@@ -96,38 +97,37 @@ proc genTupleInfo(p: PProc, typ: PType, name: Rope) =
   var s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
            "finalizer: null};$n") % [name, rope(ord(typ.kind))]
   prepend(p.g.typeInfo, s)
-  addf(p.g.typeInfo, "var NNI$1 = $2;$n",
+  p.g.typeInfo.addf("var NNI$1 = $2;$n",
        [rope(typ.id), genTupleFields(p, typ)])
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
 
 proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
-  let length = sonsLen(typ.n)
   var s: Rope = nil
-  for i in countup(0, length - 1):
-    if (typ.n.sons[i].kind != nkSym): internalError(p.config, typ.n.info, "genEnumInfo")
-    let field = typ.n.sons[i].sym
-    if i > 0: add(s, ", \L")
+  for i in 0..<typ.n.len:
+    if (typ.n[i].kind != nkSym): internalError(p.config, typ.n.info, "genEnumInfo")
+    let field = typ.n[i].sym
+    if i > 0: s.add(", \L")
     let extName = if field.ast == nil: field.name.s else: field.ast.strVal
-    addf(s, "\"$1\": {kind: 1, offset: $1, typ: $2, name: $3, len: 0, sons: null}",
+    s.addf("\"$1\": {kind: 1, offset: $1, typ: $2, name: $3, len: 0, sons: null}",
          [rope(field.position), name, makeJSString(extName)])
   var n = ("var NNI$1 = {kind: 2, offset: 0, typ: null, " &
-      "name: null, len: $2, sons: {$3}};$n") % [rope(typ.id), rope(length), s]
+      "name: null, len: $2, sons: {$3}};$n") % [rope(typ.id), rope(typ.n.len), s]
   s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
        "finalizer: null};$n") % [name, rope(ord(typ.kind))]
   prepend(p.g.typeInfo, s)
-  add(p.g.typeInfo, n)
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
-  if typ.sons[0] != nil:
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [name, genTypeInfo(p, typ.sons[0])])
+  p.g.typeInfo.add(n)
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
+  if typ[0] != nil:
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [name, genTypeInfo(p, typ[0])])
 
 proc genTypeInfo(p: PProc, typ: PType): Rope =
-  let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink})
+  let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink, tyOwned})
   result = "NTI$1" % [rope(t.id)]
   if containsOrIncl(p.g.typeInfoGenerated, t.id): return
   case t.kind
   of tyDistinct:
-    result = genTypeInfo(p, t.sons[0])
+    result = genTypeInfo(p, t[0])
   of tyPointer, tyProc, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
@@ -138,15 +138,15 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-    addf(p.g.typeInfo, "$1.base = $2;$n",
+    p.g.typeInfo.addf("$1.base = $2;$n",
          [result, genTypeInfo(p, t.lastSon)])
   of tyArray:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [result, genTypeInfo(p, t.sons[1])])
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [result, genTypeInfo(p, t[1])])
   of tyEnum: genEnumInfo(p, t, result)
   of tyObject: genObjectInfo(p, t, result)
   of tyTuple: genTupleInfo(p, t, result)

compiler/lambdalifting.nim

@@ -10,8 +10,9 @@
 # This file implements lambda lifting for the transformator.
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs,
-  idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos
+  intsets, strutils, options, ast, astalgo, msgs,
+  idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos,
+  transf, liftdestructors
 
 discard """
   The basic approach is that captured vars need to be put on the heap and
@@ -127,8 +128,8 @@ proc newCall(a: PSym, b: PNode): PNode =
 
 proc createClosureIterStateType*(g: ModuleGraph; iter: PSym): PType =
   var n = newNodeI(nkRange, iter.info)
-  addSon(n, newIntNode(nkIntLit, -1))
-  addSon(n, newIntNode(nkIntLit, 0))
+  n.add newIntNode(nkIntLit, -1)
+  n.add newIntNode(nkIntLit, 0)
   result = newType(tyRange, iter)
   result.n = n
   var intType = nilOrSysInt(g)
@@ -147,27 +148,27 @@ proc createEnvObj(g: ModuleGraph; owner: PSym; info: TLineInfo): PType =
 
 proc getClosureIterResult*(g: ModuleGraph; iter: PSym): PSym =
   if resultPos < iter.ast.len:
-    result = iter.ast.sons[resultPos].sym
+    result = iter.ast[resultPos].sym
   else:
     # XXX a bit hacky:
     result = newSym(skResult, getIdent(g.cache, ":result"), iter, iter.info, {})
-    result.typ = iter.typ.sons[0]
+    result.typ = iter.typ[0]
     incl(result.flags, sfUsed)
     iter.ast.add newSymNode(result)
 
 proc addHiddenParam(routine: PSym, param: PSym) =
   assert param.kind == skParam
-  var params = routine.ast.sons[paramsPos]
+  var params = routine.ast[paramsPos]
   # -1 is correct here as param.position is 0 based but we have at position 0
   # some nkEffect node:
   param.position = routine.typ.n.len-1
-  addSon(params, newSymNode(param))
+  params.add newSymNode(param)
   #incl(routine.typ.flags, tfCapturesEnv)
   assert sfFromGeneric in param.flags
   #echo "produced environment: ", param.id, " for ", routine.id
 
 proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
-  let params = routine.ast.sons[paramsPos]
+  let params = routine.ast[paramsPos]
   let hidden = lastSon(params)
   if hidden.kind == nkSym and hidden.sym.kind == skParam and hidden.sym.name.s == paramName:
     result = hidden.sym
@@ -178,7 +179,7 @@ proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
     result = routine
 
 proc getEnvParam*(routine: PSym): PSym =
-  let params = routine.ast.sons[paramsPos]
+  let params = routine.ast[paramsPos]
   let hidden = lastSon(params)
   if hidden.kind == nkSym and hidden.sym.name.s == paramName:
     result = hidden.sym
@@ -205,8 +206,8 @@ proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode =
   # which is however the only case when we generate an assignment in the first
   # place.
   result = newNodeI(nkAsgn, info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
 
 proc makeClosure*(g: ModuleGraph; prc: PSym; env: PNode; info: TLineInfo): PNode =
   result = newNodeIT(nkClosure, info, prc.typ)
@@ -217,6 +218,10 @@ proc makeClosure*(g: ModuleGraph; prc: PSym; env: PNode; info: TLineInfo): PNode
     if env.skipConv.kind == nkClosure:
       localError(g.config, info, "internal error: taking closure of closure")
     result.add(env)
+  #if isClosureIterator(result.typ):
+  createTypeBoundOps(g, nil, result.typ, info)
+  if tfHasAsgn in result.typ.flags or optSeqDestructors in g.config.globalOptions:
+    prc.flags.incl sfInjectDestructors
 
 proc interestingIterVar(s: PSym): bool {.inline.} =
   # XXX optimization: Only lift the variable if it lives across
@@ -232,6 +237,12 @@ proc liftingHarmful(conf: ConfigRef; owner: PSym): bool {.inline.} =
   let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
   result = conf.cmd == cmdCompileToJS and not isCompileTime
 
+proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; owner: PSym) =
+  createTypeBoundOps(g, nil, refType.lastSon, info)
+  createTypeBoundOps(g, nil, refType, info)
+  if tfHasAsgn in refType.flags or optSeqDestructors in g.config.globalOptions:
+    owner.flags.incl sfInjectDestructors
+
 proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env))
   if liftingHarmful(g.config, owner): return n
@@ -244,7 +255,7 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   var env: PNode
   if owner.isIterator:
     let it = getHiddenParam(g, owner)
-    addUniqueField(it.typ.sons[0], hp, g.cache)
+    addUniqueField(it.typ.skipTypes({tyOwned})[0], hp, g.cache)
     env = indirectAccess(newSymNode(it), hp, hp.info)
   else:
     let e = newSym(skLet, iter.name, owner, n.info)
@@ -256,11 +267,12 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
     result.add(v)
   # add 'new' statement:
   result.add newCall(getSysSym(g, n.info, "internalNew"), env)
+  createTypeBoundOpsLL(g, env.typ, n.info, owner)
   result.add makeClosure(g, iter, env, n.info)
 
 proc freshVarForClosureIter*(g: ModuleGraph; s, owner: PSym): PNode =
   let envParam = getHiddenParam(g, owner)
-  let obj = envParam.typ.lastSon
+  let obj = envParam.typ.skipTypes({tyOwned, tyRef, tyPtr})
   addField(obj, s, g.cache)
 
   var access = newSymNode(envParam)
@@ -277,8 +289,9 @@ proc freshVarForClosureIter*(g: ModuleGraph; s, owner: PSym): PNode =
 proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
   let s = n.sym
   if illegalCapture(s):
-    localError(g.config, n.info, "illegal capture '$1' of type <$2> which is declared here: $3" %
-      [s.name.s, typeToString(s.typ), g.config$s.info])
+    localError(g.config, n.info,
+      ("'$1' is of type <$2> which cannot be captured as it would violate memory" &
+       " safety, declared here: $3") % [s.name.s, typeToString(s.typ), g.config$s.info])
   elif owner.typ.callConv notin {ccClosure, ccDefault}:
     localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
       [s.name.s, owner.name.s, CallingConvToStr[owner.typ.callConv]])
@@ -318,21 +331,51 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
     rawAddSon(result, obj)
     c.ownerToType[owner.id] = result
 
+proc asOwnedRef(c: DetectionPass; t: PType): PType =
+  if optOwnedRefs in c.graph.config.globalOptions:
+    assert t.kind == tyRef
+    result = newType(tyOwned, t.owner)
+    result.flags.incl tfHasOwned
+    result.rawAddSon t
+  else:
+    result = t
+
+proc getEnvTypeForOwnerUp(c: var DetectionPass; owner: PSym;
+                          info: TLineInfo): PType =
+  var r = c.getEnvTypeForOwner(owner, info)
+  result = newType(tyPtr, owner)
+  rawAddSon(result, r.skipTypes({tyOwned, tyRef, tyPtr}))
+
 proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
   let refObj = c.getEnvTypeForOwner(dest, info) # getHiddenParam(dest).typ
-  let obj = refObj.lastSon
-  let fieldType = c.getEnvTypeForOwner(dep, info) #getHiddenParam(dep).typ
+  let obj = refObj.skipTypes({tyOwned, tyRef, tyPtr})
+  # The assumption here is that gcDestructors means we cannot deal
+  # with cycles properly, so it's better to produce a weak ref (=ptr) here.
+  # This seems to be generally correct but since it's a bit risky it's disabled
+  # for now.
+  # XXX This is wrong for the 'hamming' test, so remove this logic again.
+  let fieldType = if isDefined(c.graph.config, "nimCycleBreaker"):
+                    c.getEnvTypeForOwnerUp(dep, info) #getHiddenParam(dep).typ
+                  else:
+                    c.getEnvTypeForOwner(dep, info)
   if refObj == fieldType:
     localError(c.graph.config, dep.info, "internal error: invalid up reference computed")
 
   let upIdent = getIdent(c.graph.cache, upName)
   let upField = lookupInRecord(obj.n, upIdent)
   if upField != nil:
-    if upField.typ != fieldType:
+    if upField.typ.skipTypes({tyOwned, tyRef, tyPtr}) != fieldType.skipTypes({tyOwned, tyRef, tyPtr}):
       localError(c.graph.config, dep.info, "internal error: up references do not agree")
+
+    when false:
+      if c.graph.config.selectedGC == gcDestructors and sfCursor notin upField.flags:
+        localError(c.graph.config, dep.info, "internal error: up reference is not a .cursor")
   else:
     let result = newSym(skField, upIdent, obj.owner, obj.owner.info)
     result.typ = fieldType
+    when false:
+      if c.graph.config.selectedGC == gcDestructors:
+        result.flags.incl sfCursor
     rawAddField(obj, result)
 
 discard """
@@ -389,7 +432,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     if innerProc:
       if s.isIterator: c.somethingToDo = true
       if not c.processed.containsOrIncl(s.id):
-        detectCapturedVars(s.getBody, s, c)
+        let body = transformBody(c.graph, s, cache = true)
+        detectCapturedVars(body, s, c)
     let ow = s.skipGenericOwner
     if ow == owner:
       if owner.isIterator:
@@ -397,18 +441,15 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
         addClosureParam(c, owner, n.info)
         if interestingIterVar(s):
           if not c.capturedVars.containsOrIncl(s.id):
-            let obj = getHiddenParam(c.graph, owner).typ.lastSon
-            #let obj = c.getEnvTypeForOwner(s.owner).lastSon
+            let obj = getHiddenParam(c.graph, owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
+            #let obj = c.getEnvTypeForOwner(s.owner).skipTypes({tyOwned, tyRef, tyPtr})
 
             if s.name.id == getIdent(c.graph.cache, ":state").id:
               obj.n[0].sym.id = -s.id
             else:
               addField(obj, s, c.graph.cache)
-      # but always return because the rest of the proc is only relevant when
-      # ow != owner:
-      return
     # direct or indirect dependency:
-    if (innerProc and s.typ.callConv == ccClosure) or interestingVar(s):
+    elif (innerProc and s.typ.callConv == ccClosure) or interestingVar(s):
       discard """
         proc outer() =
           var x: int
@@ -426,8 +467,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       #echo "capturing ", n.info
       # variable 's' is actually captured:
       if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id):
-        let obj = c.getEnvTypeForOwner(ow, n.info).lastSon
-        #getHiddenParam(owner).typ.lastSon
+        let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef, tyPtr})
+        #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
         addField(obj, s, c.graph.cache)
       # create required upFields:
       var w = owner.skipGenericOwner
@@ -451,17 +492,14 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
           createUpField(c, w, up, n.info)
           w = up
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit,
-     nkTemplateDef, nkTypeSection:
+     nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef,
+     nkConverterDef, nkMacroDef, nkFuncDef, nkCommentStmt, nkTypeOfExpr:
     discard
-  of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef:
-    discard
-  of nkLambdaKinds, nkIteratorDef, nkFuncDef:
+  of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil:
       detectCapturedVars(n[namePos], owner, c)
   of nkReturnStmt:
-    if n[0].kind in {nkAsgn, nkFastAsgn}:
-      detectCapturedVars(n[0].sons[1], owner, c)
-    else: assert n[0].kind == nkEmpty
+    detectCapturedVars(n[0], owner, c)
   else:
     for i in 0..<n.len:
       detectCapturedVars(n[i], owner, c)
@@ -471,6 +509,7 @@ type
     processed: IntSet
     envVars: Table[int, PNode]
     inContainer: int
+    unownedEnvVars: Table[int, PNode] # only required for --newruntime
 
 proc initLiftingPass(fn: PSym): LiftingPass =
   result.processed = initIntSet()
@@ -484,7 +523,7 @@ proc accessViaEnvParam(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   if not envParam.isNil:
     var access = newSymNode(envParam)
     while true:
-      let obj = access.typ.sons[0]
+      let obj = access.typ[0]
       assert obj.kind == tyObject
       let field = getFieldFromObj(obj, s)
       if field != nil:
@@ -495,56 +534,75 @@ proc accessViaEnvParam(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   localError(g.config, n.info, "internal error: environment misses: " & s.name.s)
   result = n
 
-proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType): PNode =
-  var v = newSym(skVar, getIdent(cache, envName), owner, owner.info)
-  incl(v.flags, sfShadowed)
+proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo): PNode =
+  var v = newSym(skVar, getIdent(cache, envName), owner, info)
+  v.flags = {sfShadowed, sfGeneratedOp}
   v.typ = typ
   result = newSymNode(v)
   when false:
     if owner.kind == skIterator and owner.typ.callConv == ccClosure:
       let it = getHiddenParam(owner)
-      addUniqueField(it.typ.sons[0], v)
+      addUniqueField(it.typ[0], v)
       result = indirectAccess(newSymNode(it), v, v.info)
     else:
       result = newSymNode(v)
 
 proc setupEnvVar(owner: PSym; d: DetectionPass;
-                 c: var LiftingPass): PNode =
+                 c: var LiftingPass; info: TLineInfo): PNode =
   if owner.isIterator:
     return getHiddenParam(d.graph, owner).newSymNode
-  result = c.envvars.getOrDefault(owner.id)
+  result = c.envVars.getOrDefault(owner.id)
   if result.isNil:
     let envVarType = d.ownerToType.getOrDefault(owner.id)
     if envVarType.isNil:
       localError d.graph.config, owner.info, "internal error: could not determine closure type"
-    result = newEnvVar(d.graph.cache, owner, envVarType)
+    result = newEnvVar(d.graph.cache, owner, asOwnedRef(d, envVarType), info)
     c.envVars[owner.id] = result
+    if optOwnedRefs in d.graph.config.globalOptions:
+      var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), owner, info)
+      v.flags = {sfShadowed, sfGeneratedOp}
+      v.typ = envVarType
+      c.unownedEnvVars[owner.id] = newSymNode(v)
 
 proc getUpViaParam(g: ModuleGraph; owner: PSym): PNode =
   let p = getHiddenParam(g, owner)
   result = p.newSymNode
   if owner.isIterator:
-    let upField = lookupInRecord(p.typ.lastSon.n, getIdent(g.cache, upName))
+    let upField = lookupInRecord(p.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(g.cache, upName))
     if upField == nil:
       localError(g.config, owner.info, "could not find up reference for closure iter")
     else:
       result = rawIndirectAccess(result, upField, p.info)
 
 proc rawClosureCreation(owner: PSym;
-                        d: DetectionPass; c: var LiftingPass): PNode =
+                        d: DetectionPass; c: var LiftingPass;
+                        info: TLineInfo): PNode =
   result = newNodeI(nkStmtList, owner.info)
 
   var env: PNode
   if owner.isIterator:
     env = getHiddenParam(d.graph, owner).newSymNode
   else:
-    env = setupEnvVar(owner, d, c)
+    env = setupEnvVar(owner, d, c, info)
     if env.kind == nkSym:
       var v = newNodeI(nkVarSection, env.info)
       addVar(v, env)
       result.add(v)
+      if optOwnedRefs in d.graph.config.globalOptions:
+        let unowned = c.unownedEnvVars[owner.id]
+        assert unowned != nil
+        addVar(v, unowned)
+
     # add 'new' statement:
     result.add(newCall(getSysSym(d.graph, env.info, "internalNew"), env))
+    if optOwnedRefs in d.graph.config.globalOptions:
+      let unowned = c.unownedEnvVars[owner.id]
+      assert unowned != nil
+      let env2 = copyTree(env)
+      env2.typ = unowned.typ
+      result.add newAsgnStmt(unowned, env2, env.info)
+      createTypeBoundOpsLL(d.graph, unowned.typ, env.info, owner)
+
     # add assignment statements for captured parameters:
     for i in 1..<owner.typ.n.len:
       let local = owner.typ.n[i].sym
@@ -552,11 +610,14 @@ proc rawClosureCreation(owner: PSym;
         let fieldAccess = indirectAccess(env, local, env.info)
         # add ``env.param = param``
         result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info))
+        createTypeBoundOps(d.graph, nil, fieldAccess.typ, env.info)
+        if tfHasAsgn in fieldAccess.typ.flags or optSeqDestructors in d.graph.config.globalOptions:
+          owner.flags.incl sfInjectDestructors
 
-  let upField = lookupInRecord(env.typ.lastSon.n, getIdent(d.graph.cache, upName))
+  let upField = lookupInRecord(env.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
     let up = getUpViaParam(d.graph, owner)
-    if up != nil and upField.typ == up.typ:
+    if up != nil and upField.typ.skipTypes({tyOwned, tyRef, tyPtr}) == up.typ.skipTypes({tyOwned, tyRef, tyPtr}):
       result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info),
                  up, env.info))
     #elif oldenv != nil and oldenv.typ == upField.typ:
@@ -564,6 +625,18 @@ proc rawClosureCreation(owner: PSym;
     #             oldenv, env.info))
     else:
       localError(d.graph.config, env.info, "internal error: cannot create up reference")
+  # we are not in the sem'check phase anymore! so pass 'nil' for the PContext
+  # and hope for the best:
+  createTypeBoundOpsLL(d.graph, env.typ, owner.info, owner)
+
+proc finishClosureCreation(owner: PSym; d: DetectionPass; c: LiftingPass;
+                           info: TLineInfo; res: PNode) =
+  if optOwnedRefs in d.graph.config.globalOptions:
+    let unowned = c.unownedEnvVars[owner.id]
+    assert unowned != nil
+    let nilLit = newNodeIT(nkNilLit, info, unowned.typ)
+    res.add newAsgnStmt(unowned, nilLit, info)
+    createTypeBoundOpsLL(d.graph, unowned.typ, info, owner)
 
 proc closureCreationForIter(iter: PNode;
                             d: DetectionPass; c: var LiftingPass): PNode =
@@ -571,11 +644,11 @@ proc closureCreationForIter(iter: PNode;
   let owner = iter.sym.skipGenericOwner
   var v = newSym(skVar, getIdent(d.graph.cache, envName), owner, iter.info)
   incl(v.flags, sfShadowed)
-  v.typ = getHiddenParam(d.graph, iter.sym).typ
+  v.typ = asOwnedRef(d, getHiddenParam(d.graph, iter.sym).typ)
   var vnode: PNode
   if owner.isIterator:
     let it = getHiddenParam(d.graph, owner)
-    addUniqueField(it.typ.sons[0], v, d.graph.cache)
+    addUniqueField(it.typ.skipTypes({tyOwned, tyRef, tyPtr}), v, d.graph.cache)
     vnode = indirectAccess(newSymNode(it), v, v.info)
   else:
     vnode = v.newSymNode
@@ -583,11 +656,12 @@ proc closureCreationForIter(iter: PNode;
     addVar(vs, vnode)
     result.add(vs)
   result.add(newCall(getSysSym(d.graph, iter.info, "internalNew"), vnode))
+  createTypeBoundOpsLL(d.graph, vnode.typ, iter.info, owner)
 
-  let upField = lookupInRecord(v.typ.lastSon.n, getIdent(d.graph.cache, upName))
+  let upField = lookupInRecord(v.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
-    let u = setupEnvVar(owner, d, c)
-    if u.typ == upField.typ:
+    let u = setupEnvVar(owner, d, c, iter.info)
+    if u.typ.skipTypes({tyOwned, tyRef, tyPtr}) == upField.typ.skipTypes({tyOwned, tyRef, tyPtr}):
       result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info),
                  u, iter.info))
     else:
@@ -596,8 +670,10 @@ proc closureCreationForIter(iter: PNode;
 
 proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
                      c: var LiftingPass): PNode =
-  let access = setupEnvVar(owner, d, c)
-  let obj = access.typ.sons[0]
+  var access = setupEnvVar(owner, d, c, n.info)
+  if optOwnedRefs in d.graph.config.globalOptions:
+    access = c.unownedEnvVars[owner.id]
+  let obj = access.typ.skipTypes({tyOwned, tyRef, tyPtr})
   let field = getFieldFromObj(obj, n.sym)
   if field != nil:
     result = rawIndirectAccess(access, field, n.info)
@@ -606,7 +682,7 @@ proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
     result = n
 
 proc getStateField*(g: ModuleGraph; owner: PSym): PSym =
-  getHiddenParam(g, owner).typ.sons[0].n.sons[0].sym
+  getHiddenParam(g, owner).typ.skipTypes({tyOwned, tyRef, tyPtr}).n[0].sym
 
 proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
                       c: var LiftingPass): PNode
@@ -622,7 +698,7 @@ proc symToClosure(n: PNode; owner: PSym; d: DetectionPass;
     result = closureCreationForIter(n, d, c)
   elif s.skipGenericOwner == owner:
     # direct dependency, so use the outer's env variable:
-    result = makeClosure(d.graph, s, setupEnvVar(owner, d, c), n.info)
+    result = makeClosure(d.graph, s, setupEnvVar(owner, d, c, n.info), n.info)
   else:
     let available = getHiddenParam(d.graph, owner)
     let wanted = getHiddenParam(d.graph, s).typ
@@ -631,7 +707,7 @@ proc symToClosure(n: PNode; owner: PSym; d: DetectionPass;
     while true:
       if access.typ == wanted:
         return makeClosure(d.graph, s, access, n.info)
-      let obj = access.typ.sons[0]
+      let obj = access.typ.skipTypes({tyOwned, tyRef, tyPtr})
       let upField = lookupInRecord(obj.n, getIdent(d.graph.cache, upName))
       if upField == nil:
         localError(d.graph.config, n.info, "internal error: no environment found")
@@ -650,14 +726,18 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         #  echo renderTree(s.getBody, {renderIds})
         let oldInContainer = c.inContainer
         c.inContainer = 0
-        var body = liftCapturedVars(s.getBody, s, d, c)
-        if c.envvars.getOrDefault(s.id).isNil:
-          s.ast.sons[bodyPos] = body
+        var body = transformBody(d.graph, s, cache = false)
+        body = liftCapturedVars(body, s, d, c)
+        if c.envVars.getOrDefault(s.id).isNil:
+          s.transformedBody = body
         else:
-          s.ast.sons[bodyPos] = newTree(nkStmtList, rawClosureCreation(s, d, c), body)
+          s.transformedBody = newTree(nkStmtList, rawClosureCreation(s, d, c, n.info), body)
+          finishClosureCreation(s, d, c, n.info, s.transformedBody)
         c.inContainer = oldInContainer
+
       if s.typ.callConv == ccClosure:
         result = symToClosure(n, owner, d, c)
+
     elif s.id in d.capturedVars:
       if s.owner != owner:
         result = accessViaEnvParam(d.graph, n, owner)
@@ -666,20 +746,19 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
       else:
         result = accessViaEnvVar(n, owner, d, c)
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom,
-     nkTemplateDef, nkTypeSection:
-    discard
-  of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef:
+     nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef, nkConverterDef,
+     nkMacroDef, nkFuncDef:
     discard
   of nkClosure:
     if n[1].kind == nkNilLit:
-      n.sons[0] = liftCapturedVars(n[0], owner, d, c)
-      let x = n.sons[0].skipConv
+      n[0] = liftCapturedVars(n[0], owner, d, c)
+      let x = n[0].skipConv
       if x.kind == nkClosure:
         #localError(n.info, "internal error: closure to closure created")
         # now we know better, so patch it:
-        n.sons[0] = x.sons[0]
-        n.sons[1] = x.sons[1]
-  of nkLambdaKinds, nkIteratorDef, nkFuncDef:
+        n[0] = x[0]
+        n[1] = x[1]
+  of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil and n[namePos].kind == nkSym:
       let oldInContainer = c.inContainer
       c.inContainer = 0
@@ -689,44 +768,65 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
       c.inContainer = oldInContainer
   of nkHiddenStdConv:
     if n.len == 2:
-      n.sons[1] = liftCapturedVars(n[1], owner, d, c)
+      n[1] = liftCapturedVars(n[1], owner, d, c)
       if n[1].kind == nkClosure: result = n[1]
   of nkReturnStmt:
     if n[0].kind in {nkAsgn, nkFastAsgn}:
       # we have a `result = result` expression produced by the closure
       # transform, let's not touch the LHS in order to make the lifting pass
       # correct when `result` is lifted
-      n[0].sons[1] = liftCapturedVars(n[0].sons[1], owner, d, c)
-    else: assert n[0].kind == nkEmpty
+      n[0][1] = liftCapturedVars(n[0][1], owner, d, c)
+    else:
+      n[0] = liftCapturedVars(n[0], owner, d, c)
+  of nkTypeOfExpr:
+    result = n
   else:
     if owner.isIterator:
       if nfLL in n.flags:
         # special case 'when nimVm' due to bug #3636:
-        n.sons[1] = liftCapturedVars(n[1], owner, d, c)
+        n[1] = liftCapturedVars(n[1], owner, d, c)
         return
 
     let inContainer = n.kind in {nkObjConstr, nkBracket}
     if inContainer: inc c.inContainer
     for i in 0..<n.len:
-      n.sons[i] = liftCapturedVars(n[i], owner, d, c)
+      n[i] = liftCapturedVars(n[i], owner, d, c)
     if inContainer: dec c.inContainer
 
 # ------------------ old stuff -------------------------------------------
 
 proc semCaptureSym*(s, owner: PSym) =
+  discard """
+    proc outer() =
+      var x: int
+      proc inner() =
+        proc innerInner() =
+          echo x
+        innerInner()
+      inner()
+    # inner() takes a closure too!
+  """
+  proc propagateClosure(start, last: PSym) =
+    var o = start
+    while o != nil and o.kind != skModule:
+      if o == last: break
+      o.typ.callConv = ccClosure
+      o = o.skipGenericOwner
+
   if interestingVar(s) and s.kind != skResult:
     if owner.typ != nil and not isGenericRoutine(owner):
       # XXX: is this really safe?
       # if we capture a var from another generic routine,
       # it won't be consider captured.
       var o = owner.skipGenericOwner
-      while o.kind != skModule and o != nil:
+      while o != nil and o.kind != skModule:
         if s.owner == o:
           if owner.typ.callConv in {ccClosure, ccDefault} or owner.kind == skIterator:
             owner.typ.callConv = ccClosure
+            propagateClosure(owner.skipGenericOwner, s.owner)
           else:
             discard "do not produce an error here, but later"
-          #echo "computing .closure for ", owner.name.s, " ", owner.info, " because of ", s.name.s
+          #echo "computing .closure for ", owner.name.s, " because of ", s.name.s
         o = o.skipGenericOwner
     # since the analysis is not entirely correct, we don't set 'tfCapturesEnv'
     # here
@@ -737,12 +837,12 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType): PNo
   # pretend 'fn' is a closure iterator for the analysis:
   let oldKind = fn.kind
   let oldCC = fn.typ.callConv
-  fn.kind = skIterator
+  fn.transitionRoutineSymKind(skIterator)
   fn.typ.callConv = ccClosure
   d.ownerToType[fn.id] = ptrType
   detectCapturedVars(body, fn, d)
   result = liftCapturedVars(body, fn, d, c)
-  fn.kind = oldKind
+  fn.transitionRoutineSymKind(oldKind)
   fn.typ.callConv = oldCC
 
 proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PNode =
@@ -769,8 +869,9 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PN
       var c = initLiftingPass(fn)
       result = liftCapturedVars(body, fn, d, c)
       # echo renderTree(result, {renderIds})
-      if c.envvars.getOrDefault(fn.id) != nil:
-        result = newTree(nkStmtList, rawClosureCreation(fn, d, c), result)
+      if c.envVars.getOrDefault(fn.id) != nil:
+        result = newTree(nkStmtList, rawClosureCreation(fn, d, c, body.info), result)
+        finishClosureCreation(fn, d, c, body.info, result)
     else:
       result = body
     #if fn.name.s == "get2":
@@ -813,11 +914,10 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
         ...
     """
   if liftingHarmful(g.config, owner): return body
-  var L = body.len
-  if not (body.kind == nkForStmt and body[L-2].kind in nkCallKinds):
+  if not (body.kind == nkForStmt and body[^2].kind in nkCallKinds):
     localError(g.config, body.info, "ignored invalid for loop")
     return body
-  var call = body[L-2]
+  var call = body[^2]
 
   result = newNodeI(nkStmtList, body.info)
 
@@ -838,37 +938,39 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
     result.add(v)
     # add 'new' statement:
     result.add(newCall(getSysSym(g, env.info, "internalNew"), env.newSymNode))
+    createTypeBoundOpsLL(g, env.typ, body.info, owner)
+
   elif op.kind == nkStmtListExpr:
     let closure = op.lastSon
     if closure.kind == nkClosure:
-      call.sons[0] = closure
-      for i in 0 .. op.len-2:
+      call[0] = closure
+      for i in 0..<op.len-1:
         result.add op[i]
 
   var loopBody = newNodeI(nkStmtList, body.info, 3)
   var whileLoop = newNodeI(nkWhileStmt, body.info, 2)
-  whileLoop.sons[0] = newIntTypeNode(nkIntLit, 1, getSysType(g, body.info, tyBool))
-  whileLoop.sons[1] = loopBody
+  whileLoop[0] = newIntTypeNode(1, getSysType(g, body.info, tyBool))
+  whileLoop[1] = loopBody
   result.add whileLoop
 
   # setup loopBody:
   # gather vars in a tuple:
   var v2 = newNodeI(nkLetSection, body.info)
-  var vpart = newNodeI(if L == 3: nkIdentDefs else: nkVarTuple, body.info)
-  for i in 0 .. L-3:
+  var vpart = newNodeI(if body.len == 3: nkIdentDefs else: nkVarTuple, body.info)
+  for i in 0..<body.len-2:
     if body[i].kind == nkSym:
-      body[i].sym.kind = skLet
-    addSon(vpart, body[i])
+      body[i].sym.transitionToLet()
+    vpart.add body[i]
 
-  addSon(vpart, newNodeI(nkEmpty, body.info)) # no explicit type
+  vpart.add newNodeI(nkEmpty, body.info) # no explicit type
   if not env.isNil:
-    call.sons[0] = makeClosure(g, call.sons[0].sym, env.newSymNode, body.info)
-  addSon(vpart, call)
-  addSon(v2, vpart)
+    call[0] = makeClosure(g, call[0].sym, env.newSymNode, body.info)
+  vpart.add call
+  v2.add vpart
 
-  loopBody.sons[0] = v2
+  loopBody[0] = v2
   var bs = newNodeI(nkBreakState, body.info)
-  bs.addSon(call.sons[0])
+  bs.add call[0]
 
   let ibs = newNodeI(nkIfStmt, body.info)
   let elifBranch = newNodeI(nkElifBranch, body.info)
@@ -880,5 +982,5 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
   elifBranch.add(br)
   ibs.add(elifBranch)
 
-  loopBody.sons[1] = ibs
-  loopBody.sons[2] = body[L-1]
+  loopBody[1] = ibs
+  loopBody[2] = body[^1]

compiler/layouter.nim

@@ -9,13 +9,10 @@
 
 ## Layouter for nimpretty.
 
-import idents, lexer, lineinfos, llstream, options, msgs, strutils,
-  pathutils
-from os import changeFileExt
+import idents, lexer, lineinfos, llstream, options, msgs, strutils, pathutils
 
 const
-  MaxLineLen = 80
-  LineCommentColumn = 30
+  MinLineLen = 15
 
 type
   SplitKind = enum
@@ -24,41 +21,86 @@ type
   SemicolonKind = enum
     detectSemicolonKind, useSemicolon, dontTouch
 
+  LayoutToken* = enum
+    ltSpaces,
+    ltCrucialNewline, ## a semantically crucial newline (indentation!)
+    ltSplittingNewline, ## newline used for splitting up long
+                        ## expressions (like after a comma or a binary operator)
+    ltTab,
+    ltOptionalNewline, ## optional newline introduced by nimpretty
+    ltComment, ltLit, ltKeyword, ltExportMarker, ltIdent,
+    ltOther, ltOpr, ltSomeParLe, ltSomeParRi,
+    ltBeginSection, ltEndSection
+
   Emitter* = object
     config: ConfigRef
     fid: FileIndex
     lastTok: TTokType
-    inquote: bool
+    inquote, lastTokWasTerse: bool
     semicolons: SemicolonKind
-    col, lastLineNumber, lineSpan, indentLevel, indWidth: int
-    nested: int
+    col, lastLineNumber, lineSpan, indentLevel, indWidth*, inSection: int
+    keepIndents*: int
     doIndentMore*: int
-    content: string
+    kinds*: seq[LayoutToken]
+    tokens*: seq[string]
     indentStack: seq[int]
     fixedUntil: int # marks where we must not go in the content
     altSplitPos: array[SplitKind, int] # alternative split positions
+    maxLineLen*: int
 
 proc openEmitter*(em: var Emitter, cache: IdentCache;
                   config: ConfigRef, fileIdx: FileIndex) =
-  let fullPath = Absolutefile config.toFullPath(fileIdx)
-  em.indWidth = getIndentWidth(fileIdx, llStreamOpen(fullPath, fmRead),
-                               cache, config)
-  if em.indWidth == 0: em.indWidth = 2
+  let fullPath = AbsoluteFile config.toFullPath(fileIdx)
+  if em.indWidth == 0:
+    em.indWidth = getIndentWidth(fileIdx, llStreamOpen(fullPath, fmRead),
+                                cache, config)
+    if em.indWidth == 0: em.indWidth = 2
   em.config = config
   em.fid = fileIdx
   em.lastTok = tkInvalid
   em.inquote = false
   em.col = 0
-  em.content = newStringOfCap(16_000)
   em.indentStack = newSeqOfCap[int](30)
   em.indentStack.add 0
+  em.lastLineNumber = 1
 
-proc closeEmitter*(em: var Emitter) =
-  var f = llStreamOpen(em.config.outFile, fmWrite)
-  if f == nil:
-    rawMessage(em.config, errGenerated, "cannot open file: " & em.config.outFile.string)
-  f.llStreamWrite em.content
-  llStreamClose(f)
+proc computeMax(em: Emitter; pos: int): int =
+  var p = pos
+  var extraSpace = 0
+  result = 0
+  while p < em.tokens.len and em.kinds[p] != ltEndSection:
+    var lhs = 0
+    var lineLen = 0
+    var foundTab = false
+    while p < em.tokens.len and em.kinds[p] != ltEndSection:
+      if em.kinds[p] in {ltCrucialNewline, ltSplittingNewline}:
+        if foundTab and lineLen <= em.maxLineLen:
+          result = max(result, lhs + extraSpace)
+        inc p
+        break
+      if em.kinds[p] == ltTab:
+        extraSpace = if em.kinds[p-1] == ltSpaces: 0 else: 1
+        foundTab = true
+      else:
+        if not foundTab:
+          inc lhs, em.tokens[p].len
+        inc lineLen, em.tokens[p].len
+      inc p
+
+proc computeRhs(em: Emitter; pos: int): int =
+  var p = pos
+  result = 0
+  while p < em.tokens.len and em.kinds[p] notin {ltCrucialNewline, ltSplittingNewline}:
+    inc result, em.tokens[p].len
+    inc p
+
+proc isLongEnough(lineLen, startPos, endPos: int): bool =
+  result = lineLen > MinLineLen and endPos > startPos + 4
+
+proc findNewline(em: Emitter; p, lineLen: var int) =
+  while p < em.tokens.len and em.kinds[p] notin {ltCrucialNewline, ltSplittingNewline}:
+    inc lineLen, em.tokens[p].len
+    inc p
 
 proc countNewlines(s: string): int =
   result = 0
@@ -72,78 +114,348 @@ proc calcCol(em: var Emitter; s: string) =
     dec i
     inc em.col
 
-template wr(x) =
-  em.content.add x
-  inc em.col, x.len
+proc optionalIsGood(em: var Emitter; pos, currentLen: int): bool =
+  let ourIndent = em.tokens[pos].len
+  var p = pos+1
+  var lineLen = 0
+  em.findNewline(p, lineLen)
+  if p == pos+1: # optionalNewline followed by another newline
+    result = false
+  elif em.kinds[p-1] == ltComment and currentLen+lineLen < em.maxLineLen+MinLineLen:
+    result = false
+  elif p+1 < em.tokens.len and em.kinds[p+1] == ltSpaces and
+      em.kinds[p-1] == ltOptionalNewline:
+    if em.tokens[p+1].len == ourIndent:
+      # concatenate lines with the same indententation
+      var nlPos = p
+      var lineLenTotal = lineLen
+      inc p
+      em.findNewline(p, lineLenTotal)
+      if isLongEnough(lineLenTotal, nlPos, p):
+        em.kinds[nlPos] = ltOptionalNewline
+        if em.kinds[nlPos+1] == ltSpaces:
+          # inhibit extra spaces when concatenating two lines
+          em.tokens[nlPos+1] = if em.tokens[nlPos-2] == ",": " " else: ""
+      result = true
+    elif em.tokens[p+1].len < ourIndent:
+      result = isLongEnough(lineLen, pos, p)
+  elif em.kinds[pos+1] in {ltOther, ltSomeParLe, ltSomeParRi}: # note: pos+1, not p+1
+    result = false
+  else:
+    result = isLongEnough(lineLen, pos, p)
+
+proc lenOfNextTokens(em: Emitter; pos: int): int =
+  result = 0
+  for i in 1..<em.tokens.len-pos:
+    if em.kinds[pos+i] in {ltCrucialNewline, ltSplittingNewline, ltOptionalNewline}: break
+    inc result, em.tokens[pos+i].len
+
+proc guidingInd(em: Emitter; pos: int): int =
+  var i = pos - 1
+  while i >= 0 and em.kinds[i] != ltSomeParLe:
+    dec i
+  while i+1 <= em.kinds.high and em.kinds[i] != ltSomeParRi:
+    if em.kinds[i] == ltSplittingNewline and em.kinds[i+1] == ltSpaces:
+      return em.tokens[i+1].len
+    inc i
+  result = -1
+
+proc renderTokens*(em: var Emitter): string =
+  ## Render Emitter tokens to a string of code
+  template defaultCase() =
+    content.add em.tokens[i]
+    inc lineLen, em.tokens[i].len
+  var content = newStringOfCap(16_000)
+  var maxLhs = 0
+  var lineLen = 0
+  var lineBegin = 0
+  var openPars = 0
+  var i = 0
+  while i <= em.tokens.high:
+    when defined(debug):
+      echo (token: em.tokens[i], kind: em.kinds[i])
+    case em.kinds[i]
+    of ltBeginSection:
+      maxLhs = computeMax(em, lineBegin)
+    of ltEndSection:
+      maxLhs = 0
+      lineBegin = i+1
+    of ltTab:
+      if i >= 2 and em.kinds[i-2] in {ltCrucialNewline, ltSplittingNewline} and
+          em.kinds[i-1] in {ltCrucialNewline, ltSplittingNewline, ltSpaces}:
+        # a previous section has ended
+        maxLhs = 0
+
+      if maxLhs == 0:
+        if em.kinds[i-1] != ltSpaces:
+          content.add em.tokens[i]
+          inc lineLen, em.tokens[i].len
+      else:
+        # pick the shorter indentation token:
+        var spaces = maxLhs - lineLen
+        if spaces < em.tokens[i].len or computeRhs(em, i+1)+maxLhs <= em.maxLineLen+MinLineLen:
+          if spaces <= 0 and content[^1] notin {' ', '\L'}: spaces = 1
+          for j in 1..spaces: content.add ' '
+          inc lineLen, spaces
+        else:
+          content.add em.tokens[i]
+          inc lineLen, em.tokens[i].len
+    of ltCrucialNewline, ltSplittingNewline:
+      content.add em.tokens[i]
+      lineLen = 0
+      lineBegin = i+1
+    of ltOptionalNewline:
+      let totalLineLen = lineLen + lenOfNextTokens(em, i)
+      if totalLineLen > em.maxLineLen and optionalIsGood(em, i, lineLen):
+        if i-1 >= 0 and em.kinds[i-1] == ltSpaces:
+          let spaces = em.tokens[i-1].len
+          content.setLen(content.len - spaces)
+        content.add "\L"
+        let guide = if openPars > 0: guidingInd(em, i) else: -1
+        if guide >= 0:
+          content.add repeat(' ', guide)
+          lineLen = guide
+        else:
+          content.add em.tokens[i]
+          lineLen = em.tokens[i].len
+        lineBegin = i+1
+        if i+1 < em.kinds.len and em.kinds[i+1] == ltSpaces:
+          # inhibit extra spaces at the start of a new line
+          inc i
+    of ltLit:
+      let lineSpan = countNewlines(em.tokens[i])
+      if lineSpan > 0:
+        em.calcCol(em.tokens[i])
+        lineLen = em.col
+      else:
+        inc lineLen, em.tokens[i].len
+      content.add em.tokens[i]
+    of ltSomeParLe:
+      inc openPars
+      defaultCase()
+    of ltSomeParRi:
+      doAssert openPars > 0
+      dec openPars
+      defaultCase()
+    else:
+      defaultCase()
+    inc i
+
+  return content
+
+proc writeOut*(em: Emitter, content: string)  =
+  ## Write to disk
+  let outFile = em.config.absOutFile
+  if fileExists(outFile) and readFile(outFile.string) == content:
+    discard "do nothing, see #9499"
+    return
+  var f = llStreamOpen(outFile, fmWrite)
+  if f == nil:
+    rawMessage(em.config, errGenerated, "cannot open file: " & outFile.string)
+    return
+  f.llStreamWrite content
+  llStreamClose(f)
+
+proc closeEmitter*(em: var Emitter) =
+  ## Renders emitter tokens and write to a file
+  let content = renderTokens(em)
+  em.writeOut(content)
+
+proc wr(em: var Emitter; x: string; lt: LayoutToken) =
+  em.tokens.add x
+  em.kinds.add lt
+  inc em.col, x.len
+  assert em.tokens.len == em.kinds.len
+
+proc wrNewline(em: var Emitter; kind = ltCrucialNewline) =
+  em.tokens.add "\L"
+  em.kinds.add kind
+  em.col = 0
+
+proc newlineWasSplitting*(em: var Emitter) =
+  if em.kinds.len >= 3 and em.kinds[^3] == ltCrucialNewline:
+    em.kinds[^3] = ltSplittingNewline
+
+#[
+Splitting newlines can occur:
+- after commas, semicolon, '[', '('.
+- after binary operators, '='.
+- after ':' type
+
+We only need parser support for the "after type" case.
+]#
+
+proc wrSpaces(em: var Emitter; spaces: int) =
+  if spaces > 0:
+    wr(em, strutils.repeat(' ', spaces), ltSpaces)
+
+proc wrSpace(em: var Emitter) =
+  wr(em, " ", ltSpaces)
+
+proc wrTab(em: var Emitter) =
+  wr(em, " ", ltTab)
+
+proc beginSection*(em: var Emitter) =
+  let pos = max(0, em.tokens.len-2)
+  em.tokens.insert "", pos
+  em.kinds.insert ltBeginSection, pos
+  inc em.inSection
+
+#wr(em, "", ltBeginSection)
+proc endSection*(em: var Emitter) =
+  em.tokens.insert "", em.tokens.len-2
+  em.kinds.insert ltEndSection, em.kinds.len-2
+  dec em.inSection
+
+#wr(em, "", ltEndSection)
+
+proc removeSpaces(em: var Emitter) =
+  while em.kinds.len > 0 and em.kinds[^1] == ltSpaces:
+    let tokenLen = em.tokens[^1].len
+    setLen(em.tokens, em.tokens.len-1)
+    setLen(em.kinds, em.kinds.len-1)
+    dec em.col, tokenLen
 
-template goodCol(col): bool = col in 40..MaxLineLen
 
 const
   openPars = {tkParLe, tkParDotLe,
-              tkBracketLe, tkBracketLeColon, tkCurlyDotLe,
-              tkCurlyLe}
-  splitters = openPars + {tkComma, tkSemicolon}
+              tkBracketLe, tkBracketDotLe, tkBracketLeColon,
+              tkCurlyDotLe, tkCurlyLe}
+  closedPars = {tkParRi, tkParDotRi,
+                tkBracketRi, tkBracketDotRi,
+                tkCurlyDotRi, tkCurlyRi}
+
+  splitters = openPars + {tkComma, tkSemiColon} # do not add 'tkColon' here!
   oprSet = {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
             tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor}
 
-template rememberSplit(kind) =
-  if goodCol(em.col):
-    em.altSplitPos[kind] = em.content.len
+template goodCol(col): bool = col >= em.maxLineLen div 2
 
 template moreIndent(em): int =
-  (if em.doIndentMore > 0: em.indWidth*2 else: em.indWidth)
+  if em.doIndentMore > 0: em.indWidth*2 else: em.indWidth
 
-proc softLinebreak(em: var Emitter, lit: string) =
-  # XXX Use an algorithm that is outlined here:
-  # https://llvm.org/devmtg/2013-04/jasper-slides.pdf
-  # +2 because we blindly assume a comma or ' &' might follow
-  if not em.inquote and em.col+lit.len+2 >= MaxLineLen:
-    if em.lastTok in splitters:
-      while em.content.len > 0 and em.content[em.content.high] == ' ':
-        setLen(em.content, em.content.len-1)
-      wr("\L")
-      em.col = 0
-      for i in 1..em.indentLevel+moreIndent(em): wr(" ")
+template rememberSplit(kind) =
+  if goodCol(em.col) and not em.inquote:
+    let spaces = em.indentLevel+moreIndent(em)
+    if spaces < em.col and spaces > 0:
+      wr(em, strutils.repeat(' ', spaces), ltOptionalNewline)
+    #em.altSplitPos[kind] = em.tokens.len
+
+proc emitMultilineComment(em: var Emitter, lit: string, col: int; dontIndent: bool) =
+  # re-align every line in the multi-line comment:
+  var i = 0
+  var lastIndent = if em.keepIndents > 0: em.indentLevel else: em.indentStack[^1]
+  var b = 0
+  var dontIndent = dontIndent
+  var hasEmptyLine = false
+  for commentLine in splitLines(lit):
+    if i == 0 and (commentLine.endsWith("\\") or commentLine.endsWith("[")):
+      dontIndent = true
+      wr em, commentLine, ltComment
+    elif dontIndent:
+      if i > 0: wrNewline em
+      wr em, commentLine, ltComment
     else:
-      # search backwards for a good split position:
-      for a in em.altSplitPos:
-        if a > em.fixedUntil:
-          var spaces = 0
-          while a+spaces < em.content.len and em.content[a+spaces] == ' ':
-            inc spaces
-          if spaces > 0: delete(em.content, a, a+spaces-1)
-          let ws = "\L" & repeat(' ',em.indentLevel+moreIndent(em))
-          em.col = em.content.len - a
-          em.content.insert(ws, a)
-          break
+      let stripped = commentLine.strip()
+      if i == 0:
+        if em.kinds.len > 0 and em.kinds[^1] != ltTab:
+          wr(em, "", ltTab)
+      elif stripped.len == 0:
+        wrNewline em
+        hasEmptyLine = true
+      else:
+        var a = 0
+        while a < commentLine.len and commentLine[a] == ' ': inc a
+
+        if a > lastIndent:
+          b += em.indWidth
+          lastIndent = a
+        elif a < lastIndent:
+          b -= em.indWidth
+          lastIndent = a
+        wrNewline em
+        if not hasEmptyLine or col + b < 15:
+          if col + b > 0:
+            wr(em, repeat(' ', col+b), ltTab)
+          else:
+            wr(em, "", ltTab)
+        else:
+          wr(em, repeat(' ', a), ltSpaces)
+      wr em, stripped, ltComment
+    inc i
+
+proc lastChar(s: string): char =
+  result = if s.len > 0: s[s.high] else: '\0'
+
+proc endsInWhite(em: Emitter): bool =
+  var i = em.tokens.len-1
+  while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection}: dec(i)
+  result = if i >= 0: em.kinds[i] in {ltSpaces, ltCrucialNewline, ltSplittingNewline, ltTab} else: true
+
+proc endsInNewline(em: Emitter): bool =
+  var i = em.tokens.len-1
+  while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection, ltSpaces}: dec(i)
+  result = if i >= 0: em.kinds[i] in {ltCrucialNewline, ltSplittingNewline, ltTab} else: true
+
+proc endsInAlpha(em: Emitter): bool =
+  var i = em.tokens.len-1
+  while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection}: dec(i)
+  result = if i >= 0: em.tokens[i].lastChar in SymChars+{'_'} else: false
+
+proc emitComment(em: var Emitter; tok: TToken; dontIndent: bool) =
+  var col = em.col
+  let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA, tok.commentOffsetB)
+  em.lineSpan = countNewlines(lit)
+  if em.lineSpan > 0: calcCol(em, lit)
+  if em.lineSpan == 0:
+    if not endsInNewline(em):
+      wrTab em
+    wr em, lit, ltComment
+  else:
+    if not endsInWhite(em):
+      wrTab em
+      inc col
+    emitMultilineComment(em, lit, col, dontIndent)
 
 proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
+  template wasExportMarker(em): bool =
+    em.kinds.len > 0 and em.kinds[^1] == ltExportMarker
 
-  template endsInWhite(em): bool =
-    em.content.len == 0 or em.content[em.content.high] in {' ', '\L'}
-  template endsInAlpha(em): bool =
-    em.content.len > 0 and em.content[em.content.high] in SymChars+{'_'}
-
-  proc emitComment(em: var Emitter; tok: TToken) =
-    let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA, tok.commentOffsetB)
-    em.lineSpan = countNewlines(lit)
-    if em.lineSpan > 0: calcCol(em, lit)
-    if not endsInWhite(em):
-      wr(" ")
-      if em.lineSpan == 0 and max(em.col, LineCommentColumn) + lit.len <= MaxLineLen:
-        for i in 1 .. LineCommentColumn - em.col: wr(" ")
-    wr lit
+  if tok.tokType == tkComment and tok.literal.startsWith("#!nimpretty"):
+    case tok.literal
+    of "#!nimpretty off":
+      inc em.keepIndents
+      wrNewline em
+      em.lastLineNumber = tok.line + 1
+    of "#!nimpretty on":
+      dec em.keepIndents
+      em.lastLineNumber = tok.line
+    wrNewline em
+    wr em, tok.literal, ltComment
+    em.col = 0
+    em.lineSpan = 0
+    return
 
   var preventComment = false
-  if tok.tokType == tkComment and tok.line == em.lastLineNumber and tok.indent >= 0:
+  if tok.tokType == tkComment and tok.line == em.lastLineNumber:
     # we have an inline comment so handle it before the indentation token:
-    emitComment(em, tok)
+    emitComment(em, tok, dontIndent = (em.inSection == 0))
     preventComment = true
-    em.fixedUntil = em.content.high
+    em.fixedUntil = em.tokens.high
 
   elif tok.indent >= 0:
-    if em.lastTok in (splitters + oprSet):
+    var newlineKind = ltCrucialNewline
+    if em.keepIndents > 0:
       em.indentLevel = tok.indent
+    elif (em.lastTok in (splitters + oprSet) and
+        tok.tokType notin (closedPars - {tkBracketDotRi})):
+      if tok.tokType in openPars and tok.indent > em.indentStack[^1]:
+        while em.indentStack[^1] < tok.indent:
+          em.indentStack.add(em.indentStack[^1] + em.indWidth)
+      # aka: we are in an expression context:
+      let alignment = max(tok.indent - em.indentStack[^1], 0)
+      em.indentLevel = alignment + em.indentStack.high * em.indWidth
+      newlineKind = ltSplittingNewline
     else:
       if tok.indent > em.indentStack[^1]:
         em.indentStack.add tok.indent
@@ -162,114 +474,123 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
        is not touched.
     ]#
     # remove trailing whitespace:
-    while em.content.len > 0 and em.content[em.content.high] == ' ':
-      setLen(em.content, em.content.len-1)
-    wr("\L")
-    for i in 2..tok.line - em.lastLineNumber: wr("\L")
-    em.col = 0
-    for i in 1..em.indentLevel:
-      wr(" ")
-    em.fixedUntil = em.content.high
+    removeSpaces em
+    wrNewline em, newlineKind
+    for i in 2..tok.line - em.lastLineNumber: wrNewline(em)
+    wrSpaces em, em.indentLevel
+    em.fixedUntil = em.tokens.high
 
+  var lastTokWasTerse = false
   case tok.tokType
   of tokKeywordLow..tokKeywordHigh:
     if endsInAlpha(em):
-      wr(" ")
+      wrSpace em
     elif not em.inquote and not endsInWhite(em) and
-        em.lastTok notin openPars:
+        em.lastTok notin (openPars+{tkOpr, tkDotDot}) and not em.lastTokWasTerse:
       #and tok.tokType in oprSet
-      wr(" ")
+      wrSpace em
 
     if not em.inquote:
-      wr(TokTypeToStr[tok.tokType])
-
-      case tok.tokType
-      of tkAnd: rememberSplit(splitAnd)
-      of tkOr: rememberSplit(splitOr)
-      of tkIn, tkNotin:
+      wr(em, TokTypeToStr[tok.tokType], ltKeyword)
+      if tok.tokType in {tkAnd, tkOr, tkIn, tkNotin}:
         rememberSplit(splitIn)
-        wr(" ")
-      else: discard
+        wrSpace em
     else:
       # keywords in backticks are not normalized:
-      wr(tok.ident.s)
+      wr(em, tok.ident.s, ltIdent)
 
   of tkColon:
-    wr(TokTypeToStr[tok.tokType])
-    wr(" ")
-  of tkSemicolon, tkComma:
-    wr(TokTypeToStr[tok.tokType])
+    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wrSpace em
+  of tkSemiColon, tkComma:
+    wr(em, TokTypeToStr[tok.tokType], ltOther)
     rememberSplit(splitComma)
-    wr(" ")
-  of tkParDotLe, tkParLe, tkBracketDotLe, tkBracketLe,
-     tkCurlyLe, tkCurlyDotLe, tkBracketLeColon:
-    if tok.strongSpaceA > 0 and not em.endsInWhite:
-      wr(" ")
-    wr(TokTypeToStr[tok.tokType])
+    wrSpace em
+  of openPars:
+    if tok.strongSpaceA > 0 and not em.endsInWhite and
+        (not em.wasExportMarker or tok.tokType == tkCurlyDotLe):
+      wrSpace em
+    wr(em, TokTypeToStr[tok.tokType], ltSomeParLe)
     rememberSplit(splitParLe)
-  of tkParRi,
-     tkBracketRi, tkCurlyRi,
-     tkBracketDotRi,
-     tkCurlyDotRi,
-     tkParDotRi,
-     tkColonColon, tkDot:
-    wr(TokTypeToStr[tok.tokType])
+  of closedPars:
+    wr(em, TokTypeToStr[tok.tokType], ltSomeParRi)
+  of tkColonColon:
+    wr(em, TokTypeToStr[tok.tokType], ltOther)
+  of tkDot:
+    lastTokWasTerse = true
+    wr(em, TokTypeToStr[tok.tokType], ltOther)
   of tkEquals:
-    if not em.inquote and not em.endsInWhite: wr(" ")
-    wr(TokTypeToStr[tok.tokType])
-    if not em.inquote: wr(" ")
+    if not em.inquote and not em.endsInWhite: wrSpace(em)
+    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    if not em.inquote: wrSpace(em)
   of tkOpr, tkDotDot:
-    if tok.strongSpaceA == 0 and tok.strongSpaceB == 0:
+    if em.inquote or ((tok.strongSpaceA == 0 and tok.strongSpaceB == 0) and
+        tok.ident.s notin ["<", ">", "<=", ">=", "==", "!="]):
+      # bug #9504: remember to not spacify a keyword:
+      lastTokWasTerse = true
       # if not surrounded by whitespace, don't produce any whitespace either:
-      wr(tok.ident.s)
+      wr(em, tok.ident.s, ltOpr)
     else:
-      if not em.endsInWhite: wr(" ")
-      wr(tok.ident.s)
+      if not em.endsInWhite: wrSpace(em)
+      wr(em, tok.ident.s, ltOpr)
       template isUnary(tok): bool =
         tok.strongSpaceB == 0 and tok.strongSpaceA > 0
 
       if not isUnary(tok):
-        wr(" ")
         rememberSplit(splitBinary)
+        wrSpace(em)
   of tkAccent:
-    if not em.inquote and endsInAlpha(em): wr(" ")
-    wr(TokTypeToStr[tok.tokType])
+    if not em.inquote and endsInAlpha(em): wrSpace(em)
+    wr(em, TokTypeToStr[tok.tokType], ltOther)
     em.inquote = not em.inquote
   of tkComment:
     if not preventComment:
-      emitComment(em, tok)
+      emitComment(em, tok, dontIndent = false)
   of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, tkGTripleStrLit, tkCharLit:
     let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
-    softLinebreak(em, lit)
-    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wr(" ")
+    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
     em.lineSpan = countNewlines(lit)
     if em.lineSpan > 0: calcCol(em, lit)
-    wr lit
+    wr em, lit, ltLit
   of tkEof: discard
   else:
     let lit = if tok.ident != nil: tok.ident.s else: tok.literal
-    softLinebreak(em, lit)
-    if endsInAlpha(em): wr(" ")
-    wr lit
+    if endsInAlpha(em): wrSpace(em)
+    wr em, lit, ltIdent
 
   em.lastTok = tok.tokType
+  em.lastTokWasTerse = lastTokWasTerse
   em.lastLineNumber = tok.line + em.lineSpan
   em.lineSpan = 0
 
+proc endsWith(em: Emitter; k: varargs[string]): bool =
+  if em.tokens.len < k.len: return false
+  for i in 0..high(k):
+    if em.tokens[em.tokens.len - k.len + i] != k[i]: return false
+  return true
+
+proc rfind(em: Emitter, t: string): int =
+  for i in 1..5:
+    if em.tokens[^i] == t:
+      return i
+
 proc starWasExportMarker*(em: var Emitter) =
-  if em.content.endsWith(" * "):
-    setLen(em.content, em.content.len-3)
-    em.content.add("*")
+  if em.endsWith(" ", "*", " "):
+    setLen(em.tokens, em.tokens.len-3)
+    setLen(em.kinds, em.kinds.len-3)
+    em.tokens.add("*")
+    em.kinds.add ltExportMarker
     dec em.col, 2
 
 proc commaWasSemicolon*(em: var Emitter) =
   if em.semicolons == detectSemicolonKind:
-    em.semicolons = if em.content.endsWith(", "): dontTouch else: useSemicolon
-  if em.semicolons == useSemicolon and em.content.endsWith(", "):
-    setLen(em.content, em.content.len-2)
-    em.content.add("; ")
+    em.semicolons = if em.rfind(";") > 0: useSemicolon else: dontTouch
+  if em.semicolons == useSemicolon:
+    let commaPos = em.rfind(",")
+    if commaPos > 0:
+      em.tokens[^commaPos] = ";"
 
 proc curlyRiWasPragma*(em: var Emitter) =
-  if em.content.endsWith("}"):
-    setLen(em.content, em.content.len-1)
-    em.content.add(".}")
+  if em.endsWith("}"):
+    em.tokens[^1] = ".}"
+    inc em.col

compiler/lexer.nim

@@ -17,7 +17,7 @@
 
 import
   hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream,
-  wordrecg, lineinfos, pathutils
+  wordrecg, lineinfos, pathutils, parseutils
 
 const
   MaxLineLength* = 80         # lines longer than this lead to a warning
@@ -134,7 +134,7 @@ type
   TErrorHandler* = proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string)
   TLexer* = object of TBaseLexer
     fileIdx*: FileIndex
-    indentAhead*: int         # if > 0 an indendation has already been read
+    indentAhead*: int         # if > 0 an indentation has already been read
                               # this is needed because scanning comments
                               # needs so much look-ahead
     currLineIndent*: int
@@ -146,10 +146,6 @@ type
       previousToken: TLineInfo
     config*: ConfigRef
 
-when defined(nimpretty):
-  var
-    gIndentationWidth*: int
-
 proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
   result = newLineInfo(L.fileIdx, tok.line, tok.col)
   when defined(nimpretty):
@@ -173,7 +169,7 @@ proc isNimIdentifier*(s: string): bool =
       inc(i)
     result = true
 
-proc tokToStr*(tok: TToken): string =
+proc `$`*(tok: TToken): string =
   case tok.tokType
   of tkIntLit..tkInt64Lit: result = $tok.iNumber
   of tkFloatLit..tkFloat64Lit: result = $tok.fNumber
@@ -188,11 +184,11 @@ proc tokToStr*(tok: TToken): string =
 
 proc prettyTok*(tok: TToken): string =
   if isKeyword(tok.tokType): result = "keyword " & tok.ident.s
-  else: result = tokToStr(tok)
+  else: result = $tok
 
 proc printTok*(conf: ConfigRef; tok: TToken) =
   msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" &
-      TokTypeToStr[tok.tokType] & " " & tokToStr(tok))
+      TokTypeToStr[tok.tokType] & " " & $tok)
 
 proc initToken*(L: var TToken) =
   L.tokType = tkInvalid
@@ -223,8 +219,8 @@ proc fillToken(L: var TToken) =
 proc openLexer*(lex: var TLexer, fileIdx: FileIndex, inputstream: PLLStream;
                  cache: IdentCache; config: ConfigRef) =
   openBaseLexer(lex, inputstream)
-  lex.fileIdx = fileidx
-  lex.indentAhead = - 1
+  lex.fileIdx = fileIdx
+  lex.indentAhead = -1
   lex.currLineIndent = 0
   inc(lex.lineNumber, inputstream.lineOffset)
   lex.cache = cache
@@ -307,55 +303,39 @@ template tokenEndPrevious(tok, pos) =
   when defined(nimpretty):
     tok.offsetB = L.offsetBase + pos
 
-{.push overflowChecks: off.}
-# We need to parse the largest uint literal without overflow checks
-proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int =
-  var i = start
-  if i < s.len and s[i] in {'0'..'9'}:
-    b = 0
-    while i < s.len and s[i] in {'0'..'9'}:
-      b = b * 10 + (ord(s[i]) - ord('0'))
-      inc(i)
-      while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored
-    result = i - start
-{.pop.} # overflowChecks
-
-
 template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
-  add(t.literal, replacementChar)
+  t.literal.add(replacementChar)
   inc(L.bufpos)
 
 template eatChar(L: var TLexer, t: var TToken) =
-  add(t.literal, L.buf[L.bufpos])
+  t.literal.add(L.buf[L.bufpos])
   inc(L.bufpos)
 
 proc getNumber(L: var TLexer, result: var TToken) =
   proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]): Natural =
     var pos = L.bufpos              # use registers for pos, buf
-    var buf = L.buf
     result = 0
     while true:
-      if buf[pos] in chars:
-        add(tok.literal, buf[pos])
+      if L.buf[pos] in chars:
+        tok.literal.add(L.buf[pos])
         inc(pos)
         inc(result)
       else:
         break
-      if buf[pos] == '_':
-        if buf[pos+1] notin chars:
+      if L.buf[pos] == '_':
+        if L.buf[pos+1] notin chars:
           lexMessage(L, errGenerated,
             "only single underscores may occur in a token and token may not " &
             "end with an underscore: e.g. '1__1' and '1_' are invalid")
           break
-        add(tok.literal, '_')
+        tok.literal.add('_')
         inc(pos)
     L.bufpos = pos
 
   proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
     var pos = L.bufpos              # use registers for pos, buf
-    var buf = L.buf
-    while buf[pos] in chars:
-      add(tok.literal, buf[pos])
+    while L.buf[pos] in chars:
+      tok.literal.add(L.buf[pos])
       inc(pos)
     L.bufpos = pos
 
@@ -371,12 +351,12 @@ proc getNumber(L: var TLexer, result: var TToken) =
     # We must verify +/- specifically so that we're not past the literal
     if  L.buf[L.bufpos] in {'+', '-'} and
         L.buf[L.bufpos - 1] in {'e', 'E'}:
-      add(t.literal, L.buf[L.bufpos])
+      t.literal.add(L.buf[L.bufpos])
       inc(L.bufpos)
       matchChars(L, t, literalishChars)
     if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
       inc(L.bufpos)
-      add(t.literal, L.buf[L.bufpos])
+      t.literal.add(L.buf[L.bufpos])
       matchChars(L, t, {'0'..'9'})
     L.bufpos = msgPos
     lexMessage(L, msgKind, msg % t.literal)
@@ -395,7 +375,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
   result.literal = ""
   result.base = base10
   startpos = L.bufpos
-  tokenBegin(result, startPos)
+  tokenBegin(result, startpos)
 
   # First stage: find out base, make verifications, build token literal string
   # {'c', 'C'} is added for deprecation reasons to provide a clear error message
@@ -550,13 +530,13 @@ proc getNumber(L: var TLexer, result: var TToken) =
 
       case result.tokType
       of tkIntLit, tkInt64Lit: result.iNumber = xi
-      of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi))))
-      of tkInt16Lit: result.iNumber = BiggestInt(int16(toU16(int(xi))))
-      of tkInt32Lit: result.iNumber = BiggestInt(int32(toU32(int64(xi))))
+      of tkInt8Lit: result.iNumber = ashr(xi shl 56, 56)
+      of tkInt16Lit: result.iNumber = ashr(xi shl 48, 48)
+      of tkInt32Lit: result.iNumber = ashr(xi shl 32, 32)
       of tkUIntLit, tkUInt64Lit: result.iNumber = xi
-      of tkUInt8Lit: result.iNumber = BiggestInt(uint8(toU8(int(xi))))
-      of tkUInt16Lit: result.iNumber = BiggestInt(uint16(toU16(int(xi))))
-      of tkUInt32Lit: result.iNumber = BiggestInt(uint32(toU32(int64(xi))))
+      of tkUInt8Lit: result.iNumber = xi and 0xff
+      of tkUInt16Lit: result.iNumber = xi and 0xffff
+      of tkUInt32Lit: result.iNumber = xi and 0xffffffff
       of tkFloat32Lit:
         result.fNumber = (cast[PFloat32](addr(xi)))[]
         # note: this code is endian neutral!
@@ -585,34 +565,44 @@ proc getNumber(L: var TLexer, result: var TToken) =
       case result.tokType
       of floatTypes:
         result.fNumber = parseFloat(result.literal)
-      of tkUint64Lit:
-        xi = 0
-        let len = unsafeParseUInt(result.literal, xi)
-        if len != result.literal.len or len == 0:
-          raise newException(ValueError, "invalid integer: " & $xi)
-        result.iNumber = xi
+      of tkUInt64Lit, tkUIntLit:
+        var iNumber: uint64
+        var len: int
+        try:
+          len = parseBiggestUInt(result.literal, iNumber)
+        except ValueError:
+          raise newException(OverflowError, "number out of range: " & $result.literal)
+        if len != result.literal.len:
+          raise newException(ValueError, "invalid integer: " & $result.literal)
+        result.iNumber = cast[int64](iNumber)
       else:
-        result.iNumber = parseBiggestInt(result.literal)
+        var iNumber: int64
+        var len: int
+        try:
+          len = parseBiggestInt(result.literal, iNumber)
+        except ValueError:
+          raise newException(OverflowError, "number out of range: " & $result.literal)
+        if len != result.literal.len:
+          raise newException(ValueError, "invalid integer: " & $result.literal)
+        result.iNumber = iNumber
 
-      # Explicit bounds checks
+      # Explicit bounds checks. Only T.high needs to be considered
+      # since result.iNumber can't be negative.
       let outOfRange =
         case result.tokType
-        of tkInt8Lit: (result.iNumber < int8.low or result.iNumber > int8.high)
-        of tkUInt8Lit: (result.iNumber < BiggestInt(uint8.low) or
-                        result.iNumber > BiggestInt(uint8.high))
-        of tkInt16Lit: (result.iNumber < int16.low or result.iNumber > int16.high)
-        of tkUInt16Lit: (result.iNumber < BiggestInt(uint16.low) or
-                        result.iNumber > BiggestInt(uint16.high))
-        of tkInt32Lit: (result.iNumber < int32.low or result.iNumber > int32.high)
-        of tkUInt32Lit: (result.iNumber < BiggestInt(uint32.low) or
-                        result.iNumber > BiggestInt(uint32.high))
+        of tkInt8Lit: result.iNumber > int8.high
+        of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high)
+        of tkInt16Lit: result.iNumber > int16.high
+        of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high)
+        of tkInt32Lit: result.iNumber > int32.high
+        of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high)
         else: false
 
       if outOfRange: lexMessageLitNum(L, "number out of range: '$1'", startpos)
 
     # Promote int literal to int64? Not always necessary, but more consistent
     if result.tokType == tkIntLit:
-      if (result.iNumber < low(int32)) or (result.iNumber > high(int32)):
+      if result.iNumber > high(int32):
         result.tokType = tkInt64Lit
 
   except ValueError:
@@ -622,7 +612,12 @@ proc getNumber(L: var TLexer, result: var TToken) =
   tokenEnd(result, postPos-1)
   L.bufpos = postPos
 
-proc handleHexChar(L: var TLexer, xi: var int) =
+proc handleHexChar(L: var TLexer, xi: var int; position: range[0..4]) =
+  template invalid() =
+    lexMessage(L, errGenerated,
+      "expected a hex digit, but found: " & L.buf[L.bufpos] &
+        "; maybe prepend with 0")
+
   case L.buf[L.bufpos]
   of '0'..'9':
     xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('0'))
@@ -633,13 +628,58 @@ proc handleHexChar(L: var TLexer, xi: var int) =
   of 'A'..'F':
     xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('A') + 10)
     inc(L.bufpos)
-  else: discard
+  of '"', '\'':
+    if position <= 1: invalid()
+    # do not progress the bufpos here.
+    if position == 0: inc(L.bufpos)
+  else:
+    invalid()
+    # Need to progress for `nim check`
+    inc(L.bufpos)
 
 proc handleDecChars(L: var TLexer, xi: var int) =
   while L.buf[L.bufpos] in {'0'..'9'}:
     xi = (xi * 10) + (ord(L.buf[L.bufpos]) - ord('0'))
     inc(L.bufpos)
 
+proc addUnicodeCodePoint(s: var string, i: int) =
+  let i = cast[uint](i)
+  # inlined toUTF-8 to avoid unicode and strutils dependencies.
+  let pos = s.len
+  if i <= 127:
+    s.setLen(pos+1)
+    s[pos+0] = chr(i)
+  elif i <= 0x07FF:
+    s.setLen(pos+2)
+    s[pos+0] = chr((i shr 6) or 0b110_00000)
+    s[pos+1] = chr((i and ones(6)) or 0b10_0000_00)
+  elif i <= 0xFFFF:
+    s.setLen(pos+3)
+    s[pos+0] = chr(i shr 12 or 0b1110_0000)
+    s[pos+1] = chr(i shr 6 and ones(6) or 0b10_0000_00)
+    s[pos+2] = chr(i and ones(6) or 0b10_0000_00)
+  elif i <= 0x001FFFFF:
+    s.setLen(pos+4)
+    s[pos+0] = chr(i shr 18 or 0b1111_0000)
+    s[pos+1] = chr(i shr 12 and ones(6) or 0b10_0000_00)
+    s[pos+2] = chr(i shr 6 and ones(6) or 0b10_0000_00)
+    s[pos+3] = chr(i and ones(6) or 0b10_0000_00)
+  elif i <= 0x03FFFFFF:
+    s.setLen(pos+5)
+    s[pos+0] = chr(i shr 24 or 0b111110_00)
+    s[pos+1] = chr(i shr 18 and ones(6) or 0b10_0000_00)
+    s[pos+2] = chr(i shr 12 and ones(6) or 0b10_0000_00)
+    s[pos+3] = chr(i shr 6 and ones(6) or 0b10_0000_00)
+    s[pos+4] = chr(i and ones(6) or 0b10_0000_00)
+  elif i <= 0x7FFFFFFF:
+    s.setLen(pos+6)
+    s[pos+0] = chr(i shr 30 or 0b1111110_0)
+    s[pos+1] = chr(i shr 24 and ones(6) or 0b10_0000_00)
+    s[pos+2] = chr(i shr 18 and ones(6) or 0b10_0000_00)
+    s[pos+3] = chr(i shr 12 and ones(6) or 0b10_0000_00)
+    s[pos+4] = chr(i shr 6 and ones(6) or 0b10_0000_00)
+    s[pos+5] = chr(i and ones(6) or 0b10_0000_00)
+
 proc getEscapedChar(L: var TLexer, tok: var TToken) =
   inc(L.bufpos)               # skip '\'
   case L.buf[L.bufpos]
@@ -647,81 +687,88 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
     if L.config.oldNewlines:
       if tok.tokType == tkCharLit:
         lexMessage(L, errGenerated, "\\n not allowed in character literal")
-      add(tok.literal, L.config.target.tnl)
+      tok.literal.add(L.config.target.tnl)
     else:
-      add(tok.literal, '\L')
+      tok.literal.add('\L')
     inc(L.bufpos)
   of 'p', 'P':
     if tok.tokType == tkCharLit:
       lexMessage(L, errGenerated, "\\p not allowed in character literal")
-    add(tok.literal, L.config.target.tnl)
+    tok.literal.add(L.config.target.tnl)
     inc(L.bufpos)
   of 'r', 'R', 'c', 'C':
-    add(tok.literal, CR)
+    tok.literal.add(CR)
     inc(L.bufpos)
   of 'l', 'L':
-    add(tok.literal, LF)
+    tok.literal.add(LF)
     inc(L.bufpos)
   of 'f', 'F':
-    add(tok.literal, FF)
+    tok.literal.add(FF)
     inc(L.bufpos)
   of 'e', 'E':
-    add(tok.literal, ESC)
+    tok.literal.add(ESC)
     inc(L.bufpos)
   of 'a', 'A':
-    add(tok.literal, BEL)
+    tok.literal.add(BEL)
     inc(L.bufpos)
   of 'b', 'B':
-    add(tok.literal, BACKSPACE)
+    tok.literal.add(BACKSPACE)
     inc(L.bufpos)
   of 'v', 'V':
-    add(tok.literal, VT)
+    tok.literal.add(VT)
     inc(L.bufpos)
   of 't', 'T':
-    add(tok.literal, '\t')
+    tok.literal.add('\t')
     inc(L.bufpos)
   of '\'', '\"':
-    add(tok.literal, L.buf[L.bufpos])
+    tok.literal.add(L.buf[L.bufpos])
     inc(L.bufpos)
   of '\\':
-    add(tok.literal, '\\')
+    tok.literal.add('\\')
     inc(L.bufpos)
-  of 'x', 'X', 'u', 'U':
-    var tp = L.buf[L.bufpos]
+  of 'x', 'X':
     inc(L.bufpos)
     var xi = 0
-    handleHexChar(L, xi)
-    handleHexChar(L, xi)
-    if tp in {'u', 'U'}:
-      handleHexChar(L, xi)
-      handleHexChar(L, xi)
-      # inlined toUTF-8 to avoid unicode and strutils dependencies.
-      if xi <=% 127:
-        add(tok.literal, xi.char )
-      elif xi <=% 0x07FF:
-        add(tok.literal, ((xi shr 6) or 0b110_00000).char )
-        add(tok.literal, ((xi and ones(6)) or 0b10_0000_00).char )
-      elif xi <=% 0xFFFF:
-        add(tok.literal, (xi shr 12 or 0b1110_0000).char )
-        add(tok.literal, (xi shr 6 and ones(6) or 0b10_0000_00).char )
-        add(tok.literal, (xi and ones(6) or 0b10_0000_00).char )
-      else: # value is 0xFFFF
-        add(tok.literal, "\xef\xbf\xbf" )
+    handleHexChar(L, xi, 1)
+    handleHexChar(L, xi, 2)
+    tok.literal.add(chr(xi))
+  of 'u', 'U':
+    if tok.tokType == tkCharLit:
+      lexMessage(L, errGenerated, "\\u not allowed in character literal")
+    inc(L.bufpos)
+    var xi = 0
+    if L.buf[L.bufpos] == '{':
+      inc(L.bufpos)
+      var start = L.bufpos
+      while L.buf[L.bufpos] != '}':
+        handleHexChar(L, xi, 0)
+      if start == L.bufpos:
+        lexMessage(L, errGenerated,
+          "Unicode codepoint cannot be empty")
+      inc(L.bufpos)
+      if xi > 0x10FFFF:
+        let hex = ($L.buf)[start..L.bufpos-2]
+        lexMessage(L, errGenerated,
+          "Unicode codepoint must be lower than 0x10FFFF, but was: " & hex)
     else:
-      add(tok.literal, chr(xi))
+      handleHexChar(L, xi, 1)
+      handleHexChar(L, xi, 2)
+      handleHexChar(L, xi, 3)
+      handleHexChar(L, xi, 4)
+    addUnicodeCodePoint(tok.literal, xi)
   of '0'..'9':
     if matchTwoChars(L, '0', {'0'..'9'}):
       lexMessage(L, warnOctalEscape)
     var xi = 0
     handleDecChars(L, xi)
-    if (xi <= 255): add(tok.literal, chr(xi))
+    if (xi <= 255): tok.literal.add(chr(xi))
     else: lexMessage(L, errGenerated, "invalid character constant")
   else: lexMessage(L, errGenerated, "invalid character constant")
 
 proc newString(s: cstring, len: int): string =
   ## XXX, how come there is no support for this?
   result = newString(len)
-  for i in 0 ..< len:
+  for i in 0..<len:
     result[i] = s[i]
 
 proc handleCRLF(L: var TLexer, pos: int): int =
@@ -740,37 +787,40 @@ proc handleCRLF(L: var TLexer, pos: int): int =
     result = nimlexbase.handleLF(L, pos)
   else: result = pos
 
-proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
+type
+  StringMode = enum
+    normal,
+    raw,
+    generalized
+
+proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
   var pos = L.bufpos
-  var buf = L.buf                 # put `buf` in a register
   var line = L.lineNumber         # save linenumber for better error message
-  tokenBegin(tok, pos)
+  tokenBegin(tok, pos - ord(mode == raw))
   inc pos # skip "
-  if buf[pos] == '\"' and buf[pos+1] == '\"':
+  if L.buf[pos] == '\"' and L.buf[pos+1] == '\"':
     tok.tokType = tkTripleStrLit # long string literal:
     inc(pos, 2)               # skip ""
     # skip leading newline:
-    if buf[pos] in {' ', '\t'}:
+    if L.buf[pos] in {' ', '\t'}:
       var newpos = pos+1
-      while buf[newpos] in {' ', '\t'}: inc newpos
-      if buf[newpos] in {CR, LF}: pos = newpos
+      while L.buf[newpos] in {' ', '\t'}: inc newpos
+      if L.buf[newpos] in {CR, LF}: pos = newpos
     pos = handleCRLF(L, pos)
-    buf = L.buf
     while true:
-      case buf[pos]
+      case L.buf[pos]
       of '\"':
-        if buf[pos+1] == '\"' and buf[pos+2] == '\"' and
-            buf[pos+3] != '\"':
+        if L.buf[pos+1] == '\"' and L.buf[pos+2] == '\"' and
+            L.buf[pos+3] != '\"':
           tokenEndIgnore(tok, pos+2)
           L.bufpos = pos + 3 # skip the three """
           break
-        add(tok.literal, '\"')
+        tok.literal.add('\"')
         inc(pos)
       of CR, LF:
         tokenEndIgnore(tok, pos)
         pos = handleCRLF(L, pos)
-        buf = L.buf
-        add(tok.literal, "\n")
+        tok.literal.add("\n")
       of nimlexbase.EndOfFile:
         tokenEndIgnore(tok, pos)
         var line2 = L.lineNumber
@@ -780,18 +830,18 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
         L.bufpos = pos
         break
       else:
-        add(tok.literal, buf[pos])
+        tok.literal.add(L.buf[pos])
         inc(pos)
   else:
     # ordinary string literal
-    if rawMode: tok.tokType = tkRStrLit
+    if mode != normal: tok.tokType = tkRStrLit
     else: tok.tokType = tkStrLit
     while true:
-      var c = buf[pos]
+      var c = L.buf[pos]
       if c == '\"':
-        if rawMode and buf[pos+1] == '\"':
+        if mode != normal and L.buf[pos+1] == '\"':
           inc(pos, 2)
-          add(tok.literal, '"')
+          tok.literal.add('"')
         else:
           tokenEndIgnore(tok, pos)
           inc(pos) # skip '"'
@@ -800,12 +850,12 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
         tokenEndIgnore(tok, pos)
         lexMessage(L, errGenerated, "closing \" expected")
         break
-      elif (c == '\\') and not rawMode:
+      elif (c == '\\') and mode == normal:
         L.bufpos = pos
         getEscapedChar(L, tok)
         pos = L.bufpos
       else:
-        add(tok.literal, c)
+        tok.literal.add(c)
         inc(pos)
     L.bufpos = pos
 
@@ -814,7 +864,9 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
   inc(L.bufpos)               # skip '
   var c = L.buf[L.bufpos]
   case c
-  of '\0'..pred(' '), '\'': lexMessage(L, errGenerated, "invalid character literal")
+  of '\0'..pred(' '), '\'':
+    lexMessage(L, errGenerated, "invalid character literal")
+    tok.literal = $c
   of '\\': getEscapedChar(L, tok)
   else:
     tok.literal = $c
@@ -827,10 +879,10 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
 proc getSymbol(L: var TLexer, tok: var TToken) =
   var h: Hash = 0
   var pos = L.bufpos
-  var buf = L.buf
   tokenBegin(tok, pos)
+  var suspicious = false
   while true:
-    var c = buf[pos]
+    var c = L.buf[pos]
     case c
     of 'a'..'z', '0'..'9', '\x80'..'\xFF':
       h = h !& ord(c)
@@ -839,21 +891,26 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
       c = chr(ord(c) + (ord('a') - ord('A'))) # toLower()
       h = h !& ord(c)
       inc(pos)
+      suspicious = true
     of '_':
-      if buf[pos+1] notin SymChars:
+      if L.buf[pos+1] notin SymChars:
         lexMessage(L, errGenerated, "invalid token: trailing underscore")
         break
       inc(pos)
+      suspicious = true
     else: break
   tokenEnd(tok, pos-1)
   h = !$h
   tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
-  L.bufpos = pos
   if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
       (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
     tok.tokType = tkSymbol
   else:
     tok.tokType = TTokType(tok.ident.id + ord(tkSymbol))
+    if suspicious and {optStyleHint, optStyleError} * L.config.globalOptions != {}:
+      lintReport(L.config, getLineInfo(L), tok.ident.s.normalize, tok.ident.s)
+  L.bufpos = pos
+
 
 proc endOperator(L: var TLexer, tok: var TToken, pos: int,
                  hash: Hash) {.inline.} =
@@ -865,11 +922,10 @@ proc endOperator(L: var TLexer, tok: var TToken, pos: int,
 
 proc getOperator(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
-  var buf = L.buf
   tokenBegin(tok, pos)
   var h: Hash = 0
   while true:
-    var c = buf[pos]
+    var c = L.buf[pos]
     if c notin OpChars: break
     h = h !& ord(c)
     inc(pos)
@@ -878,10 +934,10 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   # advance pos but don't store it in L.bufpos so the next token (which might
   # be an operator too) gets the preceding spaces:
   tok.strongSpaceB = 0
-  while buf[pos] == ' ':
+  while L.buf[pos] == ' ':
     inc pos
     inc tok.strongSpaceB
-  if buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
+  if L.buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
 proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
@@ -891,15 +947,14 @@ proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
 
   case tok.tokType
   of tkOpr:
-    let L = tok.ident.s.len
     let relevantChar = tok.ident.s[0]
 
     # arrow like?
-    if L > 1 and tok.ident.s[L-1] == '>' and
-      tok.ident.s[L-2] in {'-', '~', '='}: return considerStrongSpaces(1)
+    if tok.ident.s.len > 1 and tok.ident.s[^1] == '>' and
+      tok.ident.s[^2] in {'-', '~', '='}: return considerStrongSpaces(1)
 
     template considerAsgn(value: untyped) =
-      result = if tok.ident.s[L-1] == '=': 1 else: value
+      result = if tok.ident.s[^1] == '=': 1 else: value
 
     case relevantChar
     of '$', '^': considerAsgn(10)
@@ -912,7 +967,7 @@ proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
     of '?': result = 2
     else: considerAsgn(2)
   of tkDiv, tkMod, tkShl, tkShr: result = 9
-  of tkIn, tkNotin, tkIs, tkIsnot, tkNot, tkOf, tkAs: result = 5
+  of tkIn, tkNotin, tkIs, tkIsnot, tkOf, tkAs: result = 5
   of tkDotDot: result = 6
   of tkAnd: result = 4
   of tkOr, tkXor, tkPtr, tkRef: result = 3
@@ -922,9 +977,8 @@ proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
 
 proc newlineFollows*(L: TLexer): bool =
   var pos = L.bufpos
-  var buf = L.buf
   while true:
-    case buf[pos]
+    case L.buf[pos]
     of ' ', '\t':
       inc(pos)
     of CR, LF:
@@ -932,49 +986,47 @@ proc newlineFollows*(L: TLexer): bool =
       break
     of '#':
       inc(pos)
-      if buf[pos] == '#': inc(pos)
-      if buf[pos] != '[': return true
+      if L.buf[pos] == '#': inc(pos)
+      if L.buf[pos] != '[': return true
     else:
       break
 
 proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
                           isDoc: bool) =
   var pos = start
-  var buf = L.buf
   var toStrip = 0
   tokenBegin(tok, pos)
   # detect the amount of indentation:
   if isDoc:
     toStrip = getColNumber(L, pos)
-    while buf[pos] == ' ': inc pos
-    if buf[pos] in {CR, LF}:
+    while L.buf[pos] == ' ': inc pos
+    if L.buf[pos] in {CR, LF}:
       pos = handleCRLF(L, pos)
-      buf = L.buf
       toStrip = 0
-      while buf[pos] == ' ':
+      while L.buf[pos] == ' ':
         inc pos
         inc toStrip
   var nesting = 0
   while true:
-    case buf[pos]
+    case L.buf[pos]
     of '#':
       if isDoc:
-        if buf[pos+1] == '#' and buf[pos+2] == '[':
+        if L.buf[pos+1] == '#' and L.buf[pos+2] == '[':
           inc nesting
         tok.literal.add '#'
-      elif buf[pos+1] == '[':
+      elif L.buf[pos+1] == '[':
         inc nesting
       inc pos
     of ']':
       if isDoc:
-        if buf[pos+1] == '#' and buf[pos+2] == '#':
+        if L.buf[pos+1] == '#' and L.buf[pos+2] == '#':
           if nesting == 0:
             tokenEndIgnore(tok, pos+2)
             inc(pos, 3)
             break
           dec nesting
         tok.literal.add ']'
-      elif buf[pos+1] == '#':
+      elif L.buf[pos+1] == '#':
         if nesting == 0:
           tokenEndIgnore(tok, pos+1)
           inc(pos, 2)
@@ -984,14 +1036,13 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
     of CR, LF:
       tokenEndIgnore(tok, pos)
       pos = handleCRLF(L, pos)
-      buf = L.buf
       # strip leading whitespace:
       when defined(nimpretty): tok.literal.add "\L"
       if isDoc:
         when not defined(nimpretty): tok.literal.add "\n"
         inc tok.iNumber
         var c = toStrip
-        while buf[pos] == ' ' and c > 0:
+        while L.buf[pos] == ' ' and c > 0:
           inc pos
           dec c
     of nimlexbase.EndOfFile:
@@ -999,7 +1050,7 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
       lexMessagePos(L, errGenerated, pos, "end of multiline comment expected")
       break
     else:
-      if isDoc or defined(nimpretty): tok.literal.add buf[pos]
+      if isDoc or defined(nimpretty): tok.literal.add L.buf[pos]
       inc(pos)
   L.bufpos = pos
   when defined(nimpretty):
@@ -1007,49 +1058,47 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
 
 proc scanComment(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
-  var buf = L.buf
   tok.tokType = tkComment
   # iNumber contains the number of '\n' in the token
   tok.iNumber = 0
-  assert buf[pos+1] == '#'
+  assert L.buf[pos+1] == '#'
   when defined(nimpretty):
     tok.commentOffsetA = L.offsetBase + pos - 1
 
-  if buf[pos+2] == '[':
+  if L.buf[pos+2] == '[':
     skipMultiLineComment(L, tok, pos+3, true)
     return
   tokenBegin(tok, pos)
   inc(pos, 2)
 
   var toStrip = 0
-  while buf[pos] == ' ':
+  while L.buf[pos] == ' ':
     inc pos
     inc toStrip
 
   while true:
     var lastBackslash = -1
-    while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
-      if buf[pos] == '\\': lastBackslash = pos+1
-      add(tok.literal, buf[pos])
+    while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
+      if L.buf[pos] == '\\': lastBackslash = pos+1
+      tok.literal.add(L.buf[pos])
       inc(pos)
     tokenEndIgnore(tok, pos)
     pos = handleCRLF(L, pos)
-    buf = L.buf
     var indent = 0
-    while buf[pos] == ' ':
+    while L.buf[pos] == ' ':
       inc(pos)
       inc(indent)
 
-    if buf[pos] == '#' and buf[pos+1] == '#':
+    if L.buf[pos] == '#' and L.buf[pos+1] == '#':
       tok.literal.add "\n"
       inc(pos, 2)
       var c = toStrip
-      while buf[pos] == ' ' and c > 0:
+      while L.buf[pos] == ' ' and c > 0:
         inc pos
         dec c
       inc tok.iNumber
     else:
-      if buf[pos] > ' ':
+      if L.buf[pos] > ' ':
         L.indentAhead = indent
       tokenEndIgnore(tok, pos)
       break
@@ -1059,56 +1108,63 @@ proc scanComment(L: var TLexer, tok: var TToken) =
 
 proc skip(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
-  var buf = L.buf
   tokenBegin(tok, pos)
   tok.strongSpaceA = 0
   when defined(nimpretty):
     var hasComment = false
+    var commentIndent = L.currLineIndent
     tok.commentOffsetA = L.offsetBase + pos
     tok.commentOffsetB = tok.commentOffsetA
+    tok.line = -1
   while true:
-    case buf[pos]
+    case L.buf[pos]
     of ' ':
       inc(pos)
       inc(tok.strongSpaceA)
     of '\t':
-      if not L.allowTabs: lexMessagePos(L, errGenerated, pos, "tabulators are not allowed")
+      if not L.allowTabs: lexMessagePos(L, errGenerated, pos, "tabs are not allowed, use spaces instead")
       inc(pos)
     of CR, LF:
       tokenEndPrevious(tok, pos)
-      when defined(nimpretty):
-        # we are not yet in a comment, so update the comment token's line information:
-        if not hasComment: inc tok.line
       pos = handleCRLF(L, pos)
-      buf = L.buf
       var indent = 0
       while true:
-        if buf[pos] == ' ':
+        if L.buf[pos] == ' ':
           inc(pos)
           inc(indent)
-        elif buf[pos] == '#' and buf[pos+1] == '[':
-          when defined(nimpretty): hasComment = true
+        elif L.buf[pos] == '#' and L.buf[pos+1] == '[':
+          when defined(nimpretty):
+            hasComment = true
+            if tok.line < 0:
+              tok.line = L.lineNumber
+              commentIndent = indent
           skipMultiLineComment(L, tok, pos+2, false)
           pos = L.bufpos
-          buf = L.buf
         else:
           break
       tok.strongSpaceA = 0
-      if buf[pos] > ' ' and (buf[pos] != '#' or buf[pos+1] == '#'):
+      when defined(nimpretty):
+        if L.buf[pos] == '#' and tok.line < 0: commentIndent = indent
+      if L.buf[pos] > ' ' and (L.buf[pos] != '#' or L.buf[pos+1] == '#'):
         tok.indent = indent
         L.currLineIndent = indent
         break
     of '#':
       # do not skip documentation comment:
-      if buf[pos+1] == '#': break
-      when defined(nimpretty): hasComment = true
-      if buf[pos+1] == '[':
+      if L.buf[pos+1] == '#': break
+      when defined(nimpretty):
+        hasComment = true
+        if tok.line < 0:
+          tok.line = L.lineNumber
+
+      if L.buf[pos+1] == '[':
         skipMultiLineComment(L, tok, pos+2, false)
         pos = L.bufpos
-        buf = L.buf
       else:
         tokenBegin(tok, pos)
-        while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos)
+        while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
+          when defined(nimpretty): tok.literal.add L.buf[pos]
+          inc(pos)
         tokenEndIgnore(tok, pos+1)
         when defined(nimpretty):
           tok.commentOffsetB = L.offsetBase + pos + 1
@@ -1120,8 +1176,7 @@ proc skip(L: var TLexer, tok: var TToken) =
     if hasComment:
       tok.commentOffsetB = L.offsetBase + pos - 1
       tok.tokType = tkComment
-    if gIndentationWidth <= 0:
-      gIndentationWidth = tok.indent
+      tok.indent = commentIndent
 
 proc rawGetTok*(L: var TLexer, tok: var TToken) =
   template atTokenEnd() {.dirty.} =
@@ -1168,7 +1223,7 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
     of 'r', 'R':
       if L.buf[L.bufpos + 1] == '\"':
         inc(L.bufpos)
-        getString(L, tok, true)
+        getString(L, tok, raw)
       else:
         getSymbol(L, tok)
     of '(':
@@ -1246,9 +1301,9 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
         lexMessage(L, errGenerated, "invalid token: " & c & " (\\" & $(ord(c)) & ')')
     of '\"':
       # check for generalized raw string literal:
-      var rawMode = L.bufpos > 0 and L.buf[L.bufpos-1] in SymChars
-      getString(L, tok, rawMode)
-      if rawMode:
+      let mode = if L.bufpos > 0 and L.buf[L.bufpos-1] in SymChars: generalized else: normal
+      getString(L, tok, mode)
+      if mode == generalized:
         # tkRStrLit -> tkGStrLit
         # tkTripleStrLit -> tkGTripleStrLit
         inc(tok.tokType, 2)
@@ -1280,10 +1335,13 @@ proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
   var tok: TToken
   initToken(tok)
   openLexer(lex, fileIdx, inputstream, cache, config)
-  while true:
+  var prevToken = tkEof
+  while tok.tokType != tkEof:
     rawGetTok(lex, tok)
-    result = tok.indent
-    if result > 0 or tok.tokType == tkEof: break
+    if tok.indent > 0 and prevToken in {tkColon, tkEquals, tkType, tkConst, tkLet, tkVar, tkUsing}:
+      result = tok.indent
+      if result > 0: break
+    prevToken = tok.tokType
   closeLexer(lex)
 
 proc getPrecedence*(ident: PIdent): int =
@@ -1292,7 +1350,7 @@ proc getPrecedence*(ident: PIdent): int =
   initToken(tok)
   tok.ident = ident
   tok.tokType =
-    if tok.ident.id in ord(tokKeywordLow) - ord(tkSymbol) .. ord(tokKeywordHigh) - ord(tkSymbol):
+    if tok.ident.id in ord(tokKeywordLow) - ord(tkSymbol)..ord(tokKeywordHigh) - ord(tkSymbol):
       TTokType(tok.ident.id + ord(tkSymbol))
     else: tkOpr
   getPrecedence(tok, false)

compiler/liftdestructors.nim

@@ -0,0 +1,891 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements lifting for type-bound operations
+## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``).
+
+# Todo:
+# - use openArray instead of array to avoid over-specializations
+
+import modulegraphs, lineinfos, idents, ast, renderer, semdata,
+  sighashes, lowerings, options, types, msgs, magicsys, tables
+
+from trees import isCaseObj
+
+type
+  TLiftCtx = object
+    g: ModuleGraph
+    info: TLineInfo # for construction
+    kind: TTypeAttachedOp
+    fn: PSym
+    asgnForType: PType
+    recurse: bool
+    c: PContext # c can be nil, then we are called from lambdalifting!
+
+proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode)
+proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
+              info: TLineInfo): PSym
+
+proc createTypeBoundOps*(g: ModuleGraph; c: PContext; orig: PType; info: TLineInfo)
+
+proc at(a, i: PNode, elemType: PType): PNode =
+  result = newNodeI(nkBracketExpr, a.info, 2)
+  result[0] = a
+  result[1] = i
+  result.typ = elemType
+
+proc fillBodyTup(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  for i in 0..<t.len:
+    let lit = lowerings.newIntLit(c.g, x.info, i)
+    let b = if c.kind == attachedTrace: y else: y.at(lit, t[i])
+    fillBody(c, t[i], body, x.at(lit, t[i]), b)
+
+proc dotField(x: PNode, f: PSym): PNode =
+  result = newNodeI(nkDotExpr, x.info, 2)
+  if x.typ.skipTypes(abstractInst).kind == tyVar:
+    result[0] = x.newDeref
+  else:
+    result[0] = x
+  result[1] = newSymNode(f, x.info)
+  result.typ = f.typ
+
+proc newAsgnStmt(le, ri: PNode): PNode =
+  result = newNodeI(nkAsgn, le.info, 2)
+  result[0] = le
+  result[1] = ri
+
+proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  if c.kind in {attachedAsgn, attachedDeepCopy, attachedSink}:
+    body.add newAsgnStmt(x, y)
+
+proc genAddr(g: ModuleGraph; x: PNode): PNode =
+  if x.kind == nkHiddenDeref:
+    checkSonsLen(x, 1, g.config)
+    result = x[0]
+  else:
+    result = newNodeIT(nkHiddenAddr, x.info, makeVarType(x.typ.owner, x.typ))
+    result.add x
+
+proc destructorCall(g: ModuleGraph; op: PSym; x: PNode): PNode =
+  result = newNodeIT(nkCall, x.info, op.typ[0])
+  result.add(newSymNode(op))
+  result.add genAddr(g, x)
+
+proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool) =
+  case n.kind
+  of nkSym:
+    let f = n.sym
+    let b = if c.kind == attachedTrace: y else: y.dotField(f)
+    if (sfCursor in f.flags and f.typ.skipTypes(abstractInst).kind in {tyRef, tyProc} and
+        c.g.config.selectedGC in {gcArc, gcOrc, gcHooks}) or
+        enforceDefaultOp:
+      defaultOp(c, f.typ, body, x.dotField(f), b)
+    else:
+      fillBody(c, f.typ, body, x.dotField(f), b)
+  of nkNilLit: discard
+  of nkRecCase:
+    # XXX This is only correct for 'attachedSink'!
+    var localEnforceDefaultOp = enforceDefaultOp
+    if c.kind == attachedSink:
+      # the value needs to be destroyed before we assign the selector
+      # or the value is lost
+      let prevKind = c.kind
+      c.kind = attachedDestructor
+      fillBodyObj(c, n, body, x, y, enforceDefaultOp = false)
+      c.kind = prevKind
+      localEnforceDefaultOp = true
+
+    # copy the selector:
+    fillBodyObj(c, n[0], body, x, y, enforceDefaultOp = false)
+    # we need to generate a case statement:
+    var caseStmt = newNodeI(nkCaseStmt, c.info)
+    # XXX generate 'if' that checks same branches
+    # generate selector:
+    var access = dotField(x, n[0].sym)
+    caseStmt.add(access)
+    var emptyBranches = 0
+    # copy the branches over, but replace the fields with the for loop body:
+    for i in 1..<n.len:
+      var branch = copyTree(n[i])
+      branch[^1] = newNodeI(nkStmtList, c.info)
+
+      fillBodyObj(c, n[i].lastSon, branch[^1], x, y,
+                  enforceDefaultOp = localEnforceDefaultOp)
+      if branch[^1].len == 0: inc emptyBranches
+      caseStmt.add(branch)
+    if emptyBranches != n.len-1:
+      body.add(caseStmt)
+  of nkRecList:
+    for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp)
+  else:
+    illFormedAstLocal(n, c.g.config)
+
+proc fillBodyObjTImpl(c: var TLiftCtx; t: PType, body, x, y: PNode) =
+  if t.len > 0 and t[0] != nil:
+    fillBody(c, skipTypes(t[0], abstractPtrs), body, x, y)
+  fillBodyObj(c, t.n, body, x, y, enforceDefaultOp = false)
+
+proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
+  var hasCase = isCaseObj(t.n)
+  var obj = t
+  while obj.len > 0 and obj[0] != nil:
+    obj = skipTypes(obj[0], abstractPtrs)
+    hasCase = hasCase or isCaseObj(obj.n)
+
+  if hasCase and c.kind in {attachedAsgn, attachedDeepCopy}:
+    # assignment for case objects is complex, we do:
+    # =destroy(dest)
+    # wasMoved(dest)
+    # for every field:
+    #   `=` dest.field, src.field
+    # ^ this is what we used to do, but for 'result = result.sons[0]' it
+    # destroys 'result' too early.
+    # So this is what we really need to do:
+    # let blob {.cursor.} = dest # remembers the old dest.kind
+    # wasMoved(dest)
+    # dest.kind = src.kind
+    # for every field (dependent on dest.kind):
+    #   `=` dest.field, src.field
+    # =destroy(blob)
+    var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.fn, c.info)
+    temp.typ = x.typ
+    incl(temp.flags, sfFromGeneric)
+    var v = newNodeI(nkVarSection, c.info)
+    let blob = newSymNode(temp)
+    v.addVar(blob, x)
+    body.add v
+    #body.add newAsgnStmt(blob, x)
+
+    var wasMovedCall = newNodeI(nkCall, c.info)
+    wasMovedCall.add(newSymNode(createMagic(c.g, "wasMoved", mWasMoved)))
+    wasMovedCall.add x # mWasMoved does not take the address
+    body.add wasMovedCall
+
+    fillBodyObjTImpl(c, t, body, x, y)
+    when false:
+      # does not work yet due to phase-ordering problems:
+      assert t.destructor != nil
+      body.add destructorCall(c.g, t.destructor, blob)
+    let prevKind = c.kind
+    c.kind = attachedDestructor
+    fillBodyObjTImpl(c, t, body, blob, y)
+    c.kind = prevKind
+  else:
+    fillBodyObjTImpl(c, t, body, x, y)
+
+proc newHookCall(g: ModuleGraph; op: PSym; x, y: PNode): PNode =
+  #if sfError in op.flags:
+  #  localError(c.config, x.info, "usage of '$1' is a user-defined error" % op.name.s)
+  result = newNodeI(nkCall, x.info)
+  result.add newSymNode(op)
+  if op.typ.sons[1].kind == tyVar:
+    result.add genAddr(g, x)
+  else:
+    result.add x
+  if y != nil:
+    result.add y
+
+proc newOpCall(op: PSym; x: PNode): PNode =
+  result = newNodeIT(nkCall, x.info, op.typ[0])
+  result.add(newSymNode(op))
+  result.add x
+
+proc newDeepCopyCall(op: PSym; x, y: PNode): PNode =
+  result = newAsgnStmt(x, newOpCall(op, y))
+
+proc useNoGc(c: TLiftCtx; t: PType): bool {.inline.} =
+  result = optSeqDestructors in c.g.config.globalOptions and
+    ({tfHasGCedMem, tfHasOwned} * t.flags != {} or t.isGCedMem)
+
+proc requiresDestructor(c: TLiftCtx; t: PType): bool {.inline.} =
+  result = optSeqDestructors in c.g.config.globalOptions and
+    containsGarbageCollectedRef(t)
+
+proc instantiateGeneric(c: var TLiftCtx; op: PSym; t, typeInst: PType): PSym =
+  if c.c != nil and typeInst != nil:
+    result = c.c.instTypeBoundOp(c.c, op, typeInst, c.info, attachedAsgn, 1)
+  else:
+    localError(c.g.config, c.info,
+      "cannot generate destructor for generic type: " & typeToString(t))
+    result = nil
+
+proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
+                        field: var PSym): bool =
+  if optSeqDestructors in c.g.config.globalOptions:
+    let op = field
+    if field != nil and sfOverriden in field.flags:
+      if sfError in op.flags:
+        incl c.fn.flags, sfError
+      #else:
+      #  markUsed(c.g.config, c.info, op, c.g.usageSym)
+      onUse(c.info, op)
+      body.add newHookCall(c.g, op, x, y)
+      result = true
+  elif tfHasAsgn in t.flags:
+    var op: PSym
+    if sameType(t, c.asgnForType):
+      # generate recursive call:
+      if c.recurse:
+        op = c.fn
+      else:
+        c.recurse = true
+        return false
+    else:
+      op = field
+      if op == nil:
+        op = produceSym(c.g, c.c, t, c.kind, c.info)
+    if sfError in op.flags:
+      incl c.fn.flags, sfError
+    #else:
+    #  markUsed(c.g.config, c.info, op, c.g.usageSym)
+    onUse(c.info, op)
+    # We also now do generic instantiations in the destructor lifting pass:
+    if op.ast[genericParamsPos].kind != nkEmpty:
+      op = instantiateGeneric(c, op, t, t.typeInst)
+      field = op
+      #echo "trying to use ", op.ast
+      #echo "for ", op.name.s, " "
+      #debug(t)
+      #return false
+    assert op.ast[genericParamsPos].kind == nkEmpty
+    body.add newHookCall(c.g, op, x, y)
+    result = true
+
+proc addDestructorCall(c: var TLiftCtx; orig: PType; body, x: PNode) =
+  let t = orig.skipTypes(abstractInst)
+  var op = t.destructor
+
+  if op != nil and sfOverriden in op.flags:
+    if op.ast[genericParamsPos].kind != nkEmpty:
+      # patch generic destructor:
+      op = instantiateGeneric(c, op, t, t.typeInst)
+      t.attachedOps[attachedDestructor] = op
+
+  if op == nil and (useNoGc(c, t) or requiresDestructor(c, t)):
+    op = produceSym(c.g, c.c, t, attachedDestructor, c.info)
+    doAssert op != nil
+    doAssert op == t.destructor
+
+  if op != nil:
+    #markUsed(c.g.config, c.info, op, c.g.usageSym)
+    onUse(c.info, op)
+    body.add destructorCall(c.g, op, x)
+  elif useNoGc(c, t):
+    internalError(c.g.config, c.info,
+      "type-bound operator could not be resolved")
+
+proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
+  case c.kind
+  of attachedDestructor:
+    var op = t.destructor
+    if op != nil and sfOverriden in op.flags:
+
+      if op.ast[genericParamsPos].kind != nkEmpty:
+        # patch generic destructor:
+        op = instantiateGeneric(c, op, t, t.typeInst)
+        t.attachedOps[attachedDestructor] = op
+
+      #markUsed(c.g.config, c.info, op, c.g.usageSym)
+      onUse(c.info, op)
+      body.add destructorCall(c.g, op, x)
+      result = true
+    #result = addDestructorCall(c, t, body, x)
+  of attachedAsgn, attachedSink, attachedTrace:
+    result = considerAsgnOrSink(c, t, body, x, y, t.attachedOps[c.kind])
+  of attachedDispose:
+    result = considerAsgnOrSink(c, t, body, x, nil, t.attachedOps[c.kind])
+  of attachedDeepCopy:
+    let op = t.attachedOps[attachedDeepCopy]
+    if op != nil:
+      #markUsed(c.g.config, c.info, op, c.g.usageSym)
+      onUse(c.info, op)
+      body.add newDeepCopyCall(op, x, y)
+      result = true
+
+proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
+  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.fn, c.info)
+  temp.typ = getSysType(c.g, body.info, tyInt)
+  incl(temp.flags, sfFromGeneric)
+
+  var v = newNodeI(nkVarSection, c.info)
+  result = newSymNode(temp)
+  v.addVar(result, lowerings.newIntLit(c.g, body.info, first))
+  body.add v
+
+proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode =
+  result = newNodeI(nkCall, i.info)
+  result.add createMagic(g, name, magic).newSymNode
+  result.add i
+
+proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
+  result = newNodeI(nkWhileStmt, c.info, 2)
+  let cmp = genBuiltin(c.g, mLtI, "<", i)
+  cmp.add genLen(c.g, dest)
+  cmp.typ = getSysType(c.g, c.info, tyBool)
+  result[0] = cmp
+  result[1] = newNodeI(nkStmtList, c.info)
+
+proc genIf(c: var TLiftCtx; cond, action: PNode): PNode =
+  result = newTree(nkIfStmt, newTree(nkElifBranch, cond, action))
+
+proc addIncStmt(c: var TLiftCtx; body, i: PNode) =
+  let incCall = genBuiltin(c.g, mInc, "inc", i)
+  incCall.add lowerings.newIntLit(c.g, c.info, 1)
+  body.add incCall
+
+proc newSeqCall(g: ModuleGraph; x, y: PNode): PNode =
+  # don't call genAddr(c, x) here:
+  result = genBuiltin(g, mNewSeq, "newSeq", x)
+  let lenCall = genBuiltin(g, mLengthSeq, "len", y)
+  lenCall.typ = getSysType(g, x.info, tyInt)
+  result.add lenCall
+
+proc setLenStrCall(g: ModuleGraph; x, y: PNode): PNode =
+  let lenCall = genBuiltin(g, mLengthStr, "len", y)
+  lenCall.typ = getSysType(g, x.info, tyInt)
+  result = genBuiltin(g, mSetLengthStr, "setLen", x) # genAddr(g, x))
+  result.add lenCall
+
+proc setLenSeqCall(c: var TLiftCtx; t: PType; x, y: PNode): PNode =
+  let lenCall = genBuiltin(c.g, mLengthSeq, "len", y)
+  lenCall.typ = getSysType(c.g, x.info, tyInt)
+  var op = getSysMagic(c.g, x.info, "setLen", mSetLengthSeq)
+  op = instantiateGeneric(c, op, t, t)
+  result = newTree(nkCall, newSymNode(op, x.info), x, lenCall)
+
+proc forallElements(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  let i = declareCounter(c, body, toInt64(firstOrd(c.g.config, t)))
+  let whileLoop = genWhileLoop(c, i, x)
+  let elemType = t.lastSon
+  let b = if c.kind == attachedTrace: y else: y.at(i, elemType)
+  fillBody(c, elemType, whileLoop[1], x.at(i, elemType), b)
+  addIncStmt(c, whileLoop[1], i)
+  body.add whileLoop
+
+proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  case c.kind
+  of attachedAsgn, attachedDeepCopy:
+    # we generate:
+    # setLen(dest, y.len)
+    # var i = 0
+    # while i < y.len: dest[i] = y[i]; inc(i)
+    # This is usually more efficient than a destroy/create pair.
+    body.add setLenSeqCall(c, t, x, y)
+    forallElements(c, t, body, x, y)
+  of attachedSink:
+    let moveCall = genBuiltin(c.g, mMove, "move", x)
+    moveCall.add y
+    doAssert t.destructor != nil
+    moveCall.add destructorCall(c.g, t.destructor, x)
+    body.add moveCall
+  of attachedDestructor:
+    # destroy all elements:
+    forallElements(c, t, body, x, y)
+    body.add genBuiltin(c.g, mDestroy, "destroy", x)
+  of attachedTrace:
+    # follow all elements:
+    forallElements(c, t, body, x, y)
+  of attachedDispose:
+    body.add genBuiltin(c.g, mDestroy, "destroy", x)
+
+proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  createTypeBoundOps(c.g, c.c, t, body.info)
+  # recursions are tricky, so we might need to forward the generated
+  # operation here:
+  var t = t
+  if t.assignment == nil or t.destructor == nil:
+    let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
+    let canon = c.g.canonTypes.getOrDefault(h)
+    if canon != nil: t = canon
+
+  case c.kind
+  of attachedAsgn, attachedDeepCopy:
+    doAssert t.assignment != nil
+    body.add newHookCall(c.g, t.assignment, x, y)
+  of attachedSink:
+    # we always inline the move for better performance:
+    let moveCall = genBuiltin(c.g, mMove, "move", x)
+    moveCall.add y
+    doAssert t.destructor != nil
+    moveCall.add destructorCall(c.g, t.destructor, x)
+    body.add moveCall
+    # alternatively we could do this:
+    when false:
+      doAssert t.asink != nil
+      body.add newHookCall(c.g, t.asink, x, y)
+  of attachedDestructor:
+    doAssert t.destructor != nil
+    body.add destructorCall(c.g, t.destructor, x)
+  of attachedTrace:
+    body.add newHookCall(c.g, t.attachedOps[c.kind], x, y)
+  of attachedDispose:
+    body.add newHookCall(c.g, t.attachedOps[c.kind], x, nil)
+
+proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  case c.kind
+  of attachedAsgn, attachedDeepCopy:
+    body.add callCodegenProc(c.g, "nimAsgnStrV2", c.info, genAddr(c.g, x), y)
+  of attachedSink:
+    let moveCall = genBuiltin(c.g, mMove, "move", x)
+    moveCall.add y
+    doAssert t.destructor != nil
+    moveCall.add destructorCall(c.g, t.destructor, x)
+    body.add moveCall
+  of attachedDestructor, attachedDispose:
+    body.add genBuiltin(c.g, mDestroy, "destroy", x)
+  of attachedTrace:
+    discard "strings are atomic and have no inner elements that are to trace"
+
+proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  var actions = newNodeI(nkStmtList, c.info)
+  let elemType = t.lastSon
+
+  if isFinal(elemType):
+    addDestructorCall(c, elemType, actions, genDeref(x, nkDerefExpr))
+    actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x)
+  else:
+    addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(x, nkDerefExpr))
+    actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, x)
+
+  let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(t)
+
+  var cond: PNode
+  if isCyclic:
+    if isFinal(elemType):
+      let typInfo = genBuiltin(c.g, mGetTypeInfo, "getTypeInfo", newNodeIT(nkType, x.info, elemType))
+      typInfo.typ = getSysType(c.g, c.info, tyPointer)
+      cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, x, typInfo)
+    else:
+      cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicDyn", c.info, x)
+  else:
+    cond = callCodegenProc(c.g, "nimDecRefIsLast", c.info, x)
+  cond.typ = getSysType(c.g, x.info, tyBool)
+
+  case c.kind
+  of attachedSink:
+    body.add genIf(c, cond, actions)
+    body.add newAsgnStmt(x, y)
+  of attachedAsgn:
+    body.add genIf(c, y, callCodegenProc(c.g,
+        if isCyclic: "nimIncRefCyclic" else: "nimIncRef", c.info, y))
+    body.add genIf(c, cond, actions)
+    body.add newAsgnStmt(x, y)
+  of attachedDestructor:
+    actions.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
+    body.add genIf(c, cond, actions)
+  of attachedDeepCopy: assert(false, "cannot happen")
+  of attachedTrace:
+    if isFinal(elemType):
+      let typInfo = genBuiltin(c.g, mGetTypeInfo, "getTypeInfo", newNodeIT(nkType, x.info, elemType))
+      typInfo.typ = getSysType(c.g, c.info, tyPointer)
+      body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x), typInfo, y)
+    else:
+      # If the ref is polymorphic we have to account for this
+      body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(x), y)
+  of attachedDispose:
+    # this is crucial! dispose is like =destroy but we don't follow refs
+    # as that is dealt within the cycle collector.
+    when false:
+      let cond = copyTree(x)
+      cond.typ = getSysType(c.g, x.info, tyBool)
+      actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x)
+      body.add genIf(c, cond, actions)
+
+proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  ## Closures are really like refs except they always use a virtual destructor
+  ## and we need to do the refcounting only on the ref field which we call 'xenv':
+  let xenv = genBuiltin(c.g, mAccessEnv, "accessEnv", x)
+  xenv.typ = getSysType(c.g, c.info, tyPointer)
+
+  var actions = newNodeI(nkStmtList, c.info)
+  actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, xenv)
+
+  let decRefProc =
+    if c.g.config.selectedGC == gcOrc: "nimDecRefIsLastCyclicDyn"
+    else: "nimDecRefIsLast"
+  let cond = callCodegenProc(c.g, decRefProc, c.info, xenv)
+  cond.typ = getSysType(c.g, x.info, tyBool)
+
+  case c.kind
+  of attachedSink:
+    body.add genIf(c, cond, actions)
+    body.add newAsgnStmt(x, y)
+  of attachedAsgn:
+    let yenv = genBuiltin(c.g, mAccessEnv, "accessEnv", y)
+    yenv.typ = getSysType(c.g, c.info, tyPointer)
+    let incRefProc =
+      if c.g.config.selectedGC == gcOrc: "nimIncRefCyclic"
+      else: "nimIncRef"
+    body.add genIf(c, yenv, callCodegenProc(c.g, incRefProc, c.info, yenv))
+    body.add genIf(c, cond, actions)
+    body.add newAsgnStmt(x, y)
+  of attachedDestructor:
+    actions.add newAsgnStmt(xenv, newNodeIT(nkNilLit, body.info, xenv.typ))
+    body.add genIf(c, cond, actions)
+  of attachedDeepCopy: assert(false, "cannot happen")
+  of attachedTrace:
+    body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv), y)
+  of attachedDispose:
+    # this is crucial! dispose is like =destroy but we don't follow refs
+    # as that is dealt within the cycle collector.
+    when false:
+      let cond = copyTree(xenv)
+      cond.typ = getSysType(c.g, xenv.info, tyBool)
+      actions.add callCodegenProc(c.g, "nimRawDispose", c.info, xenv)
+      body.add genIf(c, cond, actions)
+
+proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  case c.kind
+  of attachedSink:
+    # we 'nil' y out afterwards so we *need* to take over its reference
+    # count value:
+    body.add genIf(c, x, callCodegenProc(c.g, "nimDecWeakRef", c.info, x))
+    body.add newAsgnStmt(x, y)
+  of attachedAsgn:
+    body.add genIf(c, y, callCodegenProc(c.g, "nimIncRef", c.info, y))
+    body.add genIf(c, x, callCodegenProc(c.g, "nimDecWeakRef", c.info, x))
+    body.add newAsgnStmt(x, y)
+  of attachedDestructor:
+    # it's better to prepend the destruction of weak refs in order to
+    # prevent wrong "dangling refs exist" problems:
+    var actions = newNodeI(nkStmtList, c.info)
+    actions.add callCodegenProc(c.g, "nimDecWeakRef", c.info, x)
+    actions.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
+    let des = genIf(c, x, actions)
+    if body.len == 0:
+      body.add des
+    else:
+      body.sons.insert(des, 0)
+  of attachedDeepCopy: assert(false, "cannot happen")
+  of attachedTrace, attachedDispose: discard
+
+proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  var actions = newNodeI(nkStmtList, c.info)
+
+  let elemType = t.lastSon
+  #fillBody(c, elemType, actions, genDeref(x), genDeref(y))
+  #var disposeCall = genBuiltin(c.g, mDispose, "dispose", x)
+
+  if isFinal(elemType):
+    addDestructorCall(c, elemType, actions, genDeref(x, nkDerefExpr))
+    actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x)
+  else:
+    addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(x, nkDerefExpr))
+    actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, x)
+
+  case c.kind
+  of attachedSink, attachedAsgn:
+    body.add genIf(c, x, actions)
+    body.add newAsgnStmt(x, y)
+  of attachedDestructor:
+    actions.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
+    body.add genIf(c, x, actions)
+  of attachedDeepCopy: assert(false, "cannot happen")
+  of attachedTrace, attachedDispose: discard
+
+proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  if c.kind == attachedDeepCopy:
+    # a big problem is that we don't know the environment's type here, so we
+    # have to go through some indirection; we delegate this to the codegen:
+    let call = newNodeI(nkCall, c.info, 2)
+    call.typ = t
+    call[0] = newSymNode(createMagic(c.g, "deepCopy", mDeepCopy))
+    call[1] = y
+    body.add newAsgnStmt(x, call)
+  elif (optOwnedRefs in c.g.config.globalOptions and
+      optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcOrc}:
+    let xx = genBuiltin(c.g, mAccessEnv, "accessEnv", x)
+    xx.typ = getSysType(c.g, c.info, tyPointer)
+    case c.kind
+    of attachedSink:
+      # we 'nil' y out afterwards so we *need* to take over its reference
+      # count value:
+      body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
+      body.add newAsgnStmt(x, y)
+    of attachedAsgn:
+      let yy = genBuiltin(c.g, mAccessEnv, "accessEnv", y)
+      yy.typ = getSysType(c.g, c.info, tyPointer)
+      body.add genIf(c, yy, callCodegenProc(c.g, "nimIncRef", c.info, yy))
+      body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
+      body.add newAsgnStmt(x, y)
+    of attachedDestructor:
+      var actions = newNodeI(nkStmtList, c.info)
+      actions.add callCodegenProc(c.g, "nimDecWeakRef", c.info, xx)
+      actions.add newAsgnStmt(xx, newNodeIT(nkNilLit, body.info, xx.typ))
+      let des = genIf(c, xx, actions)
+      if body.len == 0:
+        body.add des
+      else:
+        body.sons.insert(des, 0)
+    of attachedDeepCopy: assert(false, "cannot happen")
+    of attachedTrace, attachedDispose: discard
+
+proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  let xx = genBuiltin(c.g, mAccessEnv, "accessEnv", x)
+  xx.typ = getSysType(c.g, c.info, tyPointer)
+  var actions = newNodeI(nkStmtList, c.info)
+  #discard addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(xx))
+  actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, xx)
+  case c.kind
+  of attachedSink, attachedAsgn:
+    body.add genIf(c, xx, actions)
+    body.add newAsgnStmt(x, y)
+  of attachedDestructor:
+    actions.add newAsgnStmt(xx, newNodeIT(nkNilLit, body.info, xx.typ))
+    body.add genIf(c, xx, actions)
+  of attachedDeepCopy: assert(false, "cannot happen")
+  of attachedTrace, attachedDispose: discard
+
+proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  case t.kind
+  of tyNone, tyEmpty, tyVoid: discard
+  of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString,
+      tyPtr, tyOpt, tyUncheckedArray, tyVar, tyLent:
+    defaultOp(c, t, body, x, y)
+  of tyRef:
+    if c.g.config.selectedGC in {gcArc, gcOrc}:
+      atomicRefOp(c, t, body, x, y)
+    elif (optOwnedRefs in c.g.config.globalOptions and
+        optRefCheck in c.g.config.options):
+      weakrefOp(c, t, body, x, y)
+    else:
+      defaultOp(c, t, body, x, y)
+  of tyProc:
+    if t.callConv == ccClosure:
+      if c.g.config.selectedGC in {gcArc, gcOrc}:
+        atomicClosureOp(c, t, body, x, y)
+      else:
+        closureOp(c, t, body, x, y)
+    else:
+      defaultOp(c, t, body, x, y)
+  of tyOwned:
+    let base = t.skipTypes(abstractInstOwned)
+    if optOwnedRefs in c.g.config.globalOptions:
+      case base.kind
+      of tyRef:
+        ownedRefOp(c, base, body, x, y)
+        return
+      of tyProc:
+        if base.callConv == ccClosure:
+          ownedClosureOp(c, base, body, x, y)
+          return
+      else: discard
+    defaultOp(c, base, body, x, y)
+  of tyArray:
+    if tfHasAsgn in t.flags or useNoGc(c, t):
+      forallElements(c, t, body, x, y)
+    else:
+      defaultOp(c, t, body, x, y)
+  of tySequence:
+    if useNoGc(c, t):
+      useSeqOrStrOp(c, t, body, x, y)
+    elif optSeqDestructors in c.g.config.globalOptions:
+      # note that tfHasAsgn is propagated so we need the check on
+      # 'selectedGC' here to determine if we have the new runtime.
+      discard considerUserDefinedOp(c, t, body, x, y)
+    elif tfHasAsgn in t.flags:
+      if c.kind in {attachedAsgn, attachedSink, attachedDeepCopy}:
+        body.add newSeqCall(c.g, x, y)
+      forallElements(c, t, body, x, y)
+    else:
+      defaultOp(c, t, body, x, y)
+  of tyString:
+    if useNoGc(c, t):
+      useSeqOrStrOp(c, t, body, x, y)
+    elif tfHasAsgn in t.flags:
+      discard considerUserDefinedOp(c, t, body, x, y)
+    else:
+      defaultOp(c, t, body, x, y)
+  of tyObject:
+    if not considerUserDefinedOp(c, t, body, x, y):
+      if c.kind in {attachedAsgn, attachedSink} and t.sym != nil and sfImportc in t.sym.flags:
+        body.add newAsgnStmt(x, y)
+      else:
+        fillBodyObjT(c, t, body, x, y)
+  of tyDistinct:
+    if not considerUserDefinedOp(c, t, body, x, y):
+      fillBody(c, t[0], body, x, y)
+  of tyTuple:
+    fillBodyTup(c, t, body, x, y)
+  of tyVarargs, tyOpenArray:
+    if c.kind == attachedDestructor and (tfHasAsgn in t.flags or useNoGc(c, t)):
+      forallElements(c, t, body, x, y)
+    else:
+      discard "cannot copy openArray"
+
+  of tyFromExpr, tyProxy, tyBuiltInTypeClass, tyUserTypeClass,
+     tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything,
+     tyGenericParam, tyGenericBody, tyNil, tyUntyped, tyTyped,
+     tyTypeDesc, tyGenericInvocation, tyForward:
+    #internalError(c.g.config, c.info, "assignment requested for type: " & typeToString(t))
+    discard
+  of tyOrdinal, tyRange, tyInferred,
+     tyGenericInst, tyStatic, tyAlias, tySink:
+    fillBody(c, lastSon(t), body, x, y)
+
+proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType;
+                            kind: TTypeAttachedOp; info: TLineInfo): PSym =
+  assert typ.kind == tyDistinct
+  let baseType = typ[0]
+  if baseType.attachedOps[kind] == nil:
+    discard produceSym(g, c, baseType, kind, info)
+  typ.attachedOps[kind] = baseType.attachedOps[kind]
+  result = typ.attachedOps[kind]
+
+proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
+              info: TLineInfo): PSym =
+  if typ.kind == tyDistinct:
+    return produceSymDistinctType(g, c, typ, kind, info)
+
+  var a: TLiftCtx
+  a.info = info
+  a.g = g
+  a.kind = kind
+  a.c = c
+  let body = newNodeI(nkStmtList, info)
+  let procname = getIdent(g.cache, AttachedOpToStr[kind])
+
+  result = newSym(skProc, procname, typ.owner, info)
+  a.fn = result
+  a.asgnForType = typ
+
+  let dest = newSym(skParam, getIdent(g.cache, "dest"), result, info)
+  let src = newSym(skParam, getIdent(g.cache, if kind == attachedTrace: "env" else: "src"), result, info)
+  var d: PNode
+  #if kind notin {attachedTrace, attachedDispose}:
+  dest.typ = makeVarType(typ.owner, typ)
+  d = newDeref(newSymNode(dest))
+  #else:
+  #  dest.typ = typ
+  #  d = newSymNode(dest)
+
+  if kind == attachedTrace:
+    src.typ = getSysType(g, info, tyPointer)
+  else:
+    src.typ = typ
+
+  result.typ = newProcType(info, typ.owner)
+  result.typ.addParam dest
+  if kind notin {attachedDestructor, attachedDispose}:
+    result.typ.addParam src
+
+  # register this operation already:
+  typ.attachedOps[kind] = result
+
+  if kind == attachedSink and typ.attachedOps[attachedDestructor] != nil and
+       sfOverriden in typ.attachedOps[attachedDestructor].flags:
+    ## compiler can use a combination of `=destroy` and memCopy for sink op
+    dest.flags.incl sfCursor
+    body.add newOpCall(typ.attachedOps[attachedDestructor], d[0])
+    body.add newAsgnStmt(d, newSymNode(src))
+  else:
+    var tk: TTypeKind
+    if g.config.selectedGC in {gcArc, gcOrc, gcHooks}:
+      tk = skipTypes(typ, {tyOrdinal, tyRange, tyInferred, tyGenericInst, tyStatic, tyAlias, tySink}).kind
+    else:
+      tk = tyNone # no special casing for strings and seqs
+    case tk
+    of tySequence:
+      fillSeqOp(a, typ, body, d, newSymNode(src))
+    of tyString:
+      fillStrOp(a, typ, body, d, newSymNode(src))
+    else:
+      fillBody(a, typ, body, d, newSymNode(src))
+
+  var n = newNodeI(nkProcDef, info, bodyPos+1)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = body
+  result.ast = n
+  incl result.flags, sfFromGeneric
+  incl result.flags, sfGeneratedOp
+
+template liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
+  discard "now a nop"
+
+proc patchBody(g: ModuleGraph; c: PContext; n: PNode; info: TLineInfo) =
+  if n.kind in nkCallKinds:
+    if n[0].kind == nkSym and n[0].sym.magic == mDestroy:
+      let t = n[1].typ.skipTypes(abstractVar)
+      if t.destructor == nil:
+        discard produceSym(g, c, t, attachedDestructor, info)
+
+      if t.destructor != nil:
+        if t.destructor.ast[genericParamsPos].kind != nkEmpty:
+          internalError(g.config, info, "resolved destructor is generic")
+        if t.destructor.magic == mDestroy:
+          internalError(g.config, info, "patching mDestroy with mDestroy?")
+        n[0] = newSymNode(t.destructor)
+  for x in n: patchBody(g, c, x, info)
+
+template inst(field, t) =
+  if field.ast != nil and field.ast[genericParamsPos].kind != nkEmpty:
+    if t.typeInst != nil:
+      var a: TLiftCtx
+      a.info = info
+      a.g = g
+      a.kind = k
+      a.c = c
+
+      field = instantiateGeneric(a, field, t, t.typeInst)
+      if field.ast != nil:
+        patchBody(g, c, field.ast, info)
+    else:
+      localError(g.config, info, "unresolved generic parameter")
+
+proc isTrival(s: PSym): bool {.inline.} =
+  s == nil or (s.ast != nil and s.ast[bodyPos].len == 0)
+
+
+proc isEmptyContainer(g: ModuleGraph, t: PType): bool =
+  (t.kind == tyArray and lengthOrd(g.config, t[0]) == 0) or
+    (t.kind == tySequence and t[0].kind == tyError)
+
+proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInfo) =
+  ## In the semantic pass this is called in strategic places
+  ## to ensure we lift assignment, destructors and moves properly.
+  ## The later 'injectdestructors' pass depends on it.
+  if orig == nil or {tfCheckedForDestructor, tfHasMeta} * orig.flags != {}: return
+  incl orig.flags, tfCheckedForDestructor
+
+  let skipped = orig.skipTypes({tyGenericInst, tyAlias, tySink})
+  if isEmptyContainer(skipped): return
+
+  let h = sighashes.hashType(skipped, {CoType, CoConsiderOwned, CoDistinct})
+  var canon = g.canonTypes.getOrDefault(h)
+  if canon == nil:
+    g.canonTypes[h] = skipped
+    canon = skipped
+
+  # multiple cases are to distinguish here:
+  # 1. we don't know yet if 'typ' has a nontrival destructor.
+  # 2. we have a nop destructor. --> mDestroy
+  # 3. we have a lifted destructor.
+  # 4. We have a custom destructor.
+  # 5. We have a (custom) generic destructor.
+
+  # we do not generate '=trace' nor '=dispose' procs if we
+  # have the cycle detection disabled, saves code size.
+  let lastAttached = if g.config.selectedGC == gcOrc: attachedDispose
+                     else: attachedSink
+
+  # we generate the destructor first so that other operators can depend on it:
+  for k in attachedDestructor..lastAttached:
+    if canon.attachedOps[k] == nil:
+      discard produceSym(g, c, canon, k, info)
+    else:
+      inst(canon.attachedOps[k], canon)
+    if canon != orig:
+      orig.attachedOps[k] = canon.attachedOps[k]
+
+  if not isTrival(orig.destructor):
+    #or not isTrival(orig.assignment) or
+    # not isTrival(orig.sink):
+    orig.flags.incl tfHasAsgn

compiler/liftlocals.nim

@@ -10,7 +10,7 @@
 ## This module implements the '.liftLocals' pragma.
 
 import
-  intsets, strutils, options, ast, astalgo, msgs,
+  strutils, options, ast, msgs,
   idents, renderer, types, lowerings, lineinfos
 
 from pragmas import getPragmaVal
@@ -30,10 +30,10 @@ proc lookupOrAdd(c: var Ctx; s: PSym; info: TLineInfo): PNode =
   let field = addUniqueField(c.objType, s, c.cache)
   var deref = newNodeI(nkHiddenDeref, info)
   deref.typ = c.objType
-  add(deref, newSymNode(c.partialParam, info))
+  deref.add(newSymNode(c.partialParam, info))
   result = newNodeI(nkDotExpr, info)
-  add(result, deref)
-  add(result, newSymNode(field))
+  result.add(deref)
+  result.add(newSymNode(field))
   result.typ = field.typ
 
 proc liftLocals(n: PNode; i: int; c: var Ctx) =
@@ -44,12 +44,12 @@ proc liftLocals(n: PNode; i: int; c: var Ctx) =
       n[i] = lookupOrAdd(c, it.sym, it.info)
   of procDefs, nkTypeSection: discard
   else:
-    for i in 0 ..< it.safeLen:
+    for i in 0..<it.safeLen:
       liftLocals(it, i, c)
 
 proc lookupParam(params, dest: PNode): PSym =
   if dest.kind != nkIdent: return nil
-  for i in 1 ..< params.len:
+  for i in 1..<params.len:
     if params[i].kind == nkSym and params[i].sym.name.id == dest.ident.id:
       return params[i].sym
 

compiler/lineinfos.nim

@@ -13,7 +13,7 @@
 import ropes, tables, pathutils
 
 const
-  explanationsBaseUrl* = "https://nim-lang.org/docs/manual"
+  explanationsBaseUrl* = "https://nim-lang.org/docs"
 
 type
   TMsgKind* = enum
@@ -23,6 +23,7 @@ type
     errGeneralParseError,
     errNewSectionExpected,
     errInvalidDirectiveX,
+    errProveInit,
     errGenerated,
     errUser,
     warnCannotOpenFile,
@@ -33,18 +34,24 @@ type
     warnFieldXNotSupported, warnCommentXIgnored,
     warnTypelessParam,
     warnUseBase, warnWriteToForeignHeap, warnUnsafeCode,
-    warnEachIdentIsTuple, warnShadowIdent,
-    warnProveInit, warnProveField, warnProveIndex, warnGcUnsafe, warnGcUnsafe2,
+    warnUnusedImportX,
+    warnInheritFromException,
+    warnEachIdentIsTuple,
+    warnUnsafeSetLen,
+    warnUnsafeDefault,
+    warnProveInit, warnProveField, warnProveIndex,
+    warnStaticIndexCheck, warnGcUnsafe, warnGcUnsafe2,
     warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed,
-    warnInconsistentSpacing, warnUser,
+    warnInconsistentSpacing, warnCaseTransition, warnCycleCreated, warnUser,
     hintSuccess, hintSuccessX, hintCC,
-    hintLineTooLong, hintXDeclaredButNotUsed, hintConvToBaseNotNeeded,
+    hintLineTooLong, hintXDeclaredButNotUsed,
+    hintConvToBaseNotNeeded,
     hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled,
     hintProcessing, hintCodeBegin, hintCodeEnd, hintConf, hintPath,
-    hintConditionAlwaysTrue, hintName, hintPattern,
+    hintConditionAlwaysTrue, hintConditionAlwaysFalse, hintName, hintPattern,
     hintExecuting, hintLinking, hintDependency,
     hintSource, hintPerformance, hintStackTrace, hintGCStats,
-    hintGlobalVar,
+    hintGlobalVar, hintExpandMacro,
     hintUser, hintUserRaw,
     hintExtendedContext
 
@@ -59,13 +66,14 @@ const
     errGeneralParseError: "general parse error",
     errNewSectionExpected: "new section expected",
     errInvalidDirectiveX: "invalid directive: '$1'",
+    errProveInit: "Cannot prove that '$1' is initialized.",
     errGenerated: "$1",
     errUser: "$1",
     warnCannotOpenFile: "cannot open '$1'",
     warnOctalEscape: "octal escape sequences do not exist; leading zero is ignored",
     warnXIsNeverRead: "'$1' is never read",
     warnXmightNotBeenInit: "'$1' might not have been initialized",
-    warnDeprecated: "$1 is deprecated",
+    warnDeprecated: "$1",
     warnConfigDeprecated: "config file '$1' is deprecated",
     warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)",
     warnUnknownMagic: "unknown magic '$1' might crash the compiler",
@@ -78,11 +86,16 @@ const
     warnUseBase: "use {.base.} for base methods; baseless methods are deprecated",
     warnWriteToForeignHeap: "write to foreign heap",
     warnUnsafeCode: "unsafe code: '$1'",
+    warnUnusedImportX: "imported and not used: '$1'",
+    warnInheritFromException: "inherit from a more precise exception type like ValueError, IOError or OSError",
     warnEachIdentIsTuple: "each identifier is a tuple",
-    warnShadowIdent: "shadowed identifier: '$1'",
+    warnUnsafeSetLen: "setLen can potentially expand the sequence, " &
+                      "but the element type '$1' doesn't have a valid default value",
+    warnUnsafeDefault: "The '$1' type doesn't have a valid default value",
     warnProveInit: "Cannot prove that '$1' is initialized. This will become a compile time error in the future.",
     warnProveField: "cannot prove that field '$1' is accessible",
     warnProveIndex: "cannot prove index '$1' is valid",
+    warnStaticIndexCheck: "$1",
     warnGcUnsafe: "not GC-safe: '$1'",
     warnGcUnsafe2: "$1",
     warnUninit: "'$1' might not have been initialized",
@@ -91,10 +104,13 @@ const
     warnLockLevel: "$1",
     warnResultShadowed: "Special variable 'result' is shadowed.",
     warnInconsistentSpacing: "Number of spaces around '$#' is not consistent",
+    warnCaseTransition: "Potential object case transition, instantiate new object instead",
+    warnCycleCreated: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
-    hintSuccessX: "operation successful ($# lines compiled; $# sec total; $#; $#)",
-    hintCC: "CC: \'$1\'", # unused
+    # keep in sync with `testament.isSuccess`
+    hintSuccessX: "$loc LOC; $sec sec; $mem; $build build; proj: $project; out: $output",
+    hintCC: "CC: $1",
     hintLineTooLong: "line too long",
     hintXDeclaredButNotUsed: "'$1' is declared but not used",
     hintConvToBaseNotNeeded: "conversion to base object is not needed",
@@ -107,16 +123,18 @@ const
     hintConf: "used config file '$1'",
     hintPath: "added path: '$1'",
     hintConditionAlwaysTrue: "condition is always true: '$1'",
-    hintName: "name should be: '$1'",
+    hintConditionAlwaysFalse: "condition is always false: '$1'",
+    hintName: "$1",
     hintPattern: "$1",
     hintExecuting: "$1",
-    hintLinking: "",
+    hintLinking: "$1",
     hintDependency: "$1",
     hintSource: "$1",
     hintPerformance: "$1",
     hintStackTrace: "$1",
     hintGCStats: "$1",
     hintGlobalVar: "global variable declared here",
+    hintExpandMacro: "expanded macro: $1",
     hintUser: "$1",
     hintUserRaw: "$1",
     hintExtendedContext: "$1",
@@ -131,17 +149,21 @@ const
     "LanguageXNotSupported", "FieldXNotSupported",
     "CommentXIgnored",
     "TypelessParam", "UseBase", "WriteToForeignHeap",
-    "UnsafeCode", "EachIdentIsTuple", "ShadowIdent",
-    "ProveInit", "ProveField", "ProveIndex", "GcUnsafe", "GcUnsafe2", "Uninit",
+    "UnsafeCode", "UnusedImport", "InheritFromException",
+    "EachIdentIsTuple",
+    "UnsafeSetLen", "UnsafeDefault",
+    "ProveInit", "ProveField", "ProveIndex",
+    "IndexCheck", "GcUnsafe", "GcUnsafe2", "Uninit",
     "GcMem", "Destructor", "LockLevel", "ResultShadowed",
-    "Spacing", "User"]
+    "Spacing", "CaseTransition", "CycleCreated", "User"]
 
   HintsToStr* = [
     "Success", "SuccessX", "CC", "LineTooLong",
-    "XDeclaredButNotUsed", "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded",
+    "XDeclaredButNotUsed",
+    "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded",
     "ExprAlwaysX", "QuitCalled", "Processing", "CodeBegin", "CodeEnd", "Conf",
-    "Path", "CondTrue", "Name", "Pattern", "Exec", "Link", "Dependency",
-    "Source", "Performance", "StackTrace", "GCStats", "GlobalVar",
+    "Path", "CondTrue", "CondFalse", "Name", "Pattern", "Exec", "Link", "Dependency",
+    "Source", "Performance", "StackTrace", "GCStats", "GlobalVar", "ExpandMacro",
     "User", "UserRaw", "ExtendedContext",
   ]
 
@@ -166,11 +188,11 @@ type
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
   result[3] = {low(TNoteKind)..high(TNoteKind)} - {}
   result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext}
-  result[1] = result[2] - {warnShadowIdent, warnProveField, warnProveIndex,
+  result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
     hintSource, hintGlobalVar, hintGCStats}
   result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf,
-    hintProcessing, hintPattern, hintExecuting, hintLinking}
+    hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC}
 
 const
   NotesVerbosity* = computeNotesVerbosity()
@@ -191,7 +213,7 @@ type
                                #   used for better error messages and
                                #   embedding the original source in the
                                #   generated code
-    dirtyfile*: AbsoluteFile   # the file that is actually read into memory
+    dirtyFile*: AbsoluteFile   # the file that is actually read into memory
                                # and parsed; usually "" but is used
                                # for 'nimsuggest'
     hash*: string              # the checksum of the file
@@ -219,27 +241,25 @@ type
   TErrorOutputs* = set[TErrorOutput]
 
   ERecoverableError* = object of ValueError
-  ESuggestDone* = object of Exception
+  ESuggestDone* = object of ValueError
 
 proc `==`*(a, b: FileIndex): bool {.borrow.}
 
-proc raiseRecoverableError*(msg: string) {.noinline, noreturn.} =
+proc raiseRecoverableError*(msg: string) {.noinline.} =
   raise newException(ERecoverableError, msg)
 
 const
-  InvalidFileIDX* = FileIndex(-1)
-
-proc unknownLineInfo*(): TLineInfo =
-  result.line = uint16(0)
-  result.col = int16(-1)
-  result.fileIndex = InvalidFileIDX
+  InvalidFileIdx* = FileIndex(-1)
+  unknownLineInfo* = TLineInfo(line: 0, col: -1, fileIndex: InvalidFileIdx)
 
 type
   Severity* {.pure.} = enum ## VS Code only supports these three
     Hint, Warning, Error
 
-const trackPosInvalidFileIdx* = FileIndex(-2) # special marker so that no suggestions
-                                   # are produced within comments and string literals
+const
+  trackPosInvalidFileIdx* = FileIndex(-2) # special marker so that no suggestions
+                                          # are produced within comments and string literals
+  commandLineIdx* = FileIndex(-3)
 
 type
   MsgConfig* = object ## does not need to be stored in the incremental cache
@@ -248,7 +268,7 @@ type
                             ## some close token.
 
     errorOutputs*: TErrorOutputs
-    msgContext*: seq[TLineInfo]
+    msgContext*: seq[tuple[info: TLineInfo, detail: string]]
     lastError*: TLineInfo
     filenameToIndexTbl*: Table[string, FileIndex]
     fileInfos*: seq[TFileInfo]
@@ -257,7 +277,7 @@ type
 
 proc initMsgConfig*(): MsgConfig =
   result.msgContext = @[]
-  result.lastError = unknownLineInfo()
+  result.lastError = unknownLineInfo
   result.filenameToIndexTbl = initTable[string, FileIndex]()
   result.fileInfos = @[]
   result.errorOutputs = {eStdOut, eStdErr}

compiler/linter.nim

@@ -7,14 +7,11 @@
 #    distribution, for details about the copyright.
 #
 
-## This module implements the code "prettifier". This is part of the toolchain
-## to convert Nim code into a consistent style.
+## This module implements the style checker.
 
-import
-  strutils, os, intsets, strtabs
+import strutils
 
-import options, ast, astalgo, msgs, semdata, ropes, idents,
-  lineinfos, pathutils
+import options, ast, msgs, idents, lineinfos, wordrecg
 
 const
   Letters* = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF', '_'}
@@ -23,36 +20,6 @@ proc identLen*(line: string, start: int): int =
   while start+result < line.len and line[start+result] in Letters:
     inc result
 
-when false:
-  import prettybase
-
-type
-  StyleCheck* {.pure.} = enum None, Warn, Auto
-
-var
-  gOverWrite* = true
-  gStyleCheck*: StyleCheck
-  gCheckExtern*, gOnlyMainfile*: bool
-
-proc overwriteFiles*(conf: ConfigRef) =
-  let doStrip = options.getConfigVar(conf, "pretty.strip").normalize == "on"
-  for i in 0 .. high(conf.m.fileInfos):
-    if conf.m.fileInfos[i].dirty and
-        (not gOnlyMainfile or FileIndex(i) == conf.projectMainIdx):
-      let newFile = if gOverWrite: conf.m.fileInfos[i].fullpath
-                    else: conf.m.fileInfos[i].fullpath.changeFileExt(".pretty.nim")
-      try:
-        var f = open(newFile.string, fmWrite)
-        for line in conf.m.fileInfos[i].lines:
-          if doStrip:
-            f.write line.strip(leading = false, trailing = true)
-          else:
-            f.write line
-          f.write(conf.m.fileInfos[i], "\L")
-        f.close
-      except IOError:
-        rawMessage(conf, errGenerated, "cannot open file: " & newFile.string)
-
 proc `=~`(s: string, a: openArray[string]): bool =
   for x in a:
     if s.startsWith(x): return true
@@ -70,10 +37,10 @@ proc beautifyName(s: string, k: TSymKind): string =
     # Types should start with a capital unless builtins like 'int' etc.:
     if s =~ ["int", "uint", "cint", "cuint", "clong", "cstring", "string",
              "char", "byte", "bool", "openArray", "seq", "array", "void",
-             "pointer", "float", "csize", "cdouble", "cchar", "cschar",
-             "cshort", "cu", "nil", "expr", "stmt", "typedesc", "auto", "any",
-             "range", "openarray", "varargs", "set", "cfloat"
-             ]:
+             "pointer", "float", "csize", "csize_t", "cdouble", "cchar", "cschar",
+             "cshort", "cu", "nil", "typedesc", "auto", "any",
+             "range", "openarray", "varargs", "set", "cfloat", "ref", "ptr",
+             "untyped", "typed", "static", "sink", "lent", "type", "owned"]:
       result.add s[i]
     else:
       result.add toUpperAscii(s[i])
@@ -102,75 +69,73 @@ proc beautifyName(s: string, k: TSymKind): string =
       result.add s[i]
     inc i
 
-proc differ*(line: string, a, b: int, x: string): bool =
-  let y = line[a..b]
-  result = cmpIgnoreStyle(y, x) == 0 and y != x
+proc differ*(line: string, a, b: int, x: string): string =
+  proc substrEq(s: string, pos, last: int, substr: string): bool =
+    var i = 0
+    while i < substr.len and pos+i <= last and s[pos+i] == substr[i]:
+      inc i
+    return i == substr.len
 
-proc replaceInFile(conf: ConfigRef; info: TLineInfo; newName: string) =
-  let line = conf.m.fileInfos[info.fileIndex.int].lines[info.line.int-1]
-  var first = min(info.col.int, line.len)
-  if first < 0: return
-  #inc first, skipIgnoreCase(line, "proc ", first)
-  while first > 0 and line[first-1] in Letters: dec first
-  if first < 0: return
-  if line[first] == '`': inc first
+  let last = min(b, line.len)
 
-  let last = first+identLen(line, first)-1
-  if differ(line, first, last, newName):
-    # last-first+1 != newName.len or
-    var x = line.substr(0, first-1) & newName & line.substr(last+1)
-    system.shallowCopy(conf.m.fileInfos[info.fileIndex.int].lines[info.line.int-1], x)
-    conf.m.fileInfos[info.fileIndex.int].dirty = true
+  result = ""
+  if not substrEq(line, a, b, x):
+    let y = line[a..b]
+    if cmpIgnoreStyle(y, x) == 0:
+      result = y
 
 proc checkStyle(conf: ConfigRef; cache: IdentCache; info: TLineInfo, s: string, k: TSymKind; sym: PSym) =
   let beau = beautifyName(s, k)
   if s != beau:
-    if gStyleCheck == StyleCheck.Auto:
-      sym.name = getIdent(cache, beau)
-      replaceInFile(conf, info, beau)
-    else:
-      message(conf, info, hintName, beau)
-
-proc styleCheckDefImpl(conf: ConfigRef; cache: IdentCache; info: TLineInfo; s: PSym; k: TSymKind) =
-  # operators stay as they are:
-  if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
-  if k in {skType, skGenericParam} and sfAnon in s.flags: return
-  if {sfImportc, sfExportc} * s.flags == {} or gCheckExtern:
-    checkStyle(conf, cache, info, s.name.s, k, s)
+    lintReport(conf, info, beau, s)
 
 proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
   # operators stay as they are:
   if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
   if k in {skType, skGenericParam} and sfAnon in s.flags: return
+  if s.typ != nil and s.typ.kind == tyTypeDesc: return
+  if {sfImportc, sfExportc} * s.flags != {}: return
+  if optStyleCheck notin s.options: return
   let beau = beautifyName(s.name.s, k)
   if s.name.s != beau:
-    message(conf, info, hintName, beau)
+    lintReport(conf, info, beau, s.name.s)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
-  if optCheckNep1 in conf.globalOptions:
+  if {optStyleHint, optStyleError} * conf.globalOptions != {}:
     nep1CheckDefImpl(conf, info, s, k)
-  when defined(nimfix):
-    if gStyleCheck != StyleCheck.None: styleCheckDefImpl(conf, cache, info, s, k)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) =
   styleCheckDef(conf, info, s, s.kind)
 template styleCheckDef*(conf: ConfigRef; s: PSym) =
   styleCheckDef(conf, s.info, s, s.kind)
 
-proc styleCheckUseImpl(conf: ConfigRef; info: TLineInfo; s: PSym) =
+proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string =
+  let line = sourceLine(conf, info)
+  var first = min(info.col.int, line.len)
+  if first < 0: return
+  #inc first, skipIgnoreCase(line, "proc ", first)
+  while first > 0 and line[first-1] in Letters: dec first
+  if first < 0: return
+  if first+1 < line.len and line[first] == '`': inc first
+
+  let last = first+identLen(line, first)-1
+  result = differ(line, first, last, newName)
+
+proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) =
   if info.fileIndex.int < 0: return
   # we simply convert it to what it looks like in the definition
   # for consistency
 
   # operators stay as they are:
-  if s.kind in {skResult, skTemp} or s.name.s[0] notin Letters:
+  if s.kind == skTemp or s.name.s[0] notin Letters or sfAnon in s.flags:
     return
-  if s.kind in {skType, skGenericParam} and sfAnon in s.flags: return
+
   let newName = s.name.s
+  let oldName = differs(conf, info, newName)
+  if oldName.len > 0:
+    lintReport(conf, info, newName, oldName)
 
-  replaceInFile(conf, info, newName)
-  #if newName == "File": writeStackTrace()
-
-template styleCheckUse*(info: TLineInfo; s: PSym) =
-  when defined(nimfix):
-    if gStyleCheck != StyleCheck.None: styleCheckUseImpl(conf, info, s)
+proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
+  let wanted = canonPragmaSpelling(w)
+  if pragmaName != wanted:
+    lintReport(conf, info, wanted, pragmaName)

compiler/lists.nim

@@ -1,28 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2012 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-# This module is deprecated, don't use it.
-# TODO Remove this
-
-import os
-
-static:
-  echo "WARNING: imported deprecated module compiler/lists.nim, use seq ore lists from the standard library"
-
-proc appendStr*(list: var seq[string]; data: string) {.deprecated.} =
-  # just use system.add
-  list.add(data)
-
-proc includeStr(list: var seq[string]; data: string): bool {.deprecated.} =
-  if list.contains(data):
-    result = true
-  else:
-    result = false
-    list.add data
-

compiler/llstream.nim

@@ -10,13 +10,16 @@
 ## Low-level streams for high performance.
 
 import
-  strutils, pathutils
+  pathutils
 
-# support '-d:useGnuReadline' for backwards compatibility:
-when not defined(windows) and (defined(useGnuReadline) or defined(useLinenoise)):
-  import rdstdin
+# support `useGnuReadline`, `useLinenoise` for backwards compatibility
+const hasRstdin = (defined(nimUseLinenoise) or defined(useLinenoise) or defined(useGnuReadline)) and
+  not defined(windows)
+
+when hasRstdin: import rdstdin
 
 type
+  TLLRepl* = proc (s: PLLStream, buf: pointer, bufLen: int): int
   TLLStreamKind* = enum       # enum of different stream implementations
     llsNone,                  # null stream: reading and writing has no effect
     llsString,                # stream encapsulates a string
@@ -28,6 +31,7 @@ type
     s*: string
     rd*, wr*: int             # for string streams
     lineOffset*: int          # for fake stdin line numbers
+    repl*: TLLRepl            # gives stdin control to clients
 
   PLLStream* = ref TLLStream
 
@@ -50,11 +54,13 @@ proc llStreamOpen*(): PLLStream =
   new(result)
   result.kind = llsNone
 
-proc llStreamOpenStdIn*(): PLLStream =
+proc llReadFromStdin(s: PLLStream, buf: pointer, bufLen: int): int
+proc llStreamOpenStdIn*(r: TLLRepl = llReadFromStdin): PLLStream =
   new(result)
   result.kind = llsStdIn
   result.s = ""
   result.lineOffset = -1
+  result.repl = r
 
 proc llStreamClose*(s: PLLStream) =
   case s.kind
@@ -66,10 +72,10 @@ proc llStreamClose*(s: PLLStream) =
 when not declared(readLineFromStdin):
   # fallback implementation:
   proc readLineFromStdin(prompt: string, line: var string): bool =
-    stdout.write(prompt)
+    stderr.write(prompt)
     result = readLine(stdin, line)
     if not result:
-      stdout.write("\n")
+      stderr.write("\n")
       quit(0)
 
 proc endsWith*(x: string, s: set[char]): bool =
@@ -93,7 +99,7 @@ proc continueLine(line: string, inTripleString: bool): bool {.inline.} =
 
 proc countTriples(s: string): int =
   var i = 0
-  while i < s.len:
+  while i+2 < s.len:
     if s[i] == '"' and s[i+1] == '"' and s[i+2] == '"':
       inc result
       inc i, 2
@@ -105,12 +111,12 @@ proc llReadFromStdin(s: PLLStream, buf: pointer, bufLen: int): int =
   var line = newStringOfCap(120)
   var triples = 0
   while readLineFromStdin(if s.s.len == 0: ">>> " else: "... ", line):
-    add(s.s, line)
-    add(s.s, "\n")
+    s.s.add(line)
+    s.s.add("\n")
     inc triples, countTriples(line)
     if not continueLine(line, (triples and 1) == 1): break
   inc(s.lineOffset)
-  result = min(bufLen, len(s.s) - s.rd)
+  result = min(bufLen, s.s.len - s.rd)
   if result > 0:
     copyMem(buf, addr(s.s[s.rd]), result)
     inc(s.rd, result)
@@ -120,14 +126,14 @@ proc llStreamRead*(s: PLLStream, buf: pointer, bufLen: int): int =
   of llsNone:
     result = 0
   of llsString:
-    result = min(bufLen, len(s.s) - s.rd)
+    result = min(bufLen, s.s.len - s.rd)
     if result > 0:
       copyMem(buf, addr(s.s[0 + s.rd]), result)
       inc(s.rd, result)
   of llsFile:
     result = readBuffer(s.f, buf, bufLen)
   of llsStdIn:
-    result = llReadFromStdin(s, buf, bufLen)
+    result = s.repl(s, buf, bufLen)
 
 proc llStreamReadLine*(s: PLLStream, line: var string): bool =
   setLen(line, 0)
@@ -135,7 +141,7 @@ proc llStreamReadLine*(s: PLLStream, line: var string): bool =
   of llsNone:
     result = true
   of llsString:
-    while s.rd < len(s.s):
+    while s.rd < s.s.len:
       case s.s[s.rd]
       of '\x0D':
         inc(s.rd)
@@ -145,9 +151,9 @@ proc llStreamReadLine*(s: PLLStream, line: var string): bool =
         inc(s.rd)
         break
       else:
-        add(line, s.s[s.rd])
+        line.add(s.s[s.rd])
         inc(s.rd)
-    result = line.len > 0 or s.rd < len(s.s)
+    result = line.len > 0 or s.rd < s.s.len
   of llsFile:
     result = readLine(s.f, line)
   of llsStdIn:
@@ -158,8 +164,8 @@ proc llStreamWrite*(s: PLLStream, data: string) =
   of llsNone, llsStdIn:
     discard
   of llsString:
-    add(s.s, data)
-    inc(s.wr, len(data))
+    s.s.add(data)
+    inc(s.wr, data.len)
   of llsFile:
     write(s.f, data)
 
@@ -173,7 +179,7 @@ proc llStreamWrite*(s: PLLStream, data: char) =
   of llsNone, llsStdIn:
     discard
   of llsString:
-    add(s.s, data)
+    s.s.add(data)
     inc(s.wr)
   of llsFile:
     c = data
@@ -185,7 +191,7 @@ proc llStreamWrite*(s: PLLStream, buf: pointer, buflen: int) =
     discard
   of llsString:
     if buflen > 0:
-      setLen(s.s, len(s.s) + buflen)
+      setLen(s.s, s.s.len + buflen)
       copyMem(addr(s.s[0 + s.wr]), buf, buflen)
       inc(s.wr, buflen)
   of llsFile:
@@ -200,7 +206,7 @@ proc llStreamReadAll*(s: PLLStream): string =
   of llsString:
     if s.rd == 0: result = s.s
     else: result = substr(s.s, s.rd)
-    s.rd = len(s.s)
+    s.rd = s.s.len
   of llsFile:
     result = newString(bufSize)
     var bytes = readBuffer(s.f, addr(result[0]), bufSize)

compiler/lookups.nim

@@ -11,7 +11,7 @@
 
 import
   intsets, ast, astalgo, idents, semdata, types, msgs, options,
-  renderer, wordrecg, idgen, nimfix/prettybase, lineinfos, strutils
+  renderer, nimfix/prettybase, lineinfos, strutils
 
 proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope)
 
@@ -36,11 +36,11 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
   of nkAccQuoted:
     case n.len
     of 0: handleError(n, origin)
-    of 1: result = considerQuotedIdent(c, n.sons[0], origin)
+    of 1: result = considerQuotedIdent(c, n[0], origin)
     else:
       var id = ""
       for i in 0..<n.len:
-        let x = n.sons[i]
+        let x = n[i]
         case x.kind
         of nkIdent: id.add(x.ident.s)
         of nkSym: id.add(x.sym.name.s)
@@ -49,7 +49,7 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
       result = getIdent(c.cache, id)
   of nkOpenSymChoice, nkClosedSymChoice:
     if n[0].kind == nkSym:
-      result = n.sons[0].sym.name
+      result = n[0].sym.name
     else:
       handleError(n, origin)
   else:
@@ -89,7 +89,7 @@ proc skipAlias*(s: PSym; n: PNode; conf: ConfigRef): PSym =
       prettybase.replaceDeprecated(conf, n.info, s, result)
     else:
       message(conf, n.info, warnDeprecated, "use " & result.name.s & " instead; " &
-              s.name.s)
+              s.name.s & " is deprecated")
 
 proc localSearchInScope*(c: PContext, s: PIdent): PSym =
   result = strTableGet(c.currentScope.symbols, s)
@@ -100,16 +100,15 @@ proc searchInScopes*(c: PContext, s: PIdent): PSym =
     if result != nil: return
   result = nil
 
-when declared(echo):
-  proc debugScopes*(c: PContext; limit=0) {.deprecated.} =
-    var i = 0
-    for scope in walkScopes(c.currentScope):
-      echo "scope ", i
-      for h in 0 .. high(scope.symbols.data):
-        if scope.symbols.data[h] != nil:
-          echo scope.symbols.data[h].name.s
-      if i == limit: break
-      inc i
+proc debugScopes*(c: PContext; limit=0) {.deprecated.} =
+  var i = 0
+  for scope in walkScopes(c.currentScope):
+    echo "scope ", i
+    for h in 0..high(scope.symbols.data):
+      if scope.symbols.data[h] != nil:
+        echo scope.symbols.data[h].name.s
+    if i == limit: break
+    inc i
 
 proc searchInScopes*(c: PContext, s: PIdent, filter: TSymKinds): PSym =
   for scope in walkScopes(c.currentScope):
@@ -124,7 +123,7 @@ proc errorSym*(c: PContext, n: PNode): PSym =
   ## creates an error symbol to avoid cascading errors (for IDE support)
   var m = n
   # ensure that 'considerQuotedIdent' can't fail:
-  if m.kind == nkDotExpr: m = m.sons[1]
+  if m.kind == nkDotExpr: m = m[1]
   let ident = if m.kind in {nkIdent, nkSym, nkAccQuoted}:
       considerQuotedIdent(c, m)
     else:
@@ -150,7 +149,7 @@ type
 
 proc getSymRepr*(conf: ConfigRef; s: PSym): string =
   case s.kind
-  of skProc, skFunc, skMethod, skConverter, skIterator:
+  of routineKinds, skType:
     result = getProcHeader(conf, s)
   else:
     result = s.name.s
@@ -168,13 +167,13 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) =
         localError(c.config, s.info, "implementation of '$1' expected" %
             getSymRepr(c.config, s))
       inc missingImpls
-    elif {sfUsed, sfExported} * s.flags == {} and optHints in s.options:
-      if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam}:
+    elif {sfUsed, sfExported} * s.flags == {}:
+      if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam, skEnumField}:
         # XXX: implicit type params are currently skTypes
         # maybe they can be made skGenericParam as well.
         if s.typ != nil and tfImplicitTypeParam notin s.typ.flags and
            s.typ.kind != tyGenericParam:
-          message(c.config, s.info, hintXDeclaredButNotUsed, getSymRepr(c.config, s))
+          message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s)
     s = nextIter(it, scope.symbols)
 
 proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string;
@@ -190,7 +189,7 @@ proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) =
     wrongRedefinition(c, info, sym.name.s, conflict.info)
 
 proc addDecl*(c: PContext, sym: PSym) =
-  let conflict = c.currentScope.addUniqueSym(sym)
+  let conflict = strTableInclReportConflict(c.currentScope.symbols, sym, true)
   if conflict != nil:
     wrongRedefinition(c, sym.info, sym.name.s, conflict.info)
 
@@ -233,7 +232,7 @@ proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) =
   addInterfaceDeclAux(c, sym)
 
 when defined(nimfix):
-  # when we cannot find the identifier, retry with a changed identifer:
+  # when we cannot find the identifier, retry with a changed identifier:
   proc altSpelling(x: PIdent): PIdent =
     case x.s[0]
     of 'A'..'Z': result = getIdent(toLowerAscii(x.s[0]) & x.s.substr(1))
@@ -255,9 +254,10 @@ proc errorUseQualifier*(c: PContext; info: TLineInfo; s: PSym) =
   var candidate = initIdentIter(ti, c.importTable.symbols, s.name)
   var i = 0
   while candidate != nil:
-    if i == 0: err.add " --use "
-    else: err.add " or "
-    err.add candidate.owner.name.s & "." & candidate.name.s
+    if i == 0: err.add " -- use one of the following:\n"
+    else: err.add "\n"
+    err.add "  " & candidate.owner.name.s & "." & candidate.name.s
+    err.add ": " & typeToString(candidate.typ)
     candidate = nextIdentIter(ti, c.importTable.symbols)
     inc i
   localError(c.config, info, errGenerated, err)
@@ -325,29 +325,29 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
       errorUseQualifier(c, n.info, n.sym)
   of nkDotExpr:
     result = nil
-    var m = qualifiedLookUp(c, n.sons[0], (flags*{checkUndeclared})+{checkModule})
+    var m = qualifiedLookUp(c, n[0], (flags*{checkUndeclared})+{checkModule})
     if m != nil and m.kind == skModule:
       var ident: PIdent = nil
-      if n.sons[1].kind == nkIdent:
-        ident = n.sons[1].ident
-      elif n.sons[1].kind == nkAccQuoted:
-        ident = considerQuotedIdent(c, n.sons[1])
+      if n[1].kind == nkIdent:
+        ident = n[1].ident
+      elif n[1].kind == nkAccQuoted:
+        ident = considerQuotedIdent(c, n[1])
       if ident != nil:
         if m == c.module:
           result = strTableGet(c.topLevelScope.symbols, ident).skipAlias(n, c.config)
         else:
           result = strTableGet(m.tab, ident).skipAlias(n, c.config)
         if result == nil and checkUndeclared in flags:
-          fixSpelling(n.sons[1], ident, searchInScopes)
-          errorUndeclaredIdentifier(c, n.sons[1].info, ident.s)
-          result = errorSym(c, n.sons[1])
-      elif n.sons[1].kind == nkSym:
-        result = n.sons[1].sym
+          fixSpelling(n[1], ident, searchInScopes)
+          errorUndeclaredIdentifier(c, n[1].info, ident.s)
+          result = errorSym(c, n[1])
+      elif n[1].kind == nkSym:
+        result = n[1].sym
       elif checkUndeclared in flags and
-           n.sons[1].kind notin {nkOpenSymChoice, nkClosedSymChoice}:
-        localError(c.config, n.sons[1].info, "identifier expected, but got: " &
-                   renderTree(n.sons[1]))
-        result = errorSym(c, n.sons[1])
+           n[1].kind notin {nkOpenSymChoice, nkClosedSymChoice}:
+        localError(c.config, n[1].info, "identifier expected, but got: " &
+                   renderTree(n[1]))
+        result = errorSym(c, n[1])
   else:
     result = nil
   when false:
@@ -371,13 +371,13 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
     o.mode = oimDone
   of nkDotExpr:
     o.mode = oimOtherModule
-    o.m = qualifiedLookUp(c, n.sons[0], {checkUndeclared, checkModule})
+    o.m = qualifiedLookUp(c, n[0], {checkUndeclared, checkModule})
     if o.m != nil and o.m.kind == skModule:
       var ident: PIdent = nil
-      if n.sons[1].kind == nkIdent:
-        ident = n.sons[1].ident
-      elif n.sons[1].kind == nkAccQuoted:
-        ident = considerQuotedIdent(c, n.sons[1], n)
+      if n[1].kind == nkIdent:
+        ident = n[1].ident
+      elif n[1].kind == nkAccQuoted:
+        ident = considerQuotedIdent(c, n[1], n)
       if ident != nil:
         if o.m == c.module:
           # a module may access its private members:
@@ -387,12 +387,12 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         else:
           result = initIdentIter(o.it, o.m.tab, ident).skipAlias(n, c.config)
       else:
-        noidentError(c.config, n.sons[1], n)
-        result = errorSym(c, n.sons[1])
+        noidentError(c.config, n[1], n)
+        result = errorSym(c, n[1])
   of nkClosedSymChoice, nkOpenSymChoice:
     o.mode = oimSymChoice
     if n[0].kind == nkSym:
-      result = n.sons[0].sym
+      result = n[0].sym
     else:
       o.mode = oimDone
       return nil
@@ -429,8 +429,8 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
   of oimOtherModule:
     result = nextIdentIter(o.it, o.m.tab).skipAlias(n, c.config)
   of oimSymChoice:
-    if o.symChoiceIndex < sonsLen(n):
-      result = n.sons[o.symChoiceIndex].sym
+    if o.symChoiceIndex < n.len:
+      result = n[o.symChoiceIndex].sym
       incl(o.inSymChoice, result.id)
       inc o.symChoiceIndex
     elif n.kind == nkOpenSymChoice:
@@ -438,19 +438,19 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
       o.mode = oimSymChoiceLocalLookup
       o.scope = c.currentScope
       result = firstIdentExcluding(o.it, o.scope.symbols,
-                                   n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
       while result == nil:
         o.scope = o.scope.parent
         if o.scope == nil: break
         result = firstIdentExcluding(o.it, o.scope.symbols,
-                                     n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                     n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
   of oimSymChoiceLocalLookup:
     result = nextIdentExcluding(o.it, o.scope.symbols, o.inSymChoice).skipAlias(n, c.config)
     while result == nil:
       o.scope = o.scope.parent
       if o.scope == nil: break
       result = firstIdentExcluding(o.it, o.scope.symbols,
-                                   n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
 
   when false:
     if result != nil and result.kind == skStub: loadStub(result)

compiler/lowerings.nim

@@ -14,36 +14,50 @@ const
 
 import ast, astalgo, types, idents, magicsys, msgs, options, modulegraphs,
   lineinfos
-from trees import getMagic
 
 proc newDeref*(n: PNode): PNode {.inline.} =
-  result = newNodeIT(nkHiddenDeref, n.info, n.typ.sons[0])
-  addSon(result, n)
+  result = newNodeIT(nkHiddenDeref, n.info, n.typ[0])
+  result.add n
 
 proc newTupleAccess*(g: ModuleGraph; tup: PNode, i: int): PNode =
-  result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(
-                     abstractInst).sons[i])
-  addSon(result, copyTree(tup))
-  var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
-  lit.intVal = i
-  addSon(result, lit)
+  if tup.kind == nkHiddenAddr:
+    result = newNodeIT(nkHiddenAddr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar, tyLent}))
+    result.add newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar, tyLent})[i])
+    result[0].add tup[0]
+    var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
+    lit.intVal = i
+    result[0].add lit
+  else:
+    result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(
+                       abstractInst)[i])
+    result.add copyTree(tup)
+    var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
+    lit.intVal = i
+    result.add lit
 
 proc addVar*(father, v: PNode) =
   var vpart = newNodeI(nkIdentDefs, v.info, 3)
-  vpart.sons[0] = v
-  vpart.sons[1] = newNodeI(nkEmpty, v.info)
-  vpart.sons[2] = vpart[1]
-  addSon(father, vpart)
+  vpart[0] = v
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = vpart[1]
+  father.add vpart
 
-proc newAsgnStmt(le, ri: PNode): PNode =
+proc addVar*(father, v, value: PNode) =
+  var vpart = newNodeI(nkIdentDefs, v.info, 3)
+  vpart[0] = v
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = value
+  father.add vpart
+
+proc newAsgnStmt*(le, ri: PNode): PNode =
   result = newNodeI(nkAsgn, le.info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
 
-proc newFastAsgnStmt(le, ri: PNode): PNode =
+proc newFastAsgnStmt*(le, ri: PNode): PNode =
   result = newNodeI(nkFastAsgn, le.info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
 
 proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   assert n.kind == nkVarTuple
@@ -56,58 +70,74 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
 
   var v = newNodeI(nkVarSection, value.info)
   let tempAsNode = newSymNode(temp)
-  v.addVar(tempAsNode)
+  v.addVar(tempAsNode, value)
   result.add(v)
 
+  for i in 0..<n.len-2:
+    let val = newTupleAccess(g, tempAsNode, i)
+    if n[i].kind == nkSym: v.addVar(n[i], val)
+    else: result.add newAsgnStmt(n[i], val)
+
+proc evalOnce*(g: ModuleGraph; value: PNode; owner: PSym): PNode =
+  ## Turns (value) into (let tmp = value; tmp) so that 'value' can be re-used
+  ## freely, multiple times. This is frequently required and such a builtin would also be
+  ## handy to have in macros.nim. The value that can be reused is 'result.lastSon'!
+  result = newNodeIT(nkStmtListExpr, value.info, value.typ)
+  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), owner, value.info, g.config.options)
+  temp.typ = skipTypes(value.typ, abstractInst)
+  incl(temp.flags, sfFromGeneric)
+
+  var v = newNodeI(nkLetSection, value.info)
+  let tempAsNode = newSymNode(temp)
+  v.addVar(tempAsNode)
+  result.add(v)
   result.add newAsgnStmt(tempAsNode, value)
-  for i in 0 .. n.len-3:
-    if n.sons[i].kind == nkSym: v.addVar(n.sons[i])
-    result.add newAsgnStmt(n.sons[i], newTupleAccess(g, tempAsNode, i))
+  result.add tempAsNode
 
 proc newTupleAccessRaw*(tup: PNode, i: int): PNode =
   result = newNodeI(nkBracketExpr, tup.info)
-  addSon(result, copyTree(tup))
+  result.add copyTree(tup)
   var lit = newNodeI(nkIntLit, tup.info)
   lit.intVal = i
-  addSon(result, lit)
+  result.add lit
 
 proc newTryFinally*(body, final: PNode): PNode =
-  result = newTree(nkTryStmt, body, newTree(nkFinally, final))
+  result = newTree(nkHiddenTryStmt, body, newTree(nkFinally, final))
 
 proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   let value = n.lastSon
   result = newNodeI(nkStmtList, n.info)
 
-  var temp = newSym(skLet, getIdent(g.cache, "_"), owner, value.info, owner.options)
+  var temp = newSym(skTemp, getIdent(g.cache, "_"), owner, value.info, owner.options)
   var v = newNodeI(nkLetSection, value.info)
   let tempAsNode = newSymNode(temp) #newIdentNode(getIdent(genPrefix & $temp.id), value.info)
 
   var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
-  vpart.sons[0] = tempAsNode
-  vpart.sons[1] = newNodeI(nkEmpty, value.info)
-  vpart.sons[2] = value
-  addSon(v, vpart)
+  vpart[0] = tempAsNode
+  vpart[1] = newNodeI(nkEmpty, value.info)
+  vpart[2] = value
+  v.add vpart
   result.add(v)
 
-  let lhs = n.sons[0]
-  for i in 0 .. lhs.len-1:
-    result.add newAsgnStmt(lhs.sons[i], newTupleAccessRaw(tempAsNode, i))
+  let lhs = n[0]
+  for i in 0..<lhs.len:
+    result.add newAsgnStmt(lhs[i], newTupleAccessRaw(tempAsNode, i))
 
 proc lowerSwap*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   result = newNodeI(nkStmtList, n.info)
   # note: cannot use 'skTemp' here cause we really need the copy for the VM :-(
   var temp = newSym(skVar, getIdent(g.cache, genPrefix), owner, n.info, owner.options)
-  temp.typ = n.sons[1].typ
+  temp.typ = n[1].typ
   incl(temp.flags, sfFromGeneric)
 
   var v = newNodeI(nkVarSection, n.info)
   let tempAsNode = newSymNode(temp)
 
   var vpart = newNodeI(nkIdentDefs, v.info, 3)
-  vpart.sons[0] = tempAsNode
-  vpart.sons[1] = newNodeI(nkEmpty, v.info)
-  vpart.sons[2] = n[1]
-  addSon(v, vpart)
+  vpart[0] = tempAsNode
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = n[1]
+  v.add vpart
 
   result.add(v)
   result.add newFastAsgnStmt(n[1], n[2])
@@ -127,45 +157,53 @@ proc createObj*(g: ModuleGraph; owner: PSym, info: TLineInfo; final=true): PType
   s.typ = result
   result.sym = s
 
+template fieldCheck {.dirty.} =
+  when false:
+    if tfCheckedForDestructor in obj.flags:
+      echo "missed field ", field.name.s
+      writeStackTrace()
+
 proc rawAddField*(obj: PType; field: PSym) =
   assert field.kind == skField
-  field.position = sonsLen(obj.n)
-  addSon(obj.n, newSymNode(field))
+  field.position = obj.n.len
+  obj.n.add newSymNode(field)
+  propagateToOwner(obj, field.typ)
+  fieldCheck()
 
 proc rawIndirectAccess*(a: PNode; field: PSym; info: TLineInfo): PNode =
   # returns a[].field as a node
   assert field.kind == skField
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
-  addSon(deref, a)
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc rawDirectAccess*(obj, field: PSym): PNode =
   # returns a.field as a node
   assert field.kind == skField
   result = newNodeI(nkDotExpr, field.info)
-  addSon(result, newSymNode obj)
-  addSon(result, newSymNode field)
+  result.add newSymNode(obj)
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc lookupInRecord(n: PNode, id: int): PSym =
   result = nil
   case n.kind
   of nkRecList:
-    for i in countup(0, sonsLen(n) - 1):
-      result = lookupInRecord(n.sons[i], id)
+    for i in 0..<n.len:
+      result = lookupInRecord(n[i], id)
       if result != nil: return
   of nkRecCase:
-    if n.sons[0].kind != nkSym: return
-    result = lookupInRecord(n.sons[0], id)
+    if n[0].kind != nkSym: return
+    result = lookupInRecord(n[0], id)
     if result != nil: return
-    for i in countup(1, sonsLen(n) - 1):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = lookupInRecord(lastSon(n.sons[i]), id)
+        result = lookupInRecord(lastSon(n[i]), id)
         if result != nil: return
       else: discard
   of nkSym:
@@ -180,9 +218,12 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache) =
   field.id = -s.id
   let t = skipIntLit(s.typ)
   field.typ = t
-  assert t.kind != tyStmt
-  field.position = sonsLen(obj.n)
-  addSon(obj.n, newSymNode(field))
+  assert t.kind != tyTyped
+  propagateToOwner(obj, t)
+  field.position = obj.n.len
+  field.flags = s.flags * {sfCursor}
+  obj.n.add newSymNode(field)
+  fieldCheck()
 
 proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache): PSym {.discardable.} =
   result = lookupInRecord(obj.n, s.id)
@@ -192,46 +233,47 @@ proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache): PSym {.discardable
     field.id = -s.id
     let t = skipIntLit(s.typ)
     field.typ = t
-    assert t.kind != tyStmt
-    field.position = sonsLen(obj.n)
-    addSon(obj.n, newSymNode(field))
+    assert t.kind != tyTyped
+    propagateToOwner(obj, t)
+    field.position = obj.n.len
+    obj.n.add newSymNode(field)
     result = field
 
-proc newDotExpr(obj, b: PSym): PNode =
+proc newDotExpr*(obj, b: PSym): PNode =
   result = newNodeI(nkDotExpr, obj.info)
   let field = lookupInRecord(obj.typ.n, b.id)
   assert field != nil, b.name.s
-  addSon(result, newSymNode(obj))
-  addSon(result, newSymNode(field))
+  result.add newSymNode(obj)
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc indirectAccess*(a: PNode, b: int, info: TLineInfo): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
   while true:
     assert t.kind == tyObject
     field = lookupInRecord(t.n, b)
     if field != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
   #if field == nil:
   #  echo "FIELD ", b
   #  debug deref.typ
   assert field != nil
-  addSon(deref, a)
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
-proc indirectAccess(a: PNode, b: string, info: TLineInfo; cache: IdentCache): PNode =
+proc indirectAccess*(a: PNode, b: string, info: TLineInfo; cache: IdentCache): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
   let bb = getIdent(cache, b)
@@ -239,17 +281,17 @@ proc indirectAccess(a: PNode, b: string, info: TLineInfo; cache: IdentCache): PN
     assert t.kind == tyObject
     field = getSymFromList(t.n, bb)
     if field != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
   #if field == nil:
   #  echo "FIELD ", b
   #  debug deref.typ
   assert field != nil
-  addSon(deref, a)
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc getFieldFromObj*(t: PType; v: PSym): PSym =
@@ -259,7 +301,7 @@ proc getFieldFromObj*(t: PType; v: PSym): PSym =
     assert t.kind == tyObject
     result = lookupInRecord(t.n, v.id)
     if result != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
 
@@ -270,257 +312,33 @@ proc indirectAccess*(a: PNode, b: PSym, info: TLineInfo): PNode =
 proc indirectAccess*(a, b: PSym, info: TLineInfo): PNode =
   result = indirectAccess(newSymNode(a), b, info)
 
-proc genAddrOf*(n: PNode): PNode =
+proc genAddrOf*(n: PNode, typeKind = tyPtr): PNode =
   result = newNodeI(nkAddr, n.info, 1)
-  result.sons[0] = n
-  result.typ = newType(tyPtr, n.typ.owner)
+  result[0] = n
+  result.typ = newType(typeKind, n.typ.owner)
   result.typ.rawAddSon(n.typ)
 
-proc genDeref*(n: PNode): PNode =
-  result = newNodeIT(nkHiddenDeref, n.info,
-                     n.typ.skipTypes(abstractInst).sons[0])
+proc genDeref*(n: PNode; k = nkHiddenDeref): PNode =
+  result = newNodeIT(k, n.info,
+                     n.typ.skipTypes(abstractInst)[0])
   result.add n
 
-proc callCodegenProc*(g: ModuleGraph; name: string, arg1: PNode;
-                      arg2, arg3, optionalArgs: PNode = nil): PNode =
-  result = newNodeI(nkCall, arg1.info)
+proc callCodegenProc*(g: ModuleGraph; name: string;
+                      info: TLineInfo = unknownLineInfo;
+                      arg1, arg2, arg3, optionalArgs: PNode = nil): PNode =
+  result = newNodeI(nkCall, info)
   let sym = magicsys.getCompilerProc(g, name)
   if sym == nil:
-    localError(g.config, arg1.info, "system module needs: " & name)
+    localError(g.config, info, "system module needs: " & name)
   else:
     result.add newSymNode(sym)
-    result.add arg1
+    if arg1 != nil: result.add arg1
     if arg2 != nil: result.add arg2
     if arg3 != nil: result.add arg3
     if optionalArgs != nil:
-      for i in 1..optionalArgs.len-3:
+      for i in 1..<optionalArgs.len-2:
         result.add optionalArgs[i]
-    result.typ = sym.typ.sons[0]
-
-proc callProc(a: PNode): PNode =
-  result = newNodeI(nkCall, a.info)
-  result.add a
-  result.typ = a.typ.sons[0]
-
-# we have 4 cases to consider:
-# - a void proc --> nothing to do
-# - a proc returning GC'ed memory --> requires a flowVar
-# - a proc returning non GC'ed memory --> pass as hidden 'var' parameter
-# - not in a parallel environment --> requires a flowVar for memory safety
-type
-  TSpawnResult* = enum
-    srVoid, srFlowVar, srByVar
-  TFlowVarKind = enum
-    fvInvalid # invalid type T for 'FlowVar[T]'
-    fvGC      # FlowVar of a GC'ed type
-    fvBlob    # FlowVar of a blob type
-
-proc spawnResult*(t: PType; inParallel: bool): TSpawnResult =
-  if t.isEmptyType: srVoid
-  elif inParallel and not containsGarbageCollectedRef(t): srByVar
-  else: srFlowVar
-
-proc flowVarKind(t: PType): TFlowVarKind =
-  if t.skipTypes(abstractInst).kind in {tyRef, tyString, tySequence}: fvGC
-  elif containsGarbageCollectedRef(t): fvInvalid
-  else: fvBlob
-
-proc typeNeedsNoDeepCopy(t: PType): bool =
-  var t = t.skipTypes(abstractInst)
-  # for the tconvexhull example (and others) we're a bit lax here and pretend
-  # seqs and strings are *by value* only and 'shallow' doesn't exist!
-  if t.kind == tyString: return true
-  # note that seq[T] is fine, but 'var seq[T]' is not, so we need to skip 'var'
-  # for the stricter check and likewise we can skip 'seq' for a less
-  # strict check:
-  if t.kind in {tyVar, tyLent, tySequence}: t = t.lastSon
-  result = not containsGarbageCollectedRef(t)
-
-proc hoistExpr*(varSection, expr: PNode, name: PIdent, owner: PSym): PSym =
-  result = newSym(skLet, name, owner, varSection.info, owner.options)
-  result.flags.incl sfHoisted
-  result.typ = expr.typ
-
-  var varDef = newNodeI(nkIdentDefs, varSection.info, 3)
-  varDef.sons[0] = newSymNode(result)
-  varDef.sons[1] = newNodeI(nkEmpty, varSection.info)
-  varDef.sons[2] = expr
-
-  varSection.add varDef
-
-proc addLocalVar(g: ModuleGraph; varSection, varInit: PNode; owner: PSym; typ: PType;
-                 v: PNode; useShallowCopy=false): PSym =
-  result = newSym(skTemp, getIdent(g.cache, genPrefix), owner, varSection.info,
-                  owner.options)
-  result.typ = typ
-  incl(result.flags, sfFromGeneric)
-
-  var vpart = newNodeI(nkIdentDefs, varSection.info, 3)
-  vpart.sons[0] = newSymNode(result)
-  vpart.sons[1] = newNodeI(nkEmpty, varSection.info)
-  vpart.sons[2] = if varInit.isNil: v else: vpart[1]
-  varSection.add vpart
-  if varInit != nil:
-    if useShallowCopy and typeNeedsNoDeepCopy(typ):
-      varInit.add newFastAsgnStmt(newSymNode(result), v)
-    else:
-      let deepCopyCall = newNodeI(nkCall, varInit.info, 3)
-      deepCopyCall.sons[0] = newSymNode(getSysMagic(g, varSection.info, "deepCopy", mDeepCopy))
-      deepCopyCall.sons[1] = newSymNode(result)
-      deepCopyCall.sons[2] = v
-      varInit.add deepCopyCall
-
-discard """
-We generate roughly this:
-
-proc f_wrapper(thread, args) =
-  barrierEnter(args.barrier)  # for parallel statement
-  var a = args.a # thread transfer; deepCopy or shallowCopy or no copy
-                 # depending on whether we're in a 'parallel' statement
-  var b = args.b
-  var fv = args.fv
-
-  fv.owner = thread # optional
-  nimArgsPassingDone() # signal parent that the work is done
-  #
-  args.fv.blob = f(a, b, ...)
-  nimFlowVarSignal(args.fv)
-
-  # - or -
-  f(a, b, ...)
-  barrierLeave(args.barrier)  # for parallel statement
-
-stmtList:
-  var scratchObj
-  scratchObj.a = a
-  scratchObj.b = b
-
-  nimSpawn(f_wrapper, addr scratchObj)
-  scratchObj.fv # optional
-
-"""
-
-proc createWrapperProc(g: ModuleGraph; f: PNode; threadParam, argsParam: PSym;
-                       varSection, varInit, call, barrier, fv: PNode;
-                       spawnKind: TSpawnResult): PSym =
-  var body = newNodeI(nkStmtList, f.info)
-  var threadLocalBarrier: PSym
-  if barrier != nil:
-    var varSection2 = newNodeI(nkVarSection, barrier.info)
-    threadLocalBarrier = addLocalVar(g, varSection2, nil, argsParam.owner,
-                                     barrier.typ, barrier)
-    body.add varSection2
-    body.add callCodegenProc(g, "barrierEnter", threadLocalBarrier.newSymNode)
-  var threadLocalProm: PSym
-  if spawnKind == srByVar:
-    threadLocalProm = addLocalVar(g, varSection, nil, argsParam.owner, fv.typ, fv)
-  elif fv != nil:
-    internalAssert g.config, fv.typ.kind == tyGenericInst
-    threadLocalProm = addLocalVar(g, varSection, nil, argsParam.owner, fv.typ, fv)
-  body.add varSection
-  body.add varInit
-  if fv != nil and spawnKind != srByVar:
-    # generate:
-    #   fv.owner = threadParam
-    body.add newAsgnStmt(indirectAccess(threadLocalProm.newSymNode,
-      "owner", fv.info, g.cache), threadParam.newSymNode)
-
-  body.add callCodegenProc(g, "nimArgsPassingDone", threadParam.newSymNode)
-  if spawnKind == srByVar:
-    body.add newAsgnStmt(genDeref(threadLocalProm.newSymNode), call)
-  elif fv != nil:
-    let fk = fv.typ.sons[1].flowVarKind
-    if fk == fvInvalid:
-      localError(g.config, f.info, "cannot create a flowVar of type: " &
-        typeToString(fv.typ.sons[1]))
-    body.add newAsgnStmt(indirectAccess(threadLocalProm.newSymNode,
-      if fk == fvGC: "data" else: "blob", fv.info, g.cache), call)
-    if fk == fvGC:
-      let incRefCall = newNodeI(nkCall, fv.info, 2)
-      incRefCall.sons[0] = newSymNode(getSysMagic(g, fv.info, "GCref", mGCref))
-      incRefCall.sons[1] = indirectAccess(threadLocalProm.newSymNode,
-                                          "data", fv.info, g.cache)
-      body.add incRefCall
-    if barrier == nil:
-      # by now 'fv' is shared and thus might have beeen overwritten! we need
-      # to use the thread-local view instead:
-      body.add callCodegenProc(g, "nimFlowVarSignal", threadLocalProm.newSymNode)
-  else:
-    body.add call
-  if barrier != nil:
-    body.add callCodegenProc(g, "barrierLeave", threadLocalBarrier.newSymNode)
-
-  var params = newNodeI(nkFormalParams, f.info)
-  params.add newNodeI(nkEmpty, f.info)
-  params.add threadParam.newSymNode
-  params.add argsParam.newSymNode
-
-  var t = newType(tyProc, threadParam.owner)
-  t.rawAddSon nil
-  t.rawAddSon threadParam.typ
-  t.rawAddSon argsParam.typ
-  t.n = newNodeI(nkFormalParams, f.info)
-  t.n.add newNodeI(nkEffectList, f.info)
-  t.n.add threadParam.newSymNode
-  t.n.add argsParam.newSymNode
-
-  let name = (if f.kind == nkSym: f.sym.name.s else: genPrefix) & "Wrapper"
-  result = newSym(skProc, getIdent(g.cache, name), argsParam.owner, f.info,
-                  argsParam.options)
-  let emptyNode = newNodeI(nkEmpty, f.info)
-  result.ast = newProcNode(nkProcDef, f.info, body = body,
-      params = params, name = newSymNode(result), pattern = emptyNode,
-      genericParams = emptyNode, pragmas = emptyNode,
-      exceptions = emptyNode)
-  result.typ = t
-
-proc createCastExpr(argsParam: PSym; objType: PType): PNode =
-  result = newNodeI(nkCast, argsParam.info)
-  result.add newNodeI(nkEmpty, argsParam.info)
-  result.add newSymNode(argsParam)
-  result.typ = newType(tyPtr, objType.owner)
-  result.typ.rawAddSon(objType)
-
-proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType; scratchObj: PSym,
-                             castExpr, call,
-                             varSection, varInit, result: PNode) =
-  let formals = n[0].typ.n
-  let tmpName = getIdent(g.cache, genPrefix)
-  for i in 1 ..< n.len:
-    # we pick n's type here, which hopefully is 'tyArray' and not
-    # 'tyOpenArray':
-    var argType = n[i].typ.skipTypes(abstractInst)
-    if i < formals.len and formals[i].typ.kind in {tyVar, tyLent}:
-      localError(g.config, n[i].info, "'spawn'ed function cannot have a 'var' parameter")
-    #elif containsTyRef(argType):
-    #  localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure")
-
-    let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
-    var field = newSym(skField, fieldname, objType.owner, n.info, g.config.options)
-    field.typ = argType
-    objType.addField(field, g.cache)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[i])
-
-    let temp = addLocalVar(g, varSection, varInit, objType.owner, argType,
-                           indirectAccess(castExpr, field, n.info))
-    call.add(newSymNode(temp))
-
-proc getRoot*(n: PNode): PSym =
-  ## ``getRoot`` takes a *path* ``n``. A path is an lvalue expression
-  ## like ``obj.x[i].y``. The *root* of a path is the symbol that can be
-  ## determined as the owner; ``obj`` in the example.
-  case n.kind
-  of nkSym:
-    if n.sym.kind in {skVar, skResult, skTemp, skLet, skForVar}:
-      result = n.sym
-  of nkDotExpr, nkBracketExpr, nkHiddenDeref, nkDerefExpr,
-      nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
-    result = getRoot(n.sons[0])
-  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-    result = getRoot(n.sons[1])
-  of nkCallKinds:
-    if getMagic(n) == mSlice: result = getRoot(n.sons[1])
-  else: discard
+    result.typ = sym.typ[0]
 
 proc newIntLit*(g: ModuleGraph; info: TLineInfo; value: BiggestInt): PNode =
   result = nkIntLit.newIntNode(value)
@@ -528,202 +346,30 @@ proc newIntLit*(g: ModuleGraph; info: TLineInfo; value: BiggestInt): PNode =
 
 proc genHigh*(g: ModuleGraph; n: PNode): PNode =
   if skipTypes(n.typ, abstractVar).kind == tyArray:
-    result = newIntLit(g, n.info, lastOrd(g.config, skipTypes(n.typ, abstractVar)))
+    result = newIntLit(g, n.info, toInt64(lastOrd(g.config, skipTypes(n.typ, abstractVar))))
   else:
     result = newNodeI(nkCall, n.info, 2)
     result.typ = getSysType(g, n.info, tyInt)
-    result.sons[0] = newSymNode(getSysMagic(g, n.info, "high", mHigh))
-    result.sons[1] = n
+    result[0] = newSymNode(getSysMagic(g, n.info, "high", mHigh))
+    result[1] = n
 
-proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; scratchObj: PSym;
-                             castExpr, call,
-                             varSection, varInit, result: PNode) =
-  let formals = n[0].typ.n
-  let tmpName = getIdent(g.cache, genPrefix)
-  # we need to copy the foreign scratch object fields into local variables
-  # for correctness: These are called 'threadLocal' here.
-  for i in 1 ..< n.len:
-    let n = n[i]
-    let argType = skipTypes(if i < formals.len: formals[i].typ else: n.typ,
-                            abstractInst)
-    #if containsTyRef(argType):
-    #  localError(n.info, "'spawn'ed function cannot refer to 'ref'/closure")
-
-    let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
-    var field = newSym(skField, fieldname, objType.owner, n.info, g.config.options)
-
-    if argType.kind in {tyVarargs, tyOpenArray}:
-      # important special case: we always create a zero-copy slice:
-      let slice = newNodeI(nkCall, n.info, 4)
-      slice.typ = n.typ
-      slice.sons[0] = newSymNode(createMagic(g, "slice", mSlice))
-      slice.sons[0].typ = getSysType(g, n.info, tyInt) # fake type
-      var fieldB = newSym(skField, tmpName, objType.owner, n.info, g.config.options)
-      fieldB.typ = getSysType(g, n.info, tyInt)
-      objType.addField(fieldB, g.cache)
-
-      if getMagic(n) == mSlice:
-        let a = genAddrOf(n[1])
-        field.typ = a.typ
-        objType.addField(field, g.cache)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
-
-        var fieldA = newSym(skField, tmpName, objType.owner, n.info, g.config.options)
-        fieldA.typ = getSysType(g, n.info, tyInt)
-        objType.addField(fieldA, g.cache)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldA), n[2])
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), n[3])
-
-        let threadLocal = addLocalVar(g, varSection,nil, objType.owner, fieldA.typ,
-                                      indirectAccess(castExpr, fieldA, n.info),
-                                      useShallowCopy=true)
-        slice.sons[2] = threadLocal.newSymNode
-      else:
-        let a = genAddrOf(n)
-        field.typ = a.typ
-        objType.addField(field, g.cache)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
-        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), genHigh(g, n))
-
-        slice.sons[2] = newIntLit(g, n.info, 0)
-      # the array itself does not need to go through a thread local variable:
-      slice.sons[1] = genDeref(indirectAccess(castExpr, field, n.info))
-
-      let threadLocal = addLocalVar(g, varSection,nil, objType.owner, fieldB.typ,
-                                    indirectAccess(castExpr, fieldB, n.info),
-                                    useShallowCopy=true)
-      slice.sons[3] = threadLocal.newSymNode
-      call.add slice
-    elif (let size = computeSize(g.config, argType); size < 0 or size > 16) and
-        n.getRoot != nil:
-      # it is more efficient to pass a pointer instead:
-      let a = genAddrOf(n)
-      field.typ = a.typ
-      objType.addField(field, g.cache)
-      result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
-      let threadLocal = addLocalVar(g, varSection,nil, objType.owner, field.typ,
-                                    indirectAccess(castExpr, field, n.info),
-                                    useShallowCopy=true)
-      call.add(genDeref(threadLocal.newSymNode))
-    else:
-      # boring case
-      field.typ = argType
-      objType.addField(field, g.cache)
-      result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n)
-      let threadLocal = addLocalVar(g, varSection, varInit,
-                                    objType.owner, field.typ,
-                                    indirectAccess(castExpr, field, n.info),
-                                    useShallowCopy=true)
-      call.add(threadLocal.newSymNode)
-
-proc wrapProcForSpawn*(g: ModuleGraph; owner: PSym; spawnExpr: PNode; retType: PType;
-                       barrier, dest: PNode = nil): PNode =
-  # if 'barrier' != nil, then it is in a 'parallel' section and we
-  # generate quite different code
-  let n = spawnExpr[^2]
-  let spawnKind = spawnResult(retType, barrier!=nil)
-  case spawnKind
-  of srVoid:
-    internalAssert g.config, dest == nil
-    result = newNodeI(nkStmtList, n.info)
-  of srFlowVar:
-    internalAssert g.config, dest == nil
-    result = newNodeIT(nkStmtListExpr, n.info, retType)
-  of srByVar:
-    if dest == nil: localError(g.config, n.info, "'spawn' must not be discarded")
-    result = newNodeI(nkStmtList, n.info)
-
-  if n.kind notin nkCallKinds:
-    localError(g.config, n.info, "'spawn' takes a call expression")
-    return
-  if optThreadAnalysis in g.config.globalOptions:
-    if {tfThread, tfNoSideEffect} * n[0].typ.flags == {}:
-      localError(g.config, n.info, "'spawn' takes a GC safe call expression")
-  var
-    threadParam = newSym(skParam, getIdent(g.cache, "thread"), owner, n.info, g.config.options)
-    argsParam = newSym(skParam, getIdent(g.cache, "args"), owner, n.info, g.config.options)
-  block:
-    let ptrType = getSysType(g, n.info, tyPointer)
-    threadParam.typ = ptrType
-    argsParam.typ = ptrType
-    argsParam.position = 1
-
-  var objType = createObj(g, owner, n.info)
-  incl(objType.flags, tfFinal)
-  let castExpr = createCastExpr(argsParam, objType)
-
-  var scratchObj = newSym(skVar, getIdent(g.cache, "scratch"), owner, n.info, g.config.options)
-  block:
-    scratchObj.typ = objType
-    incl(scratchObj.flags, sfFromGeneric)
-    var varSectionB = newNodeI(nkVarSection, n.info)
-    varSectionB.addVar(scratchObj.newSymNode)
-    result.add varSectionB
-
-  var call = newNodeIT(nkCall, n.info, n.typ)
-  var fn = n.sons[0]
-  # templates and macros are in fact valid here due to the nature of
-  # the transformation:
-  if fn.kind == nkClosure:
-    localError(g.config, n.info, "closure in spawn environment is not allowed")
-  if not (fn.kind == nkSym and fn.sym.kind in {skProc, skTemplate, skMacro,
-                                               skFunc, skMethod, skConverter}):
-    # for indirect calls we pass the function pointer in the scratchObj
-    var argType = n[0].typ.skipTypes(abstractInst)
-    var field = newSym(skField, getIdent(g.cache, "fn"), owner, n.info, g.config.options)
-    field.typ = argType
-    objType.addField(field, g.cache)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0])
-    fn = indirectAccess(castExpr, field, n.info)
-  elif fn.kind == nkSym and fn.sym.kind == skIterator:
-    localError(g.config, n.info, "iterator in spawn environment is not allowed")
-  elif fn.typ.callConv == ccClosure:
-    localError(g.config, n.info, "closure in spawn environment is not allowed")
-
-  call.add(fn)
-  var varSection = newNodeI(nkVarSection, n.info)
-  var varInit = newNodeI(nkStmtList, n.info)
-  if barrier.isNil:
-    setupArgsForConcurrency(g, n, objType, scratchObj, castExpr, call,
-                            varSection, varInit, result)
+proc genLen*(g: ModuleGraph; n: PNode): PNode =
+  if skipTypes(n.typ, abstractVar).kind == tyArray:
+    result = newIntLit(g, n.info, toInt64(lastOrd(g.config, skipTypes(n.typ, abstractVar)) + 1))
   else:
-    setupArgsForParallelism(g, n, objType, scratchObj, castExpr, call,
-                            varSection, varInit, result)
+    result = newNodeI(nkCall, n.info, 2)
+    result.typ = getSysType(g, n.info, tyInt)
+    result[0] = newSymNode(getSysMagic(g, n.info, "len", mLengthSeq))
+    result[1] = n
 
-  var barrierAsExpr: PNode = nil
-  if barrier != nil:
-    let typ = newType(tyPtr, owner)
-    typ.rawAddSon(magicsys.getCompilerProc(g, "Barrier").typ)
-    var field = newSym(skField, getIdent(g.cache, "barrier"), owner, n.info, g.config.options)
-    field.typ = typ
-    objType.addField(field, g.cache)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), barrier)
-    barrierAsExpr = indirectAccess(castExpr, field, n.info)
+proc hoistExpr*(varSection, expr: PNode, name: PIdent, owner: PSym): PSym =
+  result = newSym(skLet, name, owner, varSection.info, owner.options)
+  result.flags.incl sfHoisted
+  result.typ = expr.typ
 
-  var fvField, fvAsExpr: PNode = nil
-  if spawnKind == srFlowVar:
-    var field = newSym(skField, getIdent(g.cache, "fv"), owner, n.info, g.config.options)
-    field.typ = retType
-    objType.addField(field, g.cache)
-    fvField = newDotExpr(scratchObj, field)
-    fvAsExpr = indirectAccess(castExpr, field, n.info)
-    # create flowVar:
-    result.add newFastAsgnStmt(fvField, callProc(spawnExpr[^1]))
-    if barrier == nil:
-      result.add callCodegenProc(g, "nimFlowVarCreateSemaphore", fvField)
+  var varDef = newNodeI(nkIdentDefs, varSection.info, 3)
+  varDef[0] = newSymNode(result)
+  varDef[1] = newNodeI(nkEmpty, varSection.info)
+  varDef[2] = expr
 
-  elif spawnKind == srByVar:
-    var field = newSym(skField, getIdent(g.cache, "fv"), owner, n.info, g.config.options)
-    field.typ = newType(tyPtr, objType.owner)
-    field.typ.rawAddSon(retType)
-    objType.addField(field, g.cache)
-    fvAsExpr = indirectAccess(castExpr, field, n.info)
-    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), genAddrOf(dest))
-
-  let wrapper = createWrapperProc(g, fn, threadParam, argsParam,
-                                  varSection, varInit, call,
-                                  barrierAsExpr, fvAsExpr, spawnKind)
-  result.add callCodegenProc(g, "nimSpawn" & $spawnExpr.len, wrapper.newSymNode,
-                             genAddrOf(scratchObj.newSymNode), nil, spawnExpr)
-
-  if spawnKind == srFlowVar: result.add fvField
+  varSection.add varDef

compiler/macrocacheimpl.nim

@@ -9,7 +9,7 @@
 
 ## This module implements helpers for the macro cache.
 
-import lineinfos, ast, modulegraphs, vmdef, magicsys
+import lineinfos, ast, modulegraphs, vmdef
 
 proc recordInc*(c: PCtx; info: TLineInfo; key: string; by: BiggestInt) =
   var recorded = newNodeI(nkCommentStmt, info)

compiler/magicsys.nim

@@ -10,7 +10,7 @@
 # Built-in types and compilerprocs are registered here.
 
 import
-  ast, astalgo, hashes, msgs, platform, nversion, times, idents,
+  ast, astalgo, msgs, platform, idents,
   modulegraphs, lineinfos
 
 export createMagic
@@ -40,7 +40,7 @@ proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSy
   while r != nil:
     if r.magic == m:
       # prefer the tyInt variant:
-      if r.typ.sons[0] != nil and r.typ.sons[0].kind == tyInt: return r
+      if r.typ[0] != nil and r.typ[0].kind == tyInt: return r
       result = r
     r = nextIdentIter(ti, g.systemModule.tab)
   if result != nil: return result
@@ -68,7 +68,7 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
     of tyUInt64: result = sysTypeFromName("uint64")
     of tyFloat: result = sysTypeFromName("float")
     of tyFloat32: result = sysTypeFromName("float32")
-    of tyFloat64: return sysTypeFromName("float64")
+    of tyFloat64: result = sysTypeFromName("float64")
     of tyFloat128: result = sysTypeFromName("float128")
     of tyBool: result = sysTypeFromName("bool")
     of tyChar: result = sysTypeFromName("char")
@@ -79,7 +79,9 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
     else: internalError(g.config, "request for typekind: " & $kind)
     g.sysTypes[kind] = result
   if result.kind != kind:
-    internalError(g.config, "wanted: " & $kind & " got: " & $result.kind)
+    if kind == tyFloat64 and result.kind == tyFloat: discard # because of aliasing
+    else:
+      internalError(g.config, "wanted: " & $kind & " got: " & $result.kind)
   if result == nil: internalError(g.config, "type not found: " & $kind)
 
 proc resetSysTypes*(g: ModuleGraph) =
@@ -123,7 +125,7 @@ proc addSonSkipIntLit*(father, son: PType) =
   when not defined(nimNoNilSeqs):
     if isNil(father.sons): father.sons = @[]
   let s = son.skipIntLit
-  add(father.sons, s)
+  father.sons.add(s)
   propagateToOwner(father, s)
 
 proc setIntLitType*(g: ModuleGraph; result: PNode) =

compiler/main.nim

@@ -13,25 +13,24 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  llstream, strutils, ast, astalgo, lexer, syntaxes, renderer, options, msgs,
-  os, condsyms, times,
-  wordrecg, sem, semdata, idents, passes, docgen, extccomp,
-  cgen, jsgen, json, nversion,
-  platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
-  docgen2, parser, modules, ccgutils, sigmatch, ropes,
+  llstream, strutils, ast, lexer, syntaxes, options, msgs,
+  condsyms, times,
+  sem, idents, passes, extccomp,
+  cgen, json, nversion,
+  platform, nimconf, passaux, depends, vm, idgen,
+  modules,
   modulegraphs, tables, rod, lineinfos, pathutils
 
-from magicsys import resetSysTypes
-
-proc codegenPass(g: ModuleGraph) =
-  registerPass g, cgenPass
+when not defined(leanCompiler):
+  import jsgen, docgen, docgen2
 
 proc semanticPasses(g: ModuleGraph) =
   registerPass g, verbosePass
   registerPass g, semPass
 
-proc writeDepsFile(g: ModuleGraph; project: AbsoluteFile) =
-  let f = open(changeFileExt(project, "deps").string, fmWrite)
+proc writeDepsFile(g: ModuleGraph) =
+  let fname = g.config.nimcacheDir / RelativeFile(g.config.projectName & ".deps")
+  let f = open(fname.string, fmWrite)
   for m in g.modules:
     if m != nil:
       f.writeLine(toFullPath(g.config, m.position.FileIndex))
@@ -45,7 +44,7 @@ proc commandGenDepend(graph: ModuleGraph) =
   registerPass(graph, gendependPass)
   compileProject(graph)
   let project = graph.config.projectFull
-  writeDepsFile(graph, project)
+  writeDepsFile(graph)
   generateDot(graph, project)
   execExternalProgram(graph.config, "dot -Tpng -o" &
       changeFileExt(project, "png").string &
@@ -57,46 +56,80 @@ proc commandCheck(graph: ModuleGraph) =
   semanticPasses(graph)  # use an empty backend for semantic checking only
   compileProject(graph)
 
-proc commandDoc2(graph: ModuleGraph; json: bool) =
-  graph.config.errorMax = high(int)  # do not stop after first error
-  semanticPasses(graph)
-  if json: registerPass(graph, docgen2JsonPass)
-  else: registerPass(graph, docgen2Pass)
-  compileProject(graph)
-  finishDoc2Pass(graph.config.projectName)
+when not defined(leanCompiler):
+  proc commandDoc2(graph: ModuleGraph; json: bool) =
+    handleDocOutputOptions graph.config
+    graph.config.errorMax = high(int)  # do not stop after first error
+    semanticPasses(graph)
+    if json: registerPass(graph, docgen2JsonPass)
+    else: registerPass(graph, docgen2Pass)
+    compileProject(graph)
+    finishDoc2Pass(graph.config.projectName)
 
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
+
+  if conf.outDir.isEmpty:
+    conf.outDir = conf.projectPath
+  if conf.outFile.isEmpty:
+    let targetName = if optGenDynLib in conf.globalOptions:
+      platform.OS[conf.target.targetOS].dllFrmt % conf.projectName
+    else:
+      conf.projectName & platform.OS[conf.target.targetOS].exeExt
+    conf.outFile = RelativeFile targetName
+
   extccomp.initVars(conf)
   semanticPasses(graph)
   registerPass(graph, cgenPass)
 
+  if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"):
+    let proj = changeFileExt(conf.projectFull, "")
+    if not changeDetectedViaJsonBuildInstructions(conf, proj):
+      # nothing changed
+      graph.config.notes = graph.config.mainPackageNotes
+      return
+
   compileProject(graph)
+  if graph.config.errorCounter > 0:
+    return # issue #9933
   cgenWriteModules(graph.backend, conf)
   if conf.cmd != cmdRun:
-    let proj = changeFileExt(conf.projectFull, "")
-    extccomp.callCCompiler(conf, proj)
-    extccomp.writeJsonBuildInstructions(conf, proj)
+    extccomp.callCCompiler(conf)
+    # for now we do not support writing out a .json file with the build instructions when HCR is on
+    if not conf.hcrOn:
+      extccomp.writeJsonBuildInstructions(conf)
     if optGenScript in graph.config.globalOptions:
-      writeDepsFile(graph, toGeneratedFile(conf, proj, ""))
+      writeDepsFile(graph)
 
 proc commandJsonScript(graph: ModuleGraph) =
   let proj = changeFileExt(graph.config.projectFull, "")
   extccomp.runJsonBuildInstructions(graph.config, proj)
 
-proc commandCompileToJS(graph: ModuleGraph) =
-  #incl(gGlobalOptions, optSafeCode)
-  setTarget(graph.config.target, osJS, cpuJS)
-  #initDefines()
-  defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility
-  defineSymbol(graph.config.symbols, "js")
-  semanticPasses(graph)
-  registerPass(graph, JSgenPass)
-  compileProject(graph)
+when not defined(leanCompiler):
+  proc commandCompileToJS(graph: ModuleGraph) =
+    let conf = graph.config
+    conf.exc = excCpp
+
+    if conf.outDir.isEmpty:
+      conf.outDir = conf.projectPath
+    if conf.outFile.isEmpty:
+      conf.outFile = RelativeFile(conf.projectName & ".js")
+
+    #incl(gGlobalOptions, optSafeCode)
+    setTarget(graph.config.target, osJS, cpuJS)
+    #initDefines()
+    defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility
+    defineSymbol(graph.config.symbols, "js")
+    semanticPasses(graph)
+    registerPass(graph, JSgenPass)
+    compileProject(graph)
+    if optGenScript in graph.config.globalOptions:
+      writeDepsFile(graph)
 
 proc interactivePasses(graph: ModuleGraph) =
   initDefines(graph.config.symbols)
   defineSymbol(graph.config.symbols, "nimscript")
+  # note: seems redundant with -d:nimHasLibFFI
   when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
   registerPass(graph, verbosePass)
   registerPass(graph, semPass)
@@ -118,14 +151,6 @@ const evalPasses = [verbosePass, semPass, evalPass]
 proc evalNim(graph: ModuleGraph; nodes: PNode, module: PSym) =
   carryPasses(graph, nodes, module, evalPasses)
 
-proc commandEval(graph: ModuleGraph; exp: string) =
-  if graph.systemModule == nil:
-    interactivePasses(graph)
-    compileSystemModule(graph)
-  let echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
-  evalNim(graph, echoExp.parseString(graph.cache, graph.config),
-    makeStdinModule(graph))
-
 proc commandScan(cache: IdentCache, config: ConfigRef) =
   var f = addFileExt(AbsoluteFile mainCommandArg(config), NimExt)
   var stream = llStreamOpen(f, fmRead)
@@ -160,9 +185,12 @@ proc mainCommand*(graph: ModuleGraph) =
   of "c", "cc", "compile", "compiletoc":
     # compile means compileToC currently
     conf.cmd = cmdCompileToC
+    if conf.exc == excNone: conf.exc = excSetjmp
+    defineSymbol(graph.config.symbols, "c")
     commandCompileToC(graph)
   of "cpp", "compiletocpp":
     conf.cmd = cmdCompileToCpp
+    if conf.exc == excNone: conf.exc = excCpp
     defineSymbol(graph.config.symbols, "cpp")
     commandCompileToC(graph)
   of "objc", "compiletooc":
@@ -172,54 +200,94 @@ proc mainCommand*(graph: ModuleGraph) =
   of "run":
     conf.cmd = cmdRun
     when hasTinyCBackend:
-      extccomp.setCC("tcc")
+      extccomp.setCC(conf, "tcc", unknownLineInfo)
       commandCompileToC(graph)
     else:
       rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
   of "js", "compiletojs":
-    conf.cmd = cmdCompileToJS
-    commandCompileToJS(graph)
+    when defined(leanCompiler):
+      quit "compiler wasn't built with JS code generator"
+    else:
+      conf.cmd = cmdCompileToJS
+      if conf.hcrOn:
+        # XXX: At the moment, system.nim cannot be compiled in JS mode
+        # with "-d:useNimRtl". The HCR option has been processed earlier
+        # and it has added this define implictly, so we must undo that here.
+        # A better solution might be to fix system.nim
+        undefSymbol(conf.symbols, "useNimRtl")
+      commandCompileToJS(graph)
   of "doc0":
-    wantMainModule(conf)
-    conf.cmd = cmdDoc
-    loadConfigs(DocConfig, cache, conf)
-    commandDoc(cache, conf)
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      wantMainModule(conf)
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
+      commandDoc(cache, conf)
   of "doc2", "doc":
-    conf.cmd = cmdDoc
-    loadConfigs(DocConfig, cache, conf)
-    defineSymbol(conf.symbols, "nimdoc")
-    commandDoc2(graph, false)
+    conf.setNoteDefaults(warnLockLevel, false) # issue #13218
+    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+      # because currently generates lots of false positives due to conflation
+      # of labels links in doc comments, eg for random.rand:
+      #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
+      #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
+      defineSymbol(conf.symbols, "nimdoc")
+      commandDoc2(graph, false)
   of "rst2html":
-    conf.cmd = cmdRst2html
-    loadConfigs(DocConfig, cache, conf)
-    commandRst2Html(cache, conf)
+    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      conf.cmd = cmdRst2html
+      loadConfigs(DocConfig, cache, conf)
+      commandRst2Html(cache, conf)
   of "rst2tex":
-    conf.cmd = cmdRst2tex
-    loadConfigs(DocTexConfig, cache, conf)
-    commandRst2TeX(cache, conf)
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      conf.cmd = cmdRst2tex
+      loadConfigs(DocTexConfig, cache, conf)
+      commandRst2TeX(cache, conf)
   of "jsondoc0":
-    wantMainModule(conf)
-    conf.cmd = cmdDoc
-    loadConfigs(DocConfig, cache, conf)
-    wantMainModule(conf)
-    defineSymbol(conf.symbols, "nimdoc")
-    commandJson(cache, conf)
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      wantMainModule(conf)
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
+      wantMainModule(conf)
+      defineSymbol(conf.symbols, "nimdoc")
+      commandJson(cache, conf)
   of "jsondoc2", "jsondoc":
-    conf.cmd = cmdDoc
-    loadConfigs(DocConfig, cache, conf)
-    wantMainModule(conf)
-    defineSymbol(conf.symbols, "nimdoc")
-    commandDoc2(graph, true)
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
+      wantMainModule(conf)
+      defineSymbol(conf.symbols, "nimdoc")
+      commandDoc2(graph, true)
   of "ctags":
-    wantMainModule(conf)
-    conf.cmd = cmdDoc
-    loadConfigs(DocConfig, cache, conf)
-    defineSymbol(conf.symbols, "nimdoc")
-    commandTags(cache, conf)
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      wantMainModule(conf)
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
+      defineSymbol(conf.symbols, "nimdoc")
+      commandTags(cache, conf)
   of "buildindex":
-    conf.cmd = cmdDoc
-    loadConfigs(DocConfig, cache, conf)
-    commandBuildIndex(cache, conf)
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
+      commandBuildIndex(cache, conf)
   of "gendepend":
     conf.cmd = cmdGenDepend
     commandGenDepend(graph)
@@ -232,13 +300,32 @@ proc mainCommand*(graph: ModuleGraph) =
       for s in definedSymbolNames(conf.symbols): definedSymbols.elems.add(%s)
 
       var libpaths = newJArray()
+      var lazyPaths = newJArray()
       for dir in conf.searchPaths: libpaths.elems.add(%dir.string)
+      for dir in conf.lazyPaths: lazyPaths.elems.add(%dir.string)
 
-      var dumpdata = % [
+      var hints = newJObject() # consider factoring with `listHints`
+      for a in hintMin..hintMax:
+        let key = lineinfos.HintsToStr[ord(a) - ord(hintMin)]
+        hints[key] = %(a in conf.notes)
+      var warnings = newJObject()
+      for a in warnMin..warnMax:
+        let key = lineinfos.WarningsToStr[ord(a) - ord(warnMin)]
+        warnings[key] = %(a in conf.notes)
+
+      var dumpdata = %[
         (key: "version", val: %VersionAsString),
+        (key: "prefixdir", val: %conf.getPrefixDir().string),
+        (key: "libpath", val: %conf.libpath.string),
         (key: "project_path", val: %conf.projectFull.string),
         (key: "defined_symbols", val: definedSymbols),
-        (key: "lib_paths", val: libpaths)
+        (key: "lib_paths", val: %libpaths),
+        (key: "lazyPaths", val: %lazyPaths),
+        (key: "outdir", val: %conf.outDir.string),
+        (key: "out", val: %conf.outFile.string),
+        (key: "nimcache", val: %getNimcacheDir(conf).string),
+        (key: "hints", val: hints),
+        (key: "warnings", val: warnings),
       ]
 
       msgWriteln(conf, $dumpdata, {msgStdout, msgSkipHook})
@@ -265,7 +352,11 @@ proc mainCommand*(graph: ModuleGraph) =
     conf.cmd = cmdInteractive
     commandInteractive(graph)
   of "e":
-    commandEval(graph, mainCommandArg(conf))
+    if not fileExists(conf.projectFull):
+      rawMessage(conf, errGenerated, "NimScript file does not exist: " & conf.projectFull.string)
+    elif not conf.projectFull.string.endsWith(".nims"):
+      rawMessage(conf, errGenerated, "not a NimScript file: " & conf.projectFull.string)
+    # main NimScript logic handled in cmdlinehelper.nim.
   of "nop", "help":
     # prevent the "success" message:
     conf.cmd = cmdDump
@@ -277,15 +368,25 @@ proc mainCommand*(graph: ModuleGraph) =
 
   if conf.errorCounter == 0 and
      conf.cmd notin {cmdInterpret, cmdRun, cmdDump}:
-    when declared(system.getMaxMem):
-      let usedMem = formatSize(getMaxMem()) & " peakmem"
-    else:
-      let usedMem = formatSize(getTotalMem())
-    rawMessage(conf, hintSuccessX, [$conf.linesCompiled,
-               formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3),
-               usedMem,
-               if isDefined(conf, "release"): "Release Build"
-               else: "Debug Build"])
+    let mem =
+      when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem"
+      else: formatSize(getTotalMem()) & " totmem"
+    let loc = $conf.linesCompiled
+    let build = if isDefined(conf, "danger"): "Dangerous Release"
+                elif isDefined(conf, "release"): "Release"
+                else: "Debug"
+    let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3)
+    let project = if optListFullPaths in conf.globalOptions: $conf.projectFull else: $conf.projectName
+    var output = $conf.absOutFile
+    if optListFullPaths notin conf.globalOptions: output = output.AbsoluteFile.extractFilename
+    rawMessage(conf, hintSuccessX, [
+      "loc", loc,
+      "sec", sec,
+      "mem", mem,
+      "build", build,
+      "project", project,
+      "output", output,
+      ])
 
   when PrintRopeCacheStats:
     echo "rope cache stats: "

compiler/modulegraphs.nim

@@ -26,13 +26,16 @@
 ##
 
 import ast, intsets, tables, options, lineinfos, hashes, idents,
-  incremental, btrees
+  incremental, btrees, md5
 
 type
+  SigHash* = distinct MD5Digest
+
   ModuleGraph* = ref object
     modules*: seq[PSym]  ## indexed by int32 fileIdx
     packageSyms*: TStrTable
     deps*: IntSet # the dependency graph or potentially its transitive closure.
+    importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies
     suggestMode*: bool # whether we are in nimsuggest mode or not.
     invalidTransitiveClosure: bool
     inclToMod*: Table[FileIndex, FileIndex] # mapping of include file to the
@@ -47,7 +50,7 @@ type
     doStopCompile*: proc(): bool {.closure.}
     usageSym*: PSym # for nimsuggest
     owners*: seq[PSym]
-    methods*: seq[tuple[methods: TSymSeq, dispatcher: PSym]] # needs serialization!
+    methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization!
     systemModule*: PSym
     sysTypes*: array[TTypeKind, PType]
     compilerprocs*: TStrTable
@@ -56,26 +59,123 @@ type
     opContains*, opNot*: PSym
     emptyNode*: PNode
     incr*: IncrementalCtx
+    canonTypes*: Table[SigHash, PType]
+    symBodyHashes*: Table[int, SigHash] # symId to digest mapping
     importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.}
     includeFileCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PNode {.nimcall.}
     recordStmt*: proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.}
-    cacheSeqs*: Table[string, PNode] # state that is shared to suppor the 'macrocache' API
+    cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API
     cacheCounters*: Table[string, BiggestInt]
     cacheTables*: Table[string, BTree[string, PNode]]
+    passes*: seq[TPass]
+    onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
+    onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
+    onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
+    globalDestructors*: seq[PNode]
+    proofEngine*: proc (graph: ModuleGraph; assumptions: seq[PNode]; toProve: PNode): (bool, string) {.nimcall.}
+    requirementsCheck*: proc (graph: ModuleGraph; assumptions: seq[PNode];
+                             call, requirement: PNode): (bool, string) {.nimcall.}
+    compatibleProps*: proc (graph: ModuleGraph; formal, actual: PType): bool {.nimcall.}
+
+  TPassContext* = object of RootObj # the pass's context
+  PPassContext* = ref TPassContext
+
+  TPassOpen* = proc (graph: ModuleGraph; module: PSym): PPassContext {.nimcall.}
+  TPassClose* = proc (graph: ModuleGraph; p: PPassContext, n: PNode): PNode {.nimcall.}
+  TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}
+
+  TPass* = tuple[open: TPassOpen,
+                 process: TPassProcess,
+                 close: TPassClose,
+                 isFrontend: bool]
+
+
+const
+  cb64 = [
+    "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N",
+    "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
+    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n",
+    "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
+    "0", "1", "2", "3", "4", "5", "6", "7", "8", "9a",
+    "9b", "9c"]
+
+proc toBase64a(s: cstring, len: int): string =
+  ## encodes `s` into base64 representation.
+  result = newStringOfCap(((len + 2) div 3) * 4)
+  result.add "__"
+  var i = 0
+  while i < len - 2:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    let c = ord(s[i+2])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2) or ((c and 0xC0) shr 6)]
+    result.add cb64[c and 0x3F]
+    inc(i, 3)
+  if i < len-1:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2)]
+  elif i < len:
+    let a = ord(s[i])
+    result.add cb64[a shr 2]
+    result.add cb64[(a and 3) shl 4]
+
+proc `$`*(u: SigHash): string =
+  toBase64a(cast[cstring](unsafeAddr u), sizeof(u))
+
+proc `==`*(a, b: SigHash): bool =
+  result = equalMem(unsafeAddr a, unsafeAddr b, sizeof(a))
+
+proc hash*(u: SigHash): Hash =
+  result = 0
+  for x in 0..3:
+    result = (result shl 8) or u.MD5Digest[x].int
 
 proc hash*(x: FileIndex): Hash {.borrow.}
 
+when defined(nimfind):
+  template onUse*(info: TLineInfo; s: PSym) =
+    when compiles(c.c.graph):
+      if c.c.graph.onUsage != nil: c.c.graph.onUsage(c.c.graph, s, info)
+    else:
+      if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info)
+
+  template onDef*(info: TLineInfo; s: PSym) =
+    when compiles(c.c.graph):
+      if c.c.graph.onDefinition != nil: c.c.graph.onDefinition(c.c.graph, s, info)
+    else:
+      if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info)
+
+  template onDefResolveForward*(info: TLineInfo; s: PSym) =
+    when compiles(c.c.graph):
+      if c.c.graph.onDefinitionResolveForward != nil:
+        c.c.graph.onDefinitionResolveForward(c.c.graph, s, info)
+    else:
+      if c.graph.onDefinitionResolveForward != nil:
+        c.graph.onDefinitionResolveForward(c.graph, s, info)
+
+else:
+  template onUse*(info: TLineInfo; s: PSym) = discard
+  template onDef*(info: TLineInfo; s: PSym) = discard
+  template onDefResolveForward*(info: TLineInfo; s: PSym) = discard
+
 proc stopCompile*(g: ModuleGraph): bool {.inline.} =
   result = g.doStopCompile != nil and g.doStopCompile()
 
 proc createMagic*(g: ModuleGraph; name: string, m: TMagic): PSym =
-  result = newSym(skProc, getIdent(g.cache, name), nil, unknownLineInfo(), {})
+  result = newSym(skProc, getIdent(g.cache, name), nil, unknownLineInfo, {})
   result.magic = m
+  result.flags = {sfNeverRaises}
 
 proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result = ModuleGraph()
   initStrTable(result.packageSyms)
   result.deps = initIntSet()
+  result.importDeps = initTable[FileIndex, seq[FileIndex]]()
   result.modules = @[]
   result.importStack = @[]
   result.inclToMod = initTable[FileIndex, FileIndex]()
@@ -94,6 +194,8 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.cacheSeqs = initTable[string, PNode]()
   result.cacheCounters = initTable[string, BiggestInt]()
   result.cacheTables = initTable[string, BTree[string, PNode]]()
+  result.canonTypes = initTable[SigHash, PType]()
+  result.symBodyHashes = initTable[int, SigHash]()
 
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.packageSyms)
@@ -118,7 +220,7 @@ proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) =
   addModuleDep(g.incr, g.config, m.info.fileIndex, dep, isIncludeFile = false)
   if g.suggestMode:
     g.deps.incl m.position.dependsOn(dep.int)
-    # we compute the transitive closure later when quering the graph lazily.
+    # we compute the transitive closure later when querying the graph lazily.
     # this improves efficiency quite a lot:
     #invalidTransitiveClosure = true
 

compiler/modulepaths.nim

@@ -8,7 +8,7 @@
 #
 
 import ast, renderer, strutils, msgs, options, idents, os, lineinfos,
-  pathutils, nimblecmd
+  pathutils
 
 when false:
   const
@@ -114,7 +114,6 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
     try:
       result =
         pathSubs(conf, n.strVal, toFullPath(conf, n.info).splitFile().dir)
-          .replace(" ")
     except ValueError:
       localError(conf, n.info, "invalid path: " & n.strVal)
       result = n.strVal
@@ -140,17 +139,17 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
       result.add modname
   of nkPrefix:
     when false:
-      if n.sons[0].kind == nkIdent and n.sons[0].ident.s == "$":
+      if n[0].kind == nkIdent and n[0].ident.s == "$":
         result = lookupPackage(n[1], nil)
       else:
         discard
     # hacky way to implement 'x / y /../ z':
     result = renderTree(n, {renderNoComments}).replace(" ")
   of nkDotExpr:
-    localError(conf, n.info, warnDeprecated, "using '.' instead of '/' in import paths")
+    localError(conf, n.info, warnDeprecated, "using '.' instead of '/' in import paths is deprecated")
     result = renderTree(n, {renderNoComments}).replace(".", "/")
   of nkImportAs:
-    result = getModuleName(conf, n.sons[0])
+    result = getModuleName(conf, n[0])
   else:
     localError(conf, n.info, "invalid module name: '$1'" % n.renderTree)
     result = ""
@@ -163,6 +162,6 @@ proc checkModuleName*(conf: ConfigRef; n: PNode; doLocalError=true): FileIndex =
     if doLocalError:
       let m = if modulename.len > 0: modulename else: $n
       localError(conf, n.info, "cannot open file: " & m)
-    result = InvalidFileIDX
+    result = InvalidFileIdx
   else:
     result = fileInfoIdx(conf, fullPath)

compiler/modules.nim

@@ -10,27 +10,16 @@
 ## Implements the module handling, including the caching of modules.
 
 import
-  ast, astalgo, magicsys, std / sha1, msgs, cgendata, sigmatch, options,
-  idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod,
-  lineinfos, pathutils
+  ast, astalgo, magicsys, msgs, options,
+  idents, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod,
+  lineinfos, pathutils, tables
 
 proc resetSystemArtifacts*(g: ModuleGraph) =
   magicsys.resetSysTypes(g)
 
-proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
-  # We cannot call ``newSym`` here, because we have to circumvent the ID
-  # mechanism, which we do in order to assign each module a persistent ID.
-  new(result)
-  result.id = -1             # for better error checking
-  result.kind = skModule
-  let filename = toFullPath(graph.config, fileIdx)
-  result.name = getIdent(graph.cache, splitFile(filename).name)
-  if not isNimIdentifier(result.name.s):
-    rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
-
-  result.info = newLineInfo(fileIdx, 1, 1)
+proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
   let
-    pck = getPackageName(graph.config, filename)
+    pck = getPackageName(graph.config, filename.string)
     pck2 = if pck.len > 0: pck else: "unknown"
     pack = getIdent(graph.cache, pck2)
   var packSym = graph.packageSyms.strTableGet(pack)
@@ -38,35 +27,60 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
     packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, result.info)
     initStrTable(packSym.tab)
     graph.packageSyms.strTableAdd(packSym)
+  else:
+    let existing = strTableGet(packSym.tab, result.name)
+    if existing != nil and existing.info.fileIndex != result.info.fileIndex:
+      when false:
+        # we used to produce an error:
+        localError(graph.config, result.info,
+          "module names need to be unique per Nimble package; module clashes with " &
+            toFullPath(graph.config, existing.info.fileIndex))
+      else:
+        # but starting with version 0.20 we now produce a fake Nimble package instead
+        # to resolve the conflicts:
+        let pck3 = fakePackageName(graph.config, filename)
+        # this makes the new `packSym`'s owner be the original `packSym`
+        packSym = newSym(skPackage, getIdent(graph.cache, pck3), packSym, result.info)
+        initStrTable(packSym.tab)
+        graph.packageSyms.strTableAdd(packSym)
 
   result.owner = packSym
   result.position = int fileIdx
 
   if int(fileIdx) >= graph.modules.len:
     setLen(graph.modules, int(fileIdx) + 1)
-  #growCache graph.modules, int fileIdx
   graph.modules[result.position] = result
 
-  incl(result.flags, sfUsed)
   initStrTable(result.tab)
   strTableAdd(result.tab, result) # a module knows itself
-  let existing = strTableGet(packSym.tab, result.name)
-  if existing != nil and existing.info.fileIndex != result.info.fileIndex:
-    localError(graph.config, result.info,
-      "module names need to be unique per Nimble package; module clashes with " &
-        toFullPath(graph.config, existing.info.fileIndex))
-  # strTableIncl() for error corrections:
-  discard strTableIncl(packSym.tab, result)
+  strTableAdd(packSym.tab, result)
+
+proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+  # We cannot call ``newSym`` here, because we have to circumvent the ID
+  # mechanism, which we do in order to assign each module a persistent ID.
+  result = PSym(kind: skModule, id: -1, # for better error checking
+                name: getIdent(graph.cache, splitFile(filename).name),
+                info: newLineInfo(fileIdx, 1, 1))
+  if not isNimIdentifier(result.name.s):
+    rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
+  partialInitModule(result, graph, fileIdx, filename)
 
 proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym =
   result = graph.getModule(fileIdx)
   if result == nil:
-    result = newModule(graph, fileIdx)
-    result.flags = result.flags + flags
-    if sfMainModule in result.flags:
-      graph.config.mainPackageId = result.owner.id
-
-    result.id = getModuleId(graph, fileIdx, AbsoluteFile toFullPath(graph.config, fileIdx))
+    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+    let (r, id) = loadModuleSym(graph, fileIdx, filename)
+    result = r
+    if result == nil:
+      result = newModule(graph, fileIdx)
+      result.flags = result.flags + flags
+      result.id = id
+      registerModule(graph, result)
+    else:
+      partialInitModule(result, graph, fileIdx, filename)
+      result.id = id
+      assert result.id < 0
     discard processModule(graph, result,
       if sfMainModule in flags and graph.config.projectIsStdin: stdin.llStreamOpen else: nil)
   elif graph.isDirty(result):
@@ -78,19 +92,22 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
       if sfMainModule in flags and graph.config.projectIsStdin: stdin.llStreamOpen else: nil)
     graph.markClientsDirty(fileIdx)
 
-proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym {.procvar.} =
+proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
   # this is called by the semantic checking phase
   assert graph.config != nil
   result = compileModule(graph, fileIdx, {})
   graph.addDep(s, fileIdx)
+  # keep track of import relationships
+  if graph.config.hcrOn:
+    graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx)
   #if sfSystemModule in result.flags:
   #  localError(result.info, errAttemptToRedefine, result.name.s)
   # restore the notes for outer module:
   graph.config.notes =
-    if s.owner.id == graph.config.mainPackageId: graph.config.mainPackageNotes
+    if s.owner.id == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
     else: graph.config.foreignPackageNotes
 
-proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode {.procvar.} =
+proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode =
   result = syntaxes.parseFile(fileIdx, graph.cache, graph.config)
   graph.addDep(s, fileIdx)
   graph.addIncludeDep(s.position.FileIndex, fileIdx)
@@ -112,12 +129,12 @@ proc wantMainModule*(conf: ConfigRef) =
         "command expects a filename")
   conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt))
 
-proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIDX) =
+proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
   connectCallbacks(graph)
   let conf = graph.config
   wantMainModule(conf)
   let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
-  let projectFile = if projectFileIdx == InvalidFileIDX: conf.projectMainIdx else: projectFileIdx
+  let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
   graph.importStack.add projectFile
   if projectFile == systemFileIdx:
     discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})
@@ -128,6 +145,7 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIDX) =
 proc makeModule*(graph: ModuleGraph; filename: AbsoluteFile): PSym =
   result = graph.newModule(fileInfoIdx(graph.config, filename))
   result.id = getID()
+  registerModule(graph, result)
 
 proc makeModule*(graph: ModuleGraph; filename: string): PSym =
   result = makeModule(graph, AbsoluteFile filename)

compiler/msgs.nim

@@ -8,7 +8,7 @@
 #
 
 import
-  options, strutils, os, tables, ropes, platform, terminal, macros,
+  options, strutils, os, tables, ropes, terminal, macros,
   lineinfos, pathutils
 
 proc toCChar*(c: char; result: var string) =
@@ -26,13 +26,13 @@ proc makeCString*(s: string): Rope =
   const MaxLineLength = 64
   result = nil
   var res = newStringOfCap(int(s.len.toFloat * 1.1) + 1)
-  add(res, "\"")
-  for i in countup(0, len(s) - 1):
+  res.add("\"")
+  for i in 0..<s.len:
     if (i + 1) mod MaxLineLength == 0:
-      add(res, "\"\L\"")
+      res.add("\"\L\"")
     toCChar(s[i], res)
-  add(res, '\"')
-  add(result, rope(res))
+  res.add('\"')
+  result.add(rope(res))
 
 
 proc newFileInfo(fullPath: AbsoluteFile, projPath: RelativeFile): TFileInfo =
@@ -40,9 +40,8 @@ proc newFileInfo(fullPath: AbsoluteFile, projPath: RelativeFile): TFileInfo =
   #shallow(result.fullPath)
   result.projPath = projPath
   #shallow(result.projPath)
-  let fileName = fullPath.extractFilename
-  result.shortName = fileName.changeFileExt("")
-  result.quotedName = fileName.makeCString
+  result.shortName = fullPath.extractFilename
+  result.quotedName = result.shortName.makeCString
   result.quotedFullName = fullPath.string.makeCString
   result.lines = @[]
   when defined(nimpretty):
@@ -82,6 +81,7 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile; isKnownFile: var bool
     pseudoPath = true
 
   if conf.m.filenameToIndexTbl.hasKey(canon.string):
+    isKnownFile = true
     result = conf.m.filenameToIndexTbl[canon.string]
   else:
     isKnownFile = false
@@ -96,14 +96,20 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile): FileIndex =
 
 proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo =
   result.fileIndex = fileInfoIdx
-  result.line = uint16(line)
-  result.col = int16(col)
+  if line < int high(uint16):
+    result.line = uint16(line)
+  else:
+    result.line = high(uint16)
+  if col < int high(int16):
+    result.col = int16(col)
+  else:
+    result.col = -1
 
 proc newLineInfo*(conf: ConfigRef; filename: AbsoluteFile, line, col: int): TLineInfo {.inline.} =
   result = newLineInfo(fileInfoIdx(conf, filename), line, col)
 
 
-proc concat(strings: openarray[string]): string =
+proc concat(strings: openArray[string]): string =
   var totalLen = 0
   for s in strings: totalLen += s.len
   result = newStringOfCap totalLen
@@ -135,35 +141,49 @@ const
   WarningColor = fgYellow
   HintTitle    = "Hint: "
   HintColor    = fgGreen
+  # NOTE: currently line info line numbers start with 1,
+  # but column numbers start with 0, however most editors expect
+  # first column to be 1, so we need to +1 here
+  ColOffset*   = 1
 
 proc getInfoContextLen*(conf: ConfigRef): int = return conf.m.msgContext.len
 proc setInfoContextLen*(conf: ConfigRef; L: int) = setLen(conf.m.msgContext, L)
 
-proc pushInfoContext*(conf: ConfigRef; info: TLineInfo) =
-  conf.m.msgContext.add(info)
+proc pushInfoContext*(conf: ConfigRef; info: TLineInfo; detail: string = "") =
+  conf.m.msgContext.add((info, detail))
 
 proc popInfoContext*(conf: ConfigRef) =
-  setLen(conf.m.msgContext, len(conf.m.msgContext) - 1)
+  setLen(conf.m.msgContext, conf.m.msgContext.len - 1)
 
 proc getInfoContext*(conf: ConfigRef; index: int): TLineInfo =
-  let L = conf.m.msgContext.len
-  let i = if index < 0: L + index else: index
-  if i >=% L: result = unknownLineInfo()
-  else: result = conf.m.msgContext[i]
+  let i = if index < 0: conf.m.msgContext.len + index else: index
+  if i >=% conf.m.msgContext.len: result = unknownLineInfo
+  else: result = conf.m.msgContext[i].info
+
+const
+  commandLineDesc = "command line"
 
 template toFilename*(conf: ConfigRef; fileIdx: FileIndex): string =
   if fileIdx.int32 < 0 or conf == nil:
-    "???"
+    (if fileIdx == commandLineIdx: commandLineDesc else: "???")
   else:
-    conf.m.fileInfos[fileIdx.int32].projPath.string
+    conf.m.fileInfos[fileIdx.int32].shortName
+
+proc toProjPath*(conf: ConfigRef; fileIdx: FileIndex): string =
+  if fileIdx.int32 < 0 or conf == nil:
+    (if fileIdx == commandLineIdx: commandLineDesc else: "???")
+  else: conf.m.fileInfos[fileIdx.int32].projPath.string
 
 proc toFullPath*(conf: ConfigRef; fileIdx: FileIndex): string =
-  if fileIdx.int32 < 0 or conf == nil: result = "???"
-  else: result = conf.m.fileInfos[fileIdx.int32].fullPath.string
+  if fileIdx.int32 < 0 or conf == nil:
+    result = (if fileIdx == commandLineIdx: commandLineDesc else: "???")
+  else:
+    result = conf.m.fileInfos[fileIdx.int32].fullPath.string
 
 proc setDirtyFile*(conf: ConfigRef; fileIdx: FileIndex; filename: AbsoluteFile) =
   assert fileIdx.int32 >= 0
   conf.m.fileInfos[fileIdx.int32].dirtyFile = filename
+  setLen conf.m.fileInfos[fileIdx.int32].lines, 0
 
 proc setHash*(conf: ConfigRef; fileIdx: FileIndex; hash: string) =
   assert fileIdx.int32 >= 0
@@ -175,7 +195,7 @@ proc getHash*(conf: ConfigRef; fileIdx: FileIndex): string =
 
 proc toFullPathConsiderDirty*(conf: ConfigRef; fileIdx: FileIndex): AbsoluteFile =
   if fileIdx.int32 < 0:
-    result = AbsoluteFile"???"
+    result = AbsoluteFile(if fileIdx == commandLineIdx: commandLineDesc else: "???")
   elif not conf.m.fileInfos[fileIdx.int32].dirtyFile.isEmpty:
     result = conf.m.fileInfos[fileIdx.int32].dirtyFile
   else:
@@ -184,16 +204,28 @@ proc toFullPathConsiderDirty*(conf: ConfigRef; fileIdx: FileIndex): AbsoluteFile
 template toFilename*(conf: ConfigRef; info: TLineInfo): string =
   toFilename(conf, info.fileIndex)
 
+template toProjPath*(conf: ConfigRef; info: TLineInfo): string =
+  toProjPath(conf, info.fileIndex)
+
 template toFullPath*(conf: ConfigRef; info: TLineInfo): string =
   toFullPath(conf, info.fileIndex)
 
-proc toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
-  if info.fileIndex.int32 < 0:
-    result = "???"
-  elif optListFullPaths in conf.globalOptions:
-    result = conf.m.fileInfos[info.fileIndex.int32].fullPath.string
-  else:
-    result = conf.m.fileInfos[info.fileIndex.int32].projPath.string
+template toFullPathConsiderDirty*(conf: ConfigRef; info: TLineInfo): string =
+  string toFullPathConsiderDirty(conf, info.fileIndex)
+
+proc toMsgFilename*(conf: ConfigRef; info: FileIndex): string =
+  let
+    absPath = toFullPath(conf, info)
+    relPath = toProjPath(conf, info)
+  result = if (optListFullPaths in conf.globalOptions) or
+              (relPath.len > absPath.len) or
+              (relPath.count("..") > 2):
+             absPath
+           else:
+             relPath
+
+template toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
+  toMsgFilename(conf, info.fileIndex)
 
 proc toLinenumber*(info: TLineInfo): int {.inline.} =
   result = int info.line
@@ -201,11 +233,10 @@ proc toLinenumber*(info: TLineInfo): int {.inline.} =
 proc toColumn*(info: TLineInfo): int {.inline.} =
   result = info.col
 
-proc toFileLine*(conf: ConfigRef; info: TLineInfo): string {.inline.} =
-  result = toFilename(conf, info) & ":" & $info.line
-
 proc toFileLineCol*(conf: ConfigRef; info: TLineInfo): string {.inline.} =
-  result = toFilename(conf, info) & "(" & $info.line & ", " & $info.col & ")"
+  # consider calling `helpers.lineInfoToString` instead
+  result = toMsgFilename(conf, info) & "(" & $info.line & ", " &
+    $(info.col + ColOffset) & ")"
 
 proc `$`*(conf: ConfigRef; info: TLineInfo): string = toFileLineCol(conf, info)
 
@@ -302,20 +333,21 @@ proc getMessageStr(msg: TMsgKind, arg: string): string =
 type
   TErrorHandling = enum doNothing, doAbort, doRaise
 
-proc log*(s: string) {.procvar.} =
+proc log*(s: string) =
   var f: File
   if open(f, getHomeDir() / "nimsuggest.log", fmAppend):
     f.writeLine(s)
     close(f)
 
-proc quit(conf: ConfigRef; msg: TMsgKind) =
-  if defined(debug) or msg == errInternal or hintStackTrace in conf.notes:
-    if stackTraceAvailable() and isNil(conf.writelnHook):
-      writeStackTrace()
-    else:
-      styledMsgWriteln(fgRed, "No stack traceback available\n" &
-          "To create a stacktrace, rerun compilation with ./koch temp " &
-          conf.command & " <file>")
+proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} =
+  if defined(debug) or msg == errInternal or conf.hasHint(hintStackTrace):
+    {.gcsafe.}:
+      if stackTraceAvailable() and isNil(conf.writelnHook):
+        writeStackTrace()
+      else:
+        styledMsgWriteln(fgRed, "No stack traceback available\n" &
+            "To create a stacktrace, rerun compilation with ./koch temp " &
+            conf.command & " <file>")
   quit 1
 
 proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) =
@@ -340,20 +372,26 @@ proc exactEquals*(a, b: TLineInfo): bool =
 
 proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
   const instantiationFrom = "template/generic instantiation from here"
+  const instantiationOfFrom = "template/generic instantiation of `$1` from here"
   var info = lastinfo
-  for i in 0 ..< len(conf.m.msgContext):
-    if conf.m.msgContext[i] != lastinfo and conf.m.msgContext[i] != info:
+  for i in 0..<conf.m.msgContext.len:
+    let context = conf.m.msgContext[i]
+    if context.info != lastinfo and context.info != info:
       if conf.structuredErrorHook != nil:
-        conf.structuredErrorHook(conf, conf.m.msgContext[i], instantiationFrom,
-                                  Severity.Error)
+        conf.structuredErrorHook(conf, context.info, instantiationFrom,
+                                 Severity.Error)
       else:
+        let message = if context.detail == "":
+          instantiationFrom
+        else:
+          instantiationOfFrom.format(context.detail)
         styledMsgWriteln(styleBright,
-                         PosFormat % [toMsgFilename(conf, conf.m.msgContext[i]),
-                                      coordToStr(conf.m.msgContext[i].line.int),
-                                      coordToStr(conf.m.msgContext[i].col+1)],
+                         PosFormat % [toMsgFilename(conf, context.info),
+                                      coordToStr(context.info.line.int),
+                                      coordToStr(context.info.col+ColOffset)],
                          resetStyle,
-                         instantiationFrom)
-    info = conf.m.msgContext[i]
+                         message)
+    info = context.info
 
 proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
   msg >= errGenerated and conf.cmd == cmdIdeTools and optIdeDebug notin conf.globalOptions
@@ -367,22 +405,20 @@ proc rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
   case msg
   of errMin..errMax:
     sev = Severity.Error
-    writeContext(conf, unknownLineInfo())
+    writeContext(conf, unknownLineInfo)
     title = ErrorTitle
     color = ErrorColor
   of warnMin..warnMax:
     sev = Severity.Warning
-    if optWarns notin conf.options: return
-    if msg notin conf.notes: return
-    writeContext(conf, unknownLineInfo())
+    if not conf.hasWarn(msg): return
+    writeContext(conf, unknownLineInfo)
     title = WarningTitle
     color = WarningColor
     kind = WarningsToStr[ord(msg) - ord(warnMin)]
     inc(conf.warnCounter)
   of hintMin..hintMax:
     sev = Severity.Hint
-    if optHints notin conf.options: return
-    if msg notin conf.notes: return
+    if not conf.hasHint(msg): return
     title = HintTitle
     color = HintColor
     if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
@@ -390,7 +426,7 @@ proc rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
   let s = msgKindToString(msg) % args
 
   if conf.structuredErrorHook != nil:
-    conf.structuredErrorHook(conf, unknownLineInfo(),
+    conf.structuredErrorHook(conf, unknownLineInfo,
       s & (if kind.len > 0: KindFormat % kind else: ""), sev)
 
   if not ignoreMsgBecauseOfIdeTools(conf, msg):
@@ -414,9 +450,9 @@ proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
 proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
   if i.fileIndex.int32 < 0: return ""
 
-  if not optPreserveOrigSource(conf) and conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
+  if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
     try:
-      for line in lines(toFullPath(conf, i)):
+      for line in lines(toFullPathConsiderDirty(conf, i)):
         addSourceLine conf, i.fileIndex, line.string
     except IOError:
       discard
@@ -429,7 +465,8 @@ proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
 proc writeSurroundingSrc(conf: ConfigRef; info: TLineInfo) =
   const indent = "  "
   msgWriteln(conf, indent & $sourceLine(conf, info))
-  msgWriteln(conf, indent & spaces(info.col) & '^')
+  if info.col >= 0:
+    msgWriteln(conf, indent & spaces(info.col) & '^')
 
 proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): string =
   let title = case msg
@@ -437,7 +474,7 @@ proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): s
               of hintMin..hintMax: HintTitle
               else: ErrorTitle
   result = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                        coordToStr(info.col+1)] &
+                        coordToStr(info.col+ColOffset)] &
            title &
            getMessageStr(msg, arg)
 
@@ -461,7 +498,7 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
     conf.m.lastError = info
   of warnMin..warnMax:
     sev = Severity.Warning
-    ignoreMsg = optWarns notin conf.options or msg notin conf.notes
+    ignoreMsg = not conf.hasWarn(msg)
     if not ignoreMsg: writeContext(conf, info)
     title = WarningTitle
     color = WarningColor
@@ -469,16 +506,13 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
     inc(conf.warnCounter)
   of hintMin..hintMax:
     sev = Severity.Hint
-    ignoreMsg = optHints notin conf.options or msg notin conf.notes
+    ignoreMsg = not conf.hasHint(msg)
     title = HintTitle
     color = HintColor
     if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
     inc(conf.hintCounter)
-  # NOTE: currently line info line numbers start with 1,
-  # but column numbers start with 0, however most editors expect
-  # first column to be 1, so we need to +1 here
   let x = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                       coordToStr(info.col+1)]
+                       coordToStr(info.col+ColOffset)]
   let s = getMessageStr(msg, arg)
 
   if not ignoreMsg:
@@ -490,7 +524,7 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
                          KindColor, `%`(KindFormat, kind))
       else:
         styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s)
-      if hintSource in conf.notes:
+      if conf.hasHint(hintSource):
         conf.writeSurroundingSrc(info)
   handleError(conf, msg, eh, s)
 
@@ -512,7 +546,7 @@ proc localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
 proc localError*(conf: ConfigRef; info: TLineInfo, arg: string) =
   liMessage(conf, info, errGenerated, arg, doNothing)
 
-proc localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openarray[string]) =
+proc localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openArray[string]) =
   localError(conf, info, format % params)
 
 proc message*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
@@ -525,7 +559,7 @@ proc internalError*(conf: ConfigRef; info: TLineInfo, errMsg: string) =
 
 proc internalError*(conf: ConfigRef; errMsg: string) =
   if conf.cmd == cmdIdeTools and conf.structuredErrorHook.isNil: return
-  writeContext(conf, unknownLineInfo())
+  writeContext(conf, unknownLineInfo)
   rawMessage(conf, errInternal, errMsg)
 
 template assertNotNil*(conf: ConfigRef; e): untyped =
@@ -536,8 +570,9 @@ template internalAssert*(conf: ConfigRef, e: bool) =
   if not e: internalError(conf, $instantiationInfo())
 
 proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
-  assert i.fileIndex.int32 >= 0
-  if optExcessiveStackTrace in conf.globalOptions:
+  if i.fileIndex.int32 < 0:
+    result = makeCString "???"
+  elif optExcessiveStackTrace in conf.globalOptions:
     result = conf.m.fileInfos[i.fileIndex.int32].quotedFullName
   else:
     result = conf.m.fileInfos[i.fileIndex.int32].quotedName
@@ -557,3 +592,10 @@ proc listHints*(conf: ConfigRef) =
       if hint in conf.notes: "x" else: " ",
       lineinfos.HintsToStr[ord(hint) - ord(hintMin)]
     ])
+
+proc lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string) =
+  let m = "'$2' should be: '$1'" % [beau, got]
+  if optStyleError in conf.globalOptions:
+    localError(conf, info, m)
+  else:
+    message(conf, info, hintName, m)

compiler/ndi.nim

@@ -17,24 +17,33 @@ type
     enabled: bool
     f: File
     buf: string
+    filename: AbsoluteFile
+    syms: seq[PSym]
 
 proc doWrite(f: var NdiFile; s: PSym; conf: ConfigRef) =
   f.buf.setLen 0
-  f.buf.add s.info.line.int
+  f.buf.addInt s.info.line.int
   f.buf.add "\t"
-  f.buf.add s.info.col.int
+  f.buf.addInt s.info.col.int
   f.f.write(s.name.s, "\t")
   f.f.writeRope(s.loc.r)
   f.f.writeLine("\t", toFullPath(conf, s.info), "\t", f.buf)
 
 template writeMangledName*(f: NdiFile; s: PSym; conf: ConfigRef) =
-  if f.enabled: doWrite(f, s, conf)
+  if f.enabled: f.syms.add s
 
 proc open*(f: var NdiFile; filename: AbsoluteFile; conf: ConfigRef) =
   f.enabled = not filename.isEmpty
   if f.enabled:
-    f.f = open(filename.string, fmWrite, 8000)
+    f.filename = filename
     f.buf = newStringOfCap(20)
 
-proc close*(f: var NdiFile) =
-  if f.enabled: close(f.f)
+proc close*(f: var NdiFile, conf: ConfigRef) =
+  if f.enabled:
+    f.f = open(f.filename.string, fmWrite, 8000)
+    doAssert f.f != nil, f.filename.string
+    for s in f.syms:
+      doWrite(f, s, conf)
+    close(f.f)
+    f.syms.reset
+    f.filename.reset

compiler/nim.cfg

@@ -7,11 +7,17 @@ define:nimcore
 #define:nimIncremental
 #import:"$projectpath/testability"
 
+#define:staticSqlite
+
 @if windows:
   cincludes: "$lib/wrappers/libffi/common"
 @end
 
 define:useStdoutAsStdmsg
 
+@if nimHasStyleChecks:
+  styleCheck:error
+@end
+
 #define:useNodeIds
 #gc:markAndSweep

compiler/nim.nim

@@ -9,21 +9,23 @@
 
 when defined(gcc) and defined(windows):
   when defined(x86):
-    {.link: "icons/nim.res".}
+    {.link: "../icons/nim.res".}
   else:
-    {.link: "icons/nim_icon.o".}
+    {.link: "../icons/nim_icon.o".}
 
 when defined(amd64) and defined(windows) and defined(vcc):
-  {.link: "icons/nim-amd64-windows-vcc.res".}
+  {.link: "../icons/nim-amd64-windows-vcc.res".}
 when defined(i386) and defined(windows) and defined(vcc):
-  {.link: "icons/nim-i386-windows-vcc.res".}
+  {.link: "../icons/nim-i386-windows-vcc.res".}
 
 import
-  commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes,
-  extccomp, strutils, os, osproc, platform, main, parseopt,
-  nodejs, scriptconfig, idents, modulegraphs, lineinfos, cmdlinehelper,
+  commands, options, msgs,
+  extccomp, strutils, os, main, parseopt,
+  idents, lineinfos, cmdlinehelper,
   pathutils
 
+include nodejs
+
 when hasTinyCBackend:
   import tccgen
 
@@ -38,23 +40,44 @@ proc prependCurDir(f: AbsoluteFile): AbsoluteFile =
   else:
     result = f
 
+proc addCmdPrefix*(result: var string, kind: CmdLineKind) =
+  # consider moving this to std/parseopt
+  case kind
+  of cmdLongOption: result.add "--"
+  of cmdShortOption: result.add "-"
+  of cmdArgument, cmdEnd: discard
+
 proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
   var p = parseopt.initOptParser(cmd)
   var argsCount = 0
+
+  config.commandLine.setLen 0
+    # bugfix: otherwise, config.commandLine ends up duplicated
+
   while true:
     parseopt.next(p)
     case p.kind
     of cmdEnd: break
-    of cmdLongoption, cmdShortOption:
+    of cmdLongOption, cmdShortOption:
+      config.commandLine.add " "
+      config.commandLine.addCmdPrefix p.kind
+      config.commandLine.add p.key.quoteShell # quoteShell to be future proof
+      if p.val.len > 0:
+        config.commandLine.add ':'
+        config.commandLine.add p.val.quoteShell
+
       if p.key == " ":
         p.key = "-"
         if processArgument(pass, p, argsCount, config): break
       else:
         processSwitch(pass, p, config)
     of cmdArgument:
+      config.commandLine.add " "
+      config.commandLine.add p.key.quoteShell
       if processArgument(pass, p, argsCount, config): break
   if pass == passCmd2:
-    if optRun notin config.globalOptions and config.arguments.len > 0 and config.command.normalize != "run":
+    if {optRun, optWasNimscript} * config.globalOptions == {} and
+        config.arguments.len > 0 and config.command.normalize notin ["run", "e"]:
       rawMessage(config, errGenerated, errArgsNeedRunOption)
 
 proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
@@ -65,35 +88,22 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
   )
   self.initDefinesProg(conf, "nim_compiler")
   if paramCount() == 0:
-    writeCommandLineUsage(conf, conf.helpWritten)
+    writeCommandLineUsage(conf)
     return
 
   self.processCmdLineAndProjectPath(conf)
   if not self.loadConfigsAndRunMainCommand(cache, conf): return
-  if optHints in conf.options and hintGCStats in conf.notes: echo(GC_getStatistics())
+  if conf.hasHint(hintGCStats): echo(GC_getStatistics())
   #echo(GC_getStatistics())
   if conf.errorCounter != 0: return
   when hasTinyCBackend:
     if conf.cmd == cmdRun:
-      tccgen.run(conf.arguments)
+      tccgen.run(conf, conf.arguments)
   if optRun in conf.globalOptions:
+    var ex = quoteShell conf.absOutFile
     if conf.cmd == cmdCompileToJS:
-      var ex: string
-      if not conf.outFile.isEmpty:
-        ex = conf.outFile.prependCurDir.quoteShell
-      else:
-        ex = quoteShell(
-          completeCFilePath(conf, changeFileExt(conf.projectFull, "js").prependCurDir))
       execExternalProgram(conf, findNodeJs() & " " & ex & ' ' & conf.arguments)
     else:
-      var binPath: AbsoluteFile
-      if not conf.outFile.isEmpty:
-        # If the user specified an outFile path, use that directly.
-        binPath = conf.outFile.prependCurDir
-      else:
-        # Figure out ourselves a valid binary name.
-        binPath = changeFileExt(conf.projectFull, ExeExt).prependCurDir
-      var ex = quoteShell(binPath)
       execExternalProgram(conf, ex & ' ' & conf.arguments)
 
 when declared(GC_setMaxPause):

compiler/nimblecmd.nim

@@ -48,7 +48,7 @@ proc `<`*(ver: Version, ver2: Version): bool =
   # Handling for normal versions such as "0.1.0" or "1.0".
   var sVer = string(ver).split('.')
   var sVer2 = string(ver2).split('.')
-  for i in 0..max(sVer.len, sVer2.len)-1:
+  for i in 0..<max(sVer.len, sVer2.len):
     var sVerI = 0
     if i < sVer.len:
       discard parseInt(sVer[i], sVerI)
@@ -69,9 +69,10 @@ proc getPathVersion*(p: string): tuple[name, version: string] =
   result.version = ""
 
   const specialSeparator = "-#"
-  var sepIdx = p.find(specialSeparator)
+  let last = p.rfind(p.lastPathPart) # the index where the last path part begins
+  var sepIdx = p.find(specialSeparator, start = last)
   if sepIdx == -1:
-    sepIdx = p.rfind('-')
+    sepIdx = p.rfind('-', start = last)
 
   if sepIdx == -1:
     result.name = p
@@ -82,7 +83,7 @@ proc getPathVersion*(p: string): tuple[name, version: string] =
       result.name = p
       return
 
-  result.name = p[0 .. sepIdx - 1]
+  result.name = p[0..sepIdx - 1]
   result.version = p.substr(sepIdx + 1)
 
 proc addPackage(conf: ConfigRef; packages: StringTableRef, p: string; info: TLineInfo) =
@@ -129,6 +130,10 @@ proc addPathRec(conf: ConfigRef; dir: string, info: TLineInfo) =
 proc nimblePath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
   addPathRec(conf, path.string, info)
   addNimblePath(conf, path.string, info)
+  let i = conf.nimblePaths.find(path)
+  if i != -1:
+    conf.nimblePaths.delete(i)
+  conf.nimblePaths.insert(path, 0)
 
 when isMainModule:
   proc v(s: string): Version = s.newVersion
@@ -142,16 +147,16 @@ when isMainModule:
 
   let conf = newConfigRef()
   var rr = newStringTable()
-  addPackage conf, rr, "irc-#a111", unknownLineInfo()
-  addPackage conf, rr, "irc-#head", unknownLineInfo()
-  addPackage conf, rr, "irc-0.1.0", unknownLineInfo()
-  #addPackage conf, rr, "irc", unknownLineInfo()
-  #addPackage conf, rr, "another", unknownLineInfo()
-  addPackage conf, rr, "another-0.1", unknownLineInfo()
+  addPackage conf, rr, "irc-#a111", unknownLineInfo
+  addPackage conf, rr, "irc-#head", unknownLineInfo
+  addPackage conf, rr, "irc-0.1.0", unknownLineInfo
+  #addPackage conf, rr, "irc", unknownLineInfo
+  #addPackage conf, rr, "another", unknownLineInfo
+  addPackage conf, rr, "another-0.1", unknownLineInfo
 
-  addPackage conf, rr, "ab-0.1.3", unknownLineInfo()
-  addPackage conf, rr, "ab-0.1", unknownLineInfo()
-  addPackage conf, rr, "justone-1.0", unknownLineInfo()
+  addPackage conf, rr, "ab-0.1.3", unknownLineInfo
+  addPackage conf, rr, "ab-0.1", unknownLineInfo
+  addPackage conf, rr, "justone-1.0", unknownLineInfo
 
   doAssert toSeq(rr.chosen) ==
     @["irc-#head", "another-0.1", "ab-0.1.3", "justone-1.0"]

compiler/nimconf.nim

@@ -10,8 +10,8 @@
 # This module handles the reading of the config file.
 
 import
-  llstream, nversion, commands, os, strutils, msgs, platform, condsyms, lexer,
-  options, idents, wordrecg, strtabs, lineinfos, pathutils
+  llstream, commands, os, strutils, msgs, lexer,
+  options, idents, wordrecg, strtabs, lineinfos, pathutils, scriptconfig
 
 # ---------------- configuration file parser -----------------------------
 # we use Nim's scanner here to save space and work
@@ -28,7 +28,7 @@ proc parseAtom(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
     result = parseExpr(L, tok, config)
     if tok.tokType == tkParRi: ppGetTok(L, tok)
     else: lexMessage(L, errGenerated, "expected closing ')'")
-  elif tok.ident.id == ord(wNot):
+  elif tok.tokType == tkNot:
     ppGetTok(L, tok)
     result = not parseAtom(L, tok, config)
   else:
@@ -37,14 +37,14 @@ proc parseAtom(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
 
 proc parseAndExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
   result = parseAtom(L, tok, config)
-  while tok.ident.id == ord(wAnd):
+  while tok.tokType == tkAnd:
     ppGetTok(L, tok)          # skip "and"
     var b = parseAtom(L, tok, config)
     result = result and b
 
 proc parseExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
   result = parseAndExpr(L, tok, config)
-  while tok.ident.id == ord(wOr):
+  while tok.tokType == tkOr:
     ppGetTok(L, tok)          # skip "or"
     var b = parseAndExpr(L, tok, config)
     result = result or b
@@ -114,7 +114,7 @@ proc parseDirective(L: var TLexer, tok: var TToken; config: ConfigRef; condStack
   ppGetTok(L, tok)            # skip @
   case whichKeyword(tok.ident)
   of wIf:
-    setLen(condStack, len(condStack) + 1)
+    setLen(condStack, condStack.len + 1)
     let res = evalppIf(L, tok, config)
     condStack[high(condStack)] = res
     if not res: jumpToDirective(L, tok, jdElseEndif, config, condStack)
@@ -123,31 +123,31 @@ proc parseDirective(L: var TLexer, tok: var TToken; config: ConfigRef; condStack
   of wEnd: doEnd(L, tok, condStack)
   of wWrite:
     ppGetTok(L, tok)
-    msgs.msgWriteln(config, strtabs.`%`(tokToStr(tok), config.configVars,
+    msgs.msgWriteln(config, strtabs.`%`($tok, config.configVars,
                                 {useEnvironment, useKey}))
     ppGetTok(L, tok)
   else:
     case tok.ident.s.normalize
     of "putenv":
       ppGetTok(L, tok)
-      var key = tokToStr(tok)
+      var key = $tok
       ppGetTok(L, tok)
-      os.putEnv(key, tokToStr(tok))
+      os.putEnv(key, $tok)
       ppGetTok(L, tok)
     of "prependenv":
       ppGetTok(L, tok)
-      var key = tokToStr(tok)
+      var key = $tok
       ppGetTok(L, tok)
-      os.putEnv(key, tokToStr(tok) & os.getEnv(key))
+      os.putEnv(key, $tok & os.getEnv(key))
       ppGetTok(L, tok)
     of "appendenv":
       ppGetTok(L, tok)
-      var key = tokToStr(tok)
+      var key = $tok
       ppGetTok(L, tok)
-      os.putEnv(key, os.getEnv(key) & tokToStr(tok))
+      os.putEnv(key, os.getEnv(key) & $tok)
       ppGetTok(L, tok)
     else:
-      lexMessage(L, errGenerated, "invalid directive: '$1'" % tokToStr(tok))
+      lexMessage(L, errGenerated, "invalid directive: '$1'" % $tok)
 
 proc confTok(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var seq[bool]) =
   ppGetTok(L, tok)
@@ -156,44 +156,51 @@ proc confTok(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var s
 
 proc checkSymbol(L: TLexer, tok: TToken) =
   if tok.tokType notin {tkSymbol..tkInt64Lit, tkStrLit..tkTripleStrLit}:
-    lexMessage(L, errGenerated, "expected identifier, but got: " & tokToStr(tok))
+    lexMessage(L, errGenerated, "expected identifier, but got: " & $tok)
 
 proc parseAssignment(L: var TLexer, tok: var TToken;
                      config: ConfigRef; condStack: var seq[bool]) =
-  if tok.ident.s == "-" or tok.ident.s == "--":
-    confTok(L, tok, config, condStack)           # skip unnecessary prefix
+  if tok.ident != nil:
+    if tok.ident.s == "-" or tok.ident.s == "--":
+      confTok(L, tok, config, condStack)           # skip unnecessary prefix
   var info = getLineInfo(L, tok) # save for later in case of an error
   checkSymbol(L, tok)
-  var s = tokToStr(tok)
+  var s = $tok
   confTok(L, tok, config, condStack)             # skip symbol
   var val = ""
   while tok.tokType == tkDot:
-    add(s, '.')
+    s.add('.')
     confTok(L, tok, config, condStack)
     checkSymbol(L, tok)
-    add(s, tokToStr(tok))
+    s.add($tok)
     confTok(L, tok, config, condStack)
   if tok.tokType == tkBracketLe:
     # BUGFIX: val, not s!
     confTok(L, tok, config, condStack)
     checkSymbol(L, tok)
-    add(val, '[')
-    add(val, tokToStr(tok))
+    val.add('[')
+    val.add($tok)
     confTok(L, tok, config, condStack)
     if tok.tokType == tkBracketRi: confTok(L, tok, config, condStack)
     else: lexMessage(L, errGenerated, "expected closing ']'")
-    add(val, ']')
+    val.add(']')
   let percent = tok.ident != nil and tok.ident.s == "%="
   if tok.tokType in {tkColon, tkEquals} or percent:
-    if len(val) > 0: add(val, ':')
+    if val.len > 0: val.add(':')
     confTok(L, tok, config, condStack)           # skip ':' or '=' or '%'
     checkSymbol(L, tok)
-    add(val, tokToStr(tok))
+    val.add($tok)
     confTok(L, tok, config, condStack)           # skip symbol
+    if tok.tokType in {tkColon, tkEquals}:
+      val.add($tok) # add the :
+      confTok(L, tok, config, condStack)           # skip symbol
+      checkSymbol(L, tok)
+      val.add($tok) # add the token after it
+      confTok(L, tok, config, condStack)           # skip symbol
     while tok.ident != nil and tok.ident.s == "&":
       confTok(L, tok, config, condStack)
       checkSymbol(L, tok)
-      add(val, tokToStr(tok))
+      val.add($tok)
       confTok(L, tok, config, condStack)
   if percent:
     processSwitch(s, strtabs.`%`(val, config.configVars,
@@ -201,7 +208,7 @@ proc parseAssignment(L: var TLexer, tok: var TToken;
   else:
     processSwitch(s, val, passPP, info, config)
 
-proc readConfigFile(filename: AbsoluteFile; cache: IdentCache;
+proc readConfigFile*(filename: AbsoluteFile; cache: IdentCache;
                     config: ConfigRef): bool =
   var
     L: TLexer
@@ -215,7 +222,7 @@ proc readConfigFile(filename: AbsoluteFile; cache: IdentCache;
     var condStack: seq[bool] = @[]
     confTok(L, tok, config, condStack)           # read in the first token
     while tok.tokType != tkEof: parseAssignment(L, tok, config, condStack)
-    if len(condStack) > 0: lexMessage(L, errGenerated, "expected @end")
+    if condStack.len > 0: lexMessage(L, errGenerated, "expected @end")
     closeLexer(L)
     return true
 
@@ -239,19 +246,34 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef) =
   template readConfigFile(path) =
     let configPath = path
     if readConfigFile(configPath, cache, conf):
-      add(configFiles, configPath)
+      configFiles.add(configPath)
+
+  template runNimScriptIfExists(path: AbsoluteFile) =
+    let p = path # eval once
+    if fileExists(p):
+      configFiles.add(p)
+      runNimScript(cache, p, freshDefines = false, conf)
 
   if optSkipSystemConfigFile notin conf.globalOptions:
     readConfigFile(getSystemConfigPath(conf, cfg))
 
+    if cfg == DefaultConfig:
+      runNimScriptIfExists(getSystemConfigPath(conf, DefaultConfigNims))
+
   if optSkipUserConfigFile notin conf.globalOptions:
     readConfigFile(getUserConfigPath(cfg))
 
+    if cfg == DefaultConfig:
+      runNimScriptIfExists(getUserConfigPath(DefaultConfigNims))
+
   let pd = if not conf.projectPath.isEmpty: conf.projectPath else: AbsoluteDir(getCurrentDir())
   if optSkipParentConfigFiles notin conf.globalOptions:
     for dir in parentDirs(pd.string, fromRoot=true, inclusive=false):
       readConfigFile(AbsoluteDir(dir) / cfg)
 
+      if cfg == DefaultConfig:
+        runNimScriptIfExists(AbsoluteDir(dir) / DefaultConfigNims)
+
   if optSkipProjConfigFile notin conf.globalOptions:
     readConfigFile(pd / cfg)
 
@@ -262,6 +284,20 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef) =
         projectConfig = changeFileExt(conf.projectFull, "nim.cfg")
       readConfigFile(projectConfig)
 
+    if cfg == DefaultConfig:
+      runNimScriptIfExists(pd / DefaultConfigNims)
+
   for filename in configFiles:
     # delayed to here so that `hintConf` is honored
     rawMessage(conf, hintConf, filename.string)
+
+  block:
+    let scriptFile = conf.projectFull.changeFileExt("nims")
+    if conf.command != "nimsuggest":
+      runNimScriptIfExists(scriptFile)
+    else:
+      if scriptFile != conf.projectFull:
+        runNimScriptIfExists(scriptFile)
+      else:
+        # 'nimsuggest foo.nims' means to just auto-complete the NimScript file
+        discard