fixes broken CI; bump macOS version to macos-11 (#20098)

* bump macOS image on Azure CI to macos-11
* fix CI error

(cherry picked from commit 8ef509b85b)

.builds/freebsd.yml

@@ -1,5 +1,3 @@
-## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
-
 # see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd
 image: freebsd/latest
 packages:
@@ -10,23 +8,23 @@ packages:
 - devel/sfml
 - www/node
 - devel/gmake
-
-
 sources:
 - https://github.com/nim-lang/Nim
 environment:
-  NIM_TESTAMENT_BATCH: "0_1"
   CC: /usr/bin/clang
 tasks:
 - setup: |
-    set -e
     cd Nim
-    . ci/funs.sh && nimBuildCsourcesIfNeeded
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpu)
+    bin/nim c --skipUserCfg --skipParentCfg koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
-    set -e
     cd Nim
-    . ci/funs.sh && nimInternalBuildKochAndRunCI
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
 triggers:
 - action: email
   condition: failure

.builds/openbsd_0.yml

@@ -1,4 +1,4 @@
-## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
 
 image: openbsd/latest
 packages:
@@ -10,23 +10,24 @@ packages:
 - sfml
 - sdl2
 - libffi
-
-
 sources:
 - https://github.com/nim-lang/Nim
 environment:
-  NIM_TESTAMENT_BATCH: "0_2"
+  NIM_TESTAMENT_BATCH: "0_3"
   CC: /usr/bin/clang
 tasks:
 - setup: |
-    set -e
     cd Nim
-    . ci/funs.sh && nimBuildCsourcesIfNeeded
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
-    set -e
     cd Nim
-    . ci/funs.sh && nimInternalBuildKochAndRunCI
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
 triggers:
 - action: email
   condition: failure

.builds/openbsd_1.yml

@@ -1,4 +1,4 @@
-## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
 
 image: openbsd/latest
 packages:
@@ -10,23 +10,24 @@ packages:
 - sfml
 - sdl2
 - libffi
-
-
 sources:
 - https://github.com/nim-lang/Nim
 environment:
-  NIM_TESTAMENT_BATCH: "1_2"
+  NIM_TESTAMENT_BATCH: "1_3"
   CC: /usr/bin/clang
 tasks:
 - setup: |
-    set -e
     cd Nim
-    . ci/funs.sh && nimBuildCsourcesIfNeeded
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
-    set -e
     cd Nim
-    . ci/funs.sh && nimInternalBuildKochAndRunCI
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
 triggers:
 - action: email
   condition: failure

.builds/openbsd_2.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "2_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.gitattributes

@@ -4,6 +4,3 @@
 # duplicated, which is easily identifiable and fixable.
 /changelog.md merge=union
 
-# bug https://github.com/dom96/choosenim/issues/256 for WSL CRLF
-*.sh text eol=lf
-/config/build_config.txt text eol=lf

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,8 +7,6 @@ assignees: ''
 
 ---
 
-(Consider writing a PR targetting devel branch after filing this, see [contributing.html](https://nim-lang.github.io/Nim/contributing.html)).
-
 Function `echo` outputs the wrong string.
 
 ### Example

.github/ISSUE_TEMPLATE/config.yml

@@ -1,10 +0,0 @@
-contact_links:
-  - name: Forum
-    url: https://forum.nim-lang.org
-    about: Please ask and answer questions here.
-  - name: Discord
-    url: https://discord.gg/nim
-    about: Please ask and answer questions here.
-  - name: Stack Overflow
-    url: https://stackoverflow.com/questions/tagged/nim-lang
-    about: Please ask and answer questions here.

.github/workflows/ci.yml.disabled

@@ -0,0 +1,122 @@
+name: Continous Integration
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64, i386]
+        cpp: ['false', 'true']
+        pkg: ['false', 'true']
+        exclude:
+          - os: ubuntu-18.04
+            cpp: 'true'
+          - os: ubuntu-18.04
+            pkg: 'true'
+          - os: macos-10.15
+            cpu: i386
+          - os: macos-10.15
+            pkg: 'true'
+          - os: windows-2019
+            cpu: i386
+          - os: windows-2019
+            cpp: 'true'
+    name: '${{ matrix.os }} (${{ matrix.cpu }}, cpp: ${{ matrix.cpp }}, pkg: ${{ matrix.pkg }})'
+    runs-on: ${{ matrix.os }}
+    env:
+      NIM_COMPILE_TO_CPP: ${{ matrix.cpp }}
+      NIM_TEST_PACKAGES: ${{ matrix.pkg }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install node.js 8.x'
+        uses: actions/setup-node@v1
+        with:
+          node-version: '8.x'
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -yq \
+              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
+              valgrind libc6-dbg
+      - name: 'Install dependencies (Linux i386)'
+        if: runner.os == 'Linux' && matrix.cpu == 'i386'
+        run: |
+          sudo dpkg --add-architecture i386
+
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+              g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
+              libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 \
+              libffi-dev:i386
+
+          cat << EOF > bin/gcc
+          #!/bin/bash
+
+          exec $(which gcc) -m32 "\$@"
+          EOF
+          cat << EOF > bin/g++
+          #!/bin/bash
+
+          exec $(which g++) -m32 "\$@"
+          EOF
+
+          chmod 755 bin/gcc
+          chmod 755 bin/g++
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install boehmgc make sfml
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+      - name: 'Run CI'
+        shell: bash
+        run: ./koch runCI
+
+      - name: 'Show failed tests'
+        if: failure()
+        shell: bash
+        run: nim c -r tools/ci_testresults.nim

.github/workflows/ci_docs.yml

@@ -2,10 +2,10 @@ name: Nim Docs CI
 on:
   push:
     paths:
-      - 'compiler/**.nim'
+      - 'compiler/docgen.nim'
+      - 'compiler/renderverbatim.nim'
       - 'config/nimdoc.cfg'
       - 'doc/**.rst'
-      - 'doc/**.md'
       - 'doc/nimdoc.css'
       - 'lib/**.nim'
       - 'nimdoc/testproject/expected/testproject.html'
@@ -13,49 +13,43 @@ on:
       - 'tools/kochdocs.nim'
       - '.github/workflows/ci_docs.yml'
       - 'koch.nim'
-
   pull_request:
     # Run only on changes on these files.
     paths:
-      - 'compiler/**.nim'
+      - 'compiler/docgen.nim'
+      - 'compiler/renderverbatim.nim'
       - 'config/nimdoc.cfg'
       - 'doc/**.rst'
-      - 'doc/**.md'
       - 'doc/nimdoc.css'
       - 'lib/**.nim'
       - 'nimdoc/testproject/expected/testproject.html'
       - 'tools/dochack/dochack.nim'
       - 'tools/kochdocs.nim'
       - '.github/workflows/ci_docs.yml'
-      - 'koch.nim'
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
 
 jobs:
   build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
     strategy:
       fail-fast: false
       matrix:
         target: [linux, windows, osx]
         include:
           - target: linux
-            os: ubuntu-22.04
+            os: ubuntu-18.04
           - target: windows
-            os: windows-latest
+            os: windows-2019
           - target: osx
-            os: macos-15
+            os: macos-11
 
     name: ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 60 # refs bug #18178
 
     steps:
       - name: 'Checkout'
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 2
+        uses: actions/checkout@v2
 
       - name: 'Install build dependencies (macOS)'
         if: runner.os == 'macOS'
@@ -65,33 +59,64 @@ jobs:
         if: runner.os == 'Windows'
         shell: bash
         run: |
-          set -e
-          . ci/funs.sh
-          nimInternalInstallDepsWindows
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
           echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'System information'
+      - name: 'Get current csources version'
+        id: csources-version
         shell: bash
-        run: . ci/funs.sh && nimCiSystemInfo
+        run: |
+          sha=$(git ls-remote https://github.com/nim-lang/csources_v1 master | cut -f 1)
+          echo "::set-output name=sha::$sha"
 
-      - name: 'Build csourcesAny (posix)'
-        # this would work on windows and other CI use this on windows,
-        # but we ensure here that `ci/build_autogen.bat` keeps working on windows.
-        if: runner.os != 'Windows'
+      - name: 'Get prebuilt csources from cache'
+        id: csources-cache
+        uses: actions/cache@v1
+        with:
+          path: bin
+          key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}'
+
+      - name: 'Checkout csources'
+        if: steps.csources-cache.outputs.cache-hit != 'true'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources_v1
+          path: csources
+
+      - name: 'Build 1-stage compiler from csources'
         shell: bash
-        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc
-          # was previously using caching via `actions/cache@v1` but this wasn't
-          # used in other CI pipelines and it's unclear the added complexity
-          # was worth the saving; can be revisited if needed.
+        run: |
+          ext=
+          [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe
+          if [[ ! -x bin/nim-csources$ext ]]; then
+            ncpu=
+            case '${{ runner.os }}' in
+            'Linux')
+              ncpu=$(nproc)
+              ;;
+            'macOS')
+              ncpu=$(sysctl -n hw.ncpu)
+              ;;
+            'Windows')
+              ncpu=$NUMBER_OF_PROCESSORS
+              ;;
+            esac
+            [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
 
-      - name: 'Build csourcesAny (windows)'
-        if: runner.os == 'Windows'
-        shell: cmd
-        run: ci/build_autogen.bat
+            make -C csources -j $ncpu CC=gcc
+            cp bin/nim{,-csources}$ext
+          else
+            echo 'Cache hit, using prebuilt csources'
+            cp bin/nim{-csources,}$ext
+          fi
 
       - name: 'Build koch'
         shell: bash
@@ -101,6 +126,10 @@ jobs:
         shell: bash
         run: ./koch boot -d:release
 
+      - name: 'Clone fusion'
+        shell: bash
+        run: ./koch fusion
+
       - name: 'Build documentation'
         shell: bash
         run: ./koch doc --git.commit:devel
@@ -109,7 +138,7 @@ jobs:
         if: |
           github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
           matrix.target == 'linux'
-        uses: crazy-max/ghaction-github-pages@v5
+        uses: crazy-max/ghaction-github-pages@v1
         with:
           build_dir: doc/html
         env:

.github/workflows/ci_packages.yml

@@ -1,75 +1,85 @@
 name: Packages CI
-on:
-  pull_request:
-  push:
-    branches:
-      - 'devel'
-      - 'version-2-2'
-      - 'version-2-0'
-      - 'version-1-6'
-      - 'version-1-2'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+on: [push, pull_request]
 
 jobs:
   build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-14]
-        batch: ["allowed_failures", "0_3", "1_3", "2_3"] # list of `index_num`
-    name: '${{ matrix.os }} (batch: ${{ matrix.batch }})'
+        os: [ubuntu-20.04, macos-11]
+        cpu: [amd64]
+        pkg: [1, 2]
+    name: '${{ matrix.os }} (pkg: ${{ matrix.pkg }})'
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 60 # refs bug #18178
     env:
-      NIM_TEST_PACKAGES: "1"
-      NIM_TESTAMENT_BATCH: ${{ matrix.batch }}
+      NIM_TEST_PACKAGES: ${{ matrix.pkg }}
     steps:
       - name: 'Checkout'
-        uses: actions/checkout@v6
+        uses: actions/checkout@v2
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
         with:
-          fetch-depth: 2
+          repository: nim-lang/csources
+          path: csources
 
-      - name: 'Install node.js'
-        uses: actions/setup-node@v6
+      - name: 'Install node.js 12.x'
+        uses: actions/setup-node@v1
         with:
-          node-version: 24
-
+          node-version: '12.x'
       - name: 'Install dependencies (Linux amd64)'
         if: runner.os == 'Linux' && matrix.cpu == 'amd64'
         run: |
-          sudo apt-get update -qq
+          sudo apt-fast update -qq
           DEBIAN_FRONTEND='noninteractive' \
-            sudo apt-get install --no-install-recommends -yq \
+            sudo apt-fast install --no-install-recommends -yq \
               libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
-              valgrind libc6-dbg libblas-dev liblapack-dev libpcre3 xorg-dev
+              valgrind libc6-dbg libblas-dev xorg-dev
       - name: 'Install dependencies (macOS)'
         if: runner.os == 'macOS'
         run: brew install boehmgc make sfml gtk+3
-        # XXX can't find boehm and gtk on macos 13
       - name: 'Install dependencies (Windows)'
         if: runner.os == 'Windows'
         shell: bash
         run: |
-          set -e
-          . ci/funs.sh
-          nimInternalInstallDepsWindows
-          echo_run echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'System information'
+      - name: 'Build csources'
         shell: bash
-        run: . ci/funs.sh && nimCiSystemInfo
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
 
-      - name: 'Build csourcesAny'
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+      - name: 'Build koch'
         shell: bash
-        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
+        run: nim c koch
+      - name: 'Run CI'
+        shell: bash
+        run: ./koch runCI
 
-      - name: 'koch, Run CI'
+      - name: 'Show failed tests'
+        if: failure()
         shell: bash
-        run: . ci/funs.sh && nimInternalBuildKochAndRunCI
+        run: nim c -r tools/ci_testresults.nim

.github/workflows/ci_ssl.yml

@@ -0,0 +1,93 @@
+name: Nim SSL CI
+on:
+  pull_request:
+    # Run only on changes on related files
+    paths:
+      - 'lib/pure/httpclient.nim'
+      - 'lib/pure/net.nim'
+      - 'lib/pure/ssl_certs.nim'
+      - 'lib/wrappers/openssl.nim'
+      - 'tests/stdlib/thttpclient_ssl*'
+      - 'tests/untestable/thttpclient_ssl*'
+
+jobs:
+  build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64]
+    name: '${{ matrix.os }} (${{ matrix.cpu }})'
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -y libssl1.1
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install make
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build 1-stage compiler from csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build the real compiler'
+        shell: bash
+        run: ./koch boot
+
+      - name: 'Run SSL nimDisableCertificateValidation integration tests'
+        shell: bash
+        run: nim c -d:nimDisableCertificateValidation -d:ssl -r -p:. tests/untestable/thttpclient_ssl_disabled.nim
+
+      - name: 'Run SSL certificate check integration tests'
+        # Not supported on Windows due to old openssl version
+        if: runner.os != 'Windows'
+        shell: bash
+        run: nim c -d:ssl -p:. --threads:on -r tests/untestable/thttpclient_ssl.nim

.gitignore

@@ -30,7 +30,7 @@ deinstall.sh
 
 doc/html/
 doc/*.html
-doc/pdf
+doc/*.pdf
 doc/*.idx
 /web/upload
 /build/*
@@ -64,29 +64,22 @@ lib/**/*.html
 testament.db
 /tests/**/*.json
 /tests/**/*.js
-
 /csources
-/csources_v1
-
 /dist/
-# /lib/fusion # fusion is now unbundled; `git status` should reveal if it's there so users can act on it
+/lib/fusion
 
 # Private directories and files (IDEs)
 .*/
 ~*
 
-# testament cruft; TODO: generate these in a gitignore'd dir (./build) in the first place.
+# testament cruft; TODO: generate these in a gitignore'd dir in the first place.
 testresults/
 test.txt
 /test.ini
 
 tweeter.db
 tweeter_test.db
-
-/tests/megatest.nim
-/tests/ic/*_temp.nim
-/tests/navigator/*_temp.nim
-
+megatest.nim
 
 /outputExpected.txt
 /outputGotten.txt
@@ -107,5 +100,3 @@ htmldocs
 
 ## these are not needed anymore unless checkout old older versions
 nimdoc.out.css
-# except here:
-!/nimdoc/testproject/expected/*

.gitlab-ci.yml

@@ -1,6 +1,4 @@
-# xxx unused, out of date
-
-image: ubuntu:18.04
+image: ubuntu:16.04
 
 stages:
   - pre-build
@@ -56,7 +54,7 @@ test-windows:
   script:
     - call ci\deps.bat
     - nim c testament\tester
-    - testament\tester.exe all
+    - testament\tester.exe --pedantic all
   tags:
     - windows
     - fast

.travis.yml

@@ -0,0 +1,52 @@
+sudo: false
+language: c
+
+dist: xenial
+
+matrix:
+  include:
+    - os: linux
+      env:
+        - NIM_COMPILE_TO_CPP=false
+        - CPU=amd64
+
+addons:
+  apt:
+    # update the list above if more deps are introduced
+    packages:
+    - libcurl4-openssl-dev
+    - libsdl1.2-dev
+    - libgc-dev
+    - libsfml-dev
+    - libc6-dbg
+    - valgrind
+
+before_script:
+  - git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
+  - export PATH="$PWD/bin${PATH:+:$PATH}"
+  - make -C csources -j 2 LD=$CC ucpu=$CPU
+
+script:
+  - echo "travis_fold:start:nim_c_koch"
+  - nim c koch
+  - echo "travis_fold:end:nim_c_koch"
+  - echo "travis_fold:start:koch_boot"
+  - ./koch boot
+  - echo "travis_fold:end:koch_boot"
+  - echo "travis_fold:start:koch_doc"
+  - ./koch doc
+  - echo "travis_fold:end:koch_doc"
+
+before_deploy:
+  # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html
+  - cp -f ./doc/html/overview.html ./doc/html/index.html
+
+deploy:                         # https://nim-lang.github.io/Nim
+  provider: pages
+  # local-dir *has* to be a relative path from the repo root.
+  local-dir: "doc/html"
+  skip-cleanup: true
+  github-token: "$GITHUB_OAUTH_TOKEN"
+  keep-history: false
+  on:
+    branch: devel

appveyor.yml

@@ -0,0 +1,37 @@
+version: '{build}'
+
+environment:
+  DLLS_URL: https://nim-lang.org/download/dlls.zip
+  DLLS_ARCHIVE: dlls.zip
+  MINGW_DIR: mingw64
+  MINGW_URL: https://nim-lang.org/download/mingw64.7z
+  MINGW_ARCHIVE: mingw64.7z
+
+  matrix:
+    - NIM_TEST_PACKAGES: false
+    - NIM_TEST_PACKAGES: true
+
+cache:
+    - '%MINGW_ARCHIVE%'
+    - '%DLLS_ARCHIVE%'
+
+
+install:
+  - ps: Install-Product node 8 # node 8 or later is required to test js async stuff
+  - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%"
+  - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\BIN"> nul
+  - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%"
+  - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul
+  - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH%
+  - git clone --depth 1 https://github.com/nim-lang/csources
+  - cd csources
+  - build64.bat
+  - cd ..
+
+build_script:
+  - openssl version
+  - openssl version -d
+  - bin\nim c koch
+  - koch runCI
+
+deploy: off

azure-pipelines.yml

@@ -1,17 +1,12 @@
 trigger:
   branches:
     include:
-    - 'devel'
-    - 'version-*'
+    - '*'
 pr:
   branches:
     include:
     - '*'
 
-variables:
-- name: skipci
-  value: false 
-
 jobs:
 - job: packages
 
@@ -20,32 +15,29 @@ jobs:
   strategy:
     matrix:
       Linux_amd64:
-        vmImage: 'ubuntu-24.04'
+        vmImage: 'ubuntu-18.04'
+        CPU: amd64
+      Linux_i386:
+        vmImage: 'ubuntu-18.04'
+        CPU: i386
+      OSX_amd64:
+        vmImage: 'macOS-11'
+        CPU: amd64
+      OSX_amd64_cpp:
+        vmImage: 'macOS-11'
         CPU: amd64
-      # regularly breaks, refs bug #17325
-#       Linux_i386:
-#         # on 'ubuntu-16.04' (not supported anymore anyways) it errored with:
-#         # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed
-#         vmImage: 'ubuntu-18.04'
-#         CPU: i386
-      OSX_arm64:
-        vmImage: 'macos-15'
-        CPU: arm64
-      OSX_arm64_cpp:
-        vmImage: 'macos-15'
-        CPU: arm64
         NIM_COMPILE_TO_CPP: true
       Windows_amd64_batch0_3:
-        vmImage: 'windows-2025'
+        vmImage: 'windows-2019'
         CPU: amd64
         # see also: `NIM_TEST_PACKAGES`
         NIM_TESTAMENT_BATCH: "0_3"
       Windows_amd64_batch1_3:
-        vmImage: 'windows-2025'
+        vmImage: 'windows-2019'
         CPU: amd64
         NIM_TESTAMENT_BATCH: "1_3"
       Windows_amd64_batch2_3:
-        vmImage: 'windows-2025'
+        vmImage: 'windows-2019'
         CPU: amd64
         NIM_TESTAMENT_BATCH: "2_3"
 
@@ -58,115 +50,131 @@ jobs:
   steps:
     - bash: git config --global core.autocrlf false
       displayName: 'Disable auto conversion to CRLF by git (Windows-only)'
-      condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
 
     - checkout: self
-      fetchDepth: 2 # see D20210329T004830
+      fetchDepth: 1
 
-    - bash: |
-        set -e
-        . ci/funs.sh
-        if nimIsCiSkip; then
-          echo '##vso[task.setvariable variable=skipci]true'
-        fi
-      displayName: 'Check whether to skip CI'
+    - bash: git clone --depth 1 https://github.com/nim-lang/csources_v1 csources
+      displayName: 'Checkout Nim csources'
 
     - task: NodeTool@0
       inputs:
-        versionSpec: '20.x'
-      displayName: 'Install node.js 20.x'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
+        versionSpec: '12.x'
+      displayName: 'Install node.js 12.x'
 
     - bash: |
-        set -e
-        . ci/funs.sh
-        echo_run sudo add-apt-repository universe
-        echo_run sudo apt-get update -qq
+        sudo apt-fast update -qq
         DEBIAN_FRONTEND='noninteractive' \
-          echo_run sudo apt-get install --no-install-recommends -yq \
-            gcc-14 g++-14 libpcre3 liblapack-dev libpcre3 liblapack-dev libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg
-        echo_run sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 60 --slave /usr/bin/g++ g++ /usr/bin/g++-14
+          sudo apt-fast install --no-install-recommends -yq \
+            libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg
       displayName: 'Install dependencies (amd64 Linux)'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
 
     - bash: |
-        set -e
-        . ci/funs.sh
-        echo_run sudo dpkg --add-architecture i386
+        sudo dpkg --add-architecture i386
+
         # Downgrade llvm:
         # - llvm has to be downgraded to have 32bit version installed for sfml.
-
         cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
         Package: libllvm6.0
         Pin: origin "azure.archive.ubuntu.com"
         Pin-Priority: 1001
         EOF
 
-        # echo_run sudo apt-get update -qq
-        echo_run sudo apt-get update -qq || echo "failed, see bug #17343"
+        sudo apt-fast update -qq
         # `:i386` (e.g. in `libffi-dev:i386`) is needed otherwise you may get:
         # `could not load: libffi.so` during dynamic loading.
         DEBIAN_FRONTEND='noninteractive' \
-          echo_run sudo apt-get install --no-install-recommends --allow-downgrades -yq \
+          sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
             g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
             libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 libffi-dev:i386
 
-
         cat << EOF > bin/gcc
         #!/bin/bash
 
         exec $(which gcc) -m32 "\$@"
         EOF
-
         cat << EOF > bin/g++
         #!/bin/bash
 
         exec $(which g++) -m32 "\$@"
         EOF
 
-        echo_run chmod 755 bin/gcc
-        echo_run chmod 755 bin/g++
+        chmod 755 bin/gcc
+        chmod 755 bin/g++
 
       displayName: 'Install dependencies (i386 Linux)'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
 
     - bash: brew install boehmgc make sfml
       displayName: 'Install dependencies (OSX)'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Darwin'))
-      # XXX can't find boehm on macos 13
+      condition: eq(variables['Agent.OS'], 'Darwin')
 
     - bash: |
-        set -e
-        . ci/funs.sh
-        nimInternalInstallDepsWindows
-        echo_run echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
+        mkdir dist
+        curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+        curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+        7z x dist/mingw64.7z -odist
+        7z x dist/dlls.zip -obin
+        echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
 
       displayName: 'Install dependencies (Windows)'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Windows_NT'))
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
 
     - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
       displayName: 'Add build binaries to PATH'
 
-    - bash: . ci/funs.sh && nimCiSystemInfo
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
+    - bash: |
+        echo 'PATH:' "$PATH"
+        echo '##[section]gcc version'
+        gcc -v
+        echo '##[section]nodejs version'
+        node -v
+        echo '##[section]make version'
+        make -v
       displayName: 'System information'
 
-    - bash: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu=$(CPU)
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'Build csourcesAny'
+    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
+      displayName: 'Get csources version'
 
-    # this could be revived if performance justifies it (needs a few updates)
-    # - task: Cache@2
-    #   inputs:
-    #     key: 'csourcesAny | "$(Agent.OS)" | $(CPU) | $(csources_version)'
-    #     path: $(nim_csources)
-    #   condition: and(succeeded(), eq(variables['skipci'], 'false'))
-    #   displayName: 'Restore built csourcesAny'
+    - task: Cache@2
+      inputs:
+        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+        path: csources/bin
+      displayName: 'Restore built csources'
 
-    - bash: . ci/funs.sh && nimInternalBuildKochAndRunCI
-      # would could also show on failure: echo '##vso[task.complete result=Failed]'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'koch, Run CI'
+    - bash: |
+        ncpu=
+        ext=
+        case '$(Agent.OS)' in
+        'Linux')
+          ncpu=$(nproc)
+          ;;
+        'Darwin')
+          ncpu=$(sysctl -n hw.ncpu)
+          ;;
+        'Windows_NT')
+          ncpu=$NUMBER_OF_PROCESSORS
+          ext=.exe
+          ;;
+        esac
+        [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+        if [[ -x csources/bin/nim$ext ]]; then
+          echo "Found cached compiler, skipping build"
+        else
+          make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
+        fi
+
+        cp csources/bin/nim$ext bin
+      displayName: 'Build 1-stage compiler from csources'
+
+    - bash: nim c koch
+      displayName: 'Build koch'
+
+      # set result to omit the "bash exited with error code '1'" message
+    - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
+      displayName: 'Run CI'
       env:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)

bin/nim-gdb.bash

@@ -0,0 +1 @@
+nim-gdb

build_all.bat

@@ -1,29 +1,17 @@
 @echo off
-rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
-rem Build development version of the compiler; can be rerun safely
-rem bare bones version of ci/funs.sh adapted for windows.
-
-rem Read in some common shared variables (shared with other tools),
-rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file
-for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H
-SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe
-echo "building from csources: %nim_csources%"
-
-if not exist %nim_csourcesDir% (
-  git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir%
+rem build development version of the compiler; can be rerun safely
+if not exist csources (
+  git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 )
-
-if not exist %nim_csources% (
-  cd %nim_csourcesDir%
-  git checkout %nim_csourcesHash%
-  echo "%PROCESSOR_ARCHITECTURE%"
-  if "%PROCESSOR_ARCHITECTURE%"=="AMD64" (
+if not exist bin\nim.exe (
+  cd csources
+  if PROCESSOR_ARCHITECTURE == AMD64 (
     SET ARCH=64
   )
   CALL build.bat
   cd ..
-  copy /y bin\nim.exe  %nim_csources%
 )
- bin\nim.exe c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch
- koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
- koch tools --skipUserCfg --skipParentCfg --hints:off
+bin\nim.exe c --skipUserCfg --skipParentCfg koch
+koch.exe boot -d:release --skipUserCfg --skipParentCfg
+koch.exe tools --skipUserCfg --skipParentCfg
+

build_all.sh

@@ -1,17 +1,52 @@
-#!/bin/sh
-# DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
+#! /bin/sh
 
 # build development version of the compiler; can be rerun safely.
-# arguments can be passed, e.g.:
-# CC=gcc ucpu=amd64 uos=darwin
+# arguments can be passed, e.g. `--os freebsd`
 
 set -u # error on undefined variables
 set -e # exit on first error
 
-. ci/funs.sh
-nimBuildCsourcesIfNeeded "$@"
+echo_run(){
+  echo "$*"
+  "$@"
+}
 
-echo_run bin/nim c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch
-echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
-echo_run ./koch tools --skipUserCfg --skipParentCfg --hints:off
+[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources_v1.git csources
+
+nim_csources=bin/nim_csources
+
+build_nim_csources_via_script(){
+  echo_run cd csources
+  echo_run sh build.sh "$@"
+}
+
+build_nim_csources(){
+  # avoid changing dir in case of failure
+  (
+    if [ $# -ne 0 ]; then
+      # some args were passed (e.g.: `--cpu i386`), need to call build.sh
+      build_nim_csources_via_script "$@"
+    else
+      # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s)
+      makeX=make
+      unamestr=$(uname)
+      if [ "$unamestr" = 'FreeBSD' ]; then
+        makeX=gmake
+      fi
+      nCPU=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
+      which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script
+    fi
+  )
+  # keep $nim_csources in case needed to investigate bootstrap issues
+  # without having to rebuild from csources
+  echo_run cp bin/nim $nim_csources
+}
+
+[ -f $nim_csources ] || echo_run build_nim_csources $@
+
+# Note: if fails, may need to `cd csources && git pull`
+echo_run bin/nim c --skipUserCfg --skipParentCfg koch
+
+echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg
+echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools.
 

changelog.md

@@ -1,116 +1,83 @@
-# v1.8.x - yyyy-mm-dd
+# v1.6.x - yyyy-mm-dd
+
 
 
 ## Changes affecting backward compatibility
 
-- `addr` is now available for all addressable locations,
-  `unsafeAddr` is now deprecated and an alias for `addr`.
+- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected.
+  The downside is that these defines now have custom logic that doesn't apply for
+  other defines.
 
-- `io`, `assertions`, `formatfloat`, and `` dollars.`$` `` for objects are about to move out of the `system` module. You may instead import `std/syncio`, `std/assertions`, `std/formatfloat` and `std/objectdollar`.
-  The `-d:nimPreviewSlimSystem` option makes these imports required.
-
-- The `gc:v2` option is removed.
-
-- The `mainmodule` and `m` options are removed.
-
-- The `threads:on` option is now the default.
-
-- Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via
-  `experimental:flexibleOptionalParams`.
 
 ## Standard library additions and changes
+- Added support for parenthesized expressions in `strformat`
 
-- Pointer to `cstring` conversion now triggers a `[PtrToCstringConv]` warning.
-  This warning will become an error in future versions! Use a `cast` operation
-  like `cast[cstring](x)` instead.
+- Fixed buffer overflow bugs in `net`
 
-- `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`.
+- Added `sections` iterator in `parsecfg`.
 
-- Object fields now support default values, see https://nim-lang.github.io/Nim/manual.html#types-default-values-for-object-fields for details.
+- Make custom op in macros.quote work for all statements.
 
-- Redefining templates with the same signature was previously
-  allowed to support certain macro code. To do this explicitly, the
-  `{.redefine.}` pragma has been added. Note that this is only for templates.
-  Implicit redefinition of templates is now deprecated and will give an error in the future.
+- On Windows the SSL library now checks for valid certificates.
+  It uses the `cacert.pem` file for this purpose which was extracted
+  from `https://curl.se/ca/cacert.pem`. Besides
+  the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
+  now also need to ship `cacert.pem` with your `.exe` file.
 
-- Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead.
 
-- Several Standard libraries are moved to nimble packages, use `nimble` to install them:
-  - `std/punycode` => `punycode`
-  - `std/asyncftpclient` => `asyncftpclient`
-  - `std/smtp` => `smtp`
-  - `std/db_common` => `db_connector/db_common`
-  - `std/db_sqlite` => `db_connector/db_sqlite`
-  - `std/db_mysql` => `db_connector/db_mysql`
-  - `std/db_postgres` => `db_connector/db_postgres`
-  - `std/db_odbc` => `db_connector/db_odbc`
+- Make `{.requiresInit.}` pragma to work for `distinct` types.
 
-- Previously, calls like `foo(a, b): ...` or `foo(a, b) do: ...` where the final argument of
-  `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`.
-  This behavior is now deprecated. Use `foo(a, b) do (): ...` or `foo(a, b, proc () = ...)` instead.
+- Added a macros `enumLen` for returning the number of items in an enum to the
+  `typetraits.nim` module.
 
-- If no exception or any exception deriving from Exception but not Defect or CatchableError given in except, a `warnBareExcept` warning will be triggered.
+- `prelude` now works with the JavaScript target.
 
-## Standard library additions and changes
+- Added `ioutils` module containing `duplicate` and `duplicateTo` to duplicate `FileHandle` using C function `dup` and `dup2`.
 
-- `macros.parseExpr` and `macros.parseStmt` now accept an optional
-  filename argument for more informative errors.
-- Module `colors` expanded with missing colors from the CSS color standard.
-- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)).
+- The JSON module can now handle integer literals and floating point literals of arbitrary length and precision.
+  Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are kept as string literals and
+  one can use external BigNum libraries to handle these. The `parseFloat` family of functions also has now optional
+  `rawIntegers` and `rawFloats` parameters that can be used to enforce that all integer or float literals remain
+  in the "raw" string form so that client code can easily treat small and large numbers uniformly.
 
+- Added `randState` template that exposes the default random number generator. Useful for library authors.
+
+- Added `asyncdispatch.activeDescriptors` that returns the number of currently
+  active async event handles/file descriptors
+
+- Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place.
 
 ## Language changes
 
+- `nimscript` now handles `except Exception as e`
 
-- Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`,
-  allowing multiple type definitions to be injected in place of the original type definition.
+- The `cstring` doesn't support `[]=` operator in JS backend.
 
-  ```nim
-  import macros
 
-  macro multiply(amount: static int, s: untyped): untyped =
-    let name = $s[0].basename
-    result = newNimNode(nnkTypeSection)
-    for i in 1 .. amount:
-      result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2]))
-
-  type
-    Foo = object
-    Bar {.multiply: 3.} = object
-      x, y, z: int
-    Baz = object
-
-  # becomes
-
-  type
-    Foo = object
-    Bar1 = object
-      x, y, z: int
-    Bar2 = object
-      x, y, z: int
-    Bar3 = object
-      x, y, z: int
-    Baz = object
-  ```
-- [Case statement macros](manual.html#macros-case-statement-macros) are no longer experimental,
-  meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them.
 
 ## Compiler changes
 
-- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`,
-  without requiring `-d:nimVersion140` which is now a noop.
 
-- `--styleCheck`, `--hintAsError` and `--warningAsError` now only applies to the current package.
 
+- Added `unsafeIsolate` and `extract` to `std/isolation`.
+
+- `--hint:CC` now goes to stderr (like all other hints) instead of stdout.
+
+- json build instructions are now generated in `$nimcache/outFileBasename.json`
+  instead of `$nimcache/projectName.json`. This allows avoiding recompiling a given project
+  compiled with different options if the output file differs.
+
+- `--usenimcache` (implied by `nim r main`) now generates an output file that includes a hash of
+  some of the compilation options, which allows caching generated binaries:
+  nim r main # recompiles
+  nim r -d:foo main # recompiles
+  nim r main # uses cached binary
+  nim r main arg1 arg2 # ditto (runtime arguments are irrelevant)
+
+- The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch
+  enforces that every symbol is written as it was declared, not enforcing
+  the official Nim style guide. To be enabled, this has to be combined either
+  with `--styleCheck:error` or `--styleCheck:hint`.
 
 ## Tool changes
 
-- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the
-  reality better. (Nim moved away from all techniques based on "tracing".)
-
-- Nim now supports Nimble version 0.14 which added support for lock-files. This is done by
-  a simple configuration change setting that you can do yourself too. In `$nim/config/nim.cfg`
-  replace `pkgs` by `pkgs2`.
-
-- There is a new switch `--nimMainPrefix:prefix` to influence the `NimMain` that the
-  compiler produces. This is particularly useful for generating static libraries.

changelogs/changelog.md

@@ -1,31 +0,0 @@
-# v1.xx.x - yyyy-mm-dd
-
-## Changes affecting backward compatibility
-
-## Standard library additions and changes
-
-### New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall
-
-This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17.
-
-On linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall.
-
-When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be:
-```sh
-$ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details
-$ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8
-$ sh build.sh  # per unix install instructions
-$ bin/nim c koch  # per unix install instructions
-$ ./koch boot -d:release  # per unix install instructions
-$ ./koch tools -d:nimNoGetRandom  # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall
-```
-
-This is necessary to pass when building nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for stdlib (sysrand in particular).
-
-## Language changes
-
-
-## Compiler changes
-
-
-## Tool changes

changelogs/changelog_0_19_0.md

@@ -15,7 +15,7 @@
   at runtime. Use ``ref object`` with inheritance rather than ``object`` with
   inheritance to prevent this issue.
 - The ``not nil`` type annotation now has to be enabled explicitly
-  via ``{.experimental: "notnil".}`` as we are still not pleased with how this
+  via ``{.experimental: "notnil"}`` as we are still not pleased with how this
   feature works with Nim's containers.
 - The parser now warns about inconsistent spacing around binary operators as
   these can easily be confused with unary operators. This warning will likely

changelogs/changelog_1_6_0.md

@@ -1,957 +0,0 @@
-# v1.6.0 - 2021-10-14
-
-
-
-# Major new features
-
-With so many new features, pinpointing the most salient ones is a subjective exercise,
-but here are a select few:
-
-
-## `iterable[T]`
-
-The `iterable[T]` type class was added to match called iterators,
-which solves a number of long-standing issues related to iterators.
-Example:
-
-```nim
-iterator iota(n: int): int =
-  for i in 0..<n: yield i
-
-# previously, you'd need `untyped`, which caused other problems such as lack
-# of type inference, overloading issues, and MCS.
-template sumOld(a: untyped): untyped = # no type inference possible
-  var result: typeof(block:(for ai in a: ai))
-  for ai in a: result += ai
-  result
-
-assert sumOld(iota(3)) == 0 + 1 + 2
-
-# now, you can write:
-template sum[T](a: iterable[T]): T =
-  # `template sum(a: iterable): auto =` would also be possible
-  var result: T
-  for ai in a: result += ai
-  result
-
-assert sum(iota(3)) == 0 + 1 + 2 # or `iota(3).sum`
-```
-
-In particular iterable arguments can now be used with the method call syntax. For example:
-```nim
-import std/[sequtils, os]
-echo walkFiles("*").toSeq # now works
-```
-See PR [#17196](https://github.com/nim-lang/Nim/pull/17196) for additional details.
-
-
-## Strict effects
-
-The effect system was refined and there is a new `.effectsOf` annotation that does
-explicitly what was previously done implicitly. See the
-[manual](https://nim-lang.github.io/Nim/manual.html#effect-system-effectsof-annotation)
-for more details.
-To write code that is portable with older Nim versions, use this idiom:
-
-```nim
-when defined(nimHasEffectsOf):
-  {.experimental: "strictEffects".}
-else:
-  {.pragma: effectsOf.}
-
-proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.}
-```
-
-To enable the new effect system, compile with `--experimental:strictEffects`.
-See also [#18777](https://github.com/nim-lang/Nim/pull/18777) and RFC
-[#408](https://github.com/nim-lang/RFCs/issues/408).
-
-
-## Private imports and private field access
-
-A new import syntax `import foo {.all.}` now allows importing all symbols
-(public or private) from `foo`.
-This can be useful for testing purposes or for more flexibility in project organization.
-
-Example:
-```nim
-from system {.all.} as system2 import nil
-echo system2.ThisIsSystem # ThisIsSystem is private in `system`
-import os {.all.} # weirdTarget is private in `os`
-echo weirdTarget # or `os.weirdTarget`
-```
-
-Added a new module `std/importutils`, and an API `privateAccess`, which allows access
-to private fields for an object type in the current scope.
-
-Example:
-```nim
-import times
-from std/importutils import privateAccess
-block:
-  let t = now()
-  # echo t.monthdayZero # Error: undeclared field: 'monthdayZero' for type times.DateTime
-  privateAccess(typeof(t)) # enables private access in this scope
-  echo t.monthdayZero # ok
-```
-
-See PR [#17706](https://github.com/nim-lang/Nim/pull/17706) for additional details.
-
-
-## `nim --eval:cmd`
-
-Added `nim --eval:cmd` to evaluate a command directly, e.g.: `nim --eval:"echo 1"`.
-It defaults to `e` (nimscript) but can also work with other commands, e.g.:
-```bash
-find . | nim r --eval:'import strutils; for a in stdin.lines: echo a.toUpper'
-```
-
-```bash
-# use as a calculator:
-nim --eval:'echo 3.1 / (1.2+7)'
-# explore a module's APIs, including private symbols:
-nim --eval:'import os {.all.}; echo weirdTarget'
-# use a custom backend:
-nim r -b:js --eval:"import std/jsbigints; echo 2'big ** 64'big"
-```
-
-See PR [#15687](https://github.com/nim-lang/Nim/pull/15687) for more details.
-
-
-## Round-trip float to string
-
-`system.addFloat` and `system.$` now can produce string representations of
-floating point numbers that are minimal in size and possess round-trip and correct
-rounding guarantees (via the
-[Dragonbox](https://raw.githubusercontent.com/jk-jeon/dragonbox/master/other_files/Dragonbox.pdf) algorithm).
-This currently has to be enabled via `-d:nimPreviewFloatRoundtrip`.
-It is expected that this behavior becomes the new default in upcoming versions,
-as with other `nimPreviewX` define flags.
-
-Example:
-```nim
-from math import round
-let a = round(9.779999999999999, 2)
-assert a == 9.78
-echo a # with `-d:nimPreviewFloatRoundtrip`: 9.78, like in python3 (instead of  9.779999999999999)
-```
-
-
-## New `std/jsbigints` module
-
-Provides arbitrary precision integers for the JS target. See PR
-[#16409](https://github.com/nim-lang/Nim/pull/16409).
-Example:
-```nim
-import std/jsbigints
-assert 2'big ** 65'big == 36893488147419103232'big
-echo 0xdeadbeef'big shl 4'big # 59774856944n
-```
-
-
-## New `std/sysrand` module
-Cryptographically secure pseudorandom number generator,
-allows generating random numbers from a secure source provided by the operating system.
-Example:
-```nim
-import std/sysrand
-assert urandom(1234) != urandom(1234) # unlikely to fail in practice
-```
-See PR [#16459](https://github.com/nim-lang/Nim/pull/16459).
-
-
-## New module: `std/tempfiles`
-
-Allows creating temporary files and directories, see PR
-[#17361](https://github.com/nim-lang/Nim/pull/17361) and followups.
-```nim
-import std/tempfiles
-let tmpPath = genTempPath("prefix", "suffix.log", "/tmp/")
-# tmpPath looks like: /tmp/prefixpmW1P2KLsuffix.log
-
-let dir = createTempDir("tmpprefix_", "_end")
-# created dir looks like: getTempDir() / "tmpprefix_YEl9VuVj_end"
-
-let (cfile, path) = createTempFile("tmpprefix_", "_end.tmp")
-# path looks like: getTempDir() / "tmpprefix_FDCIRZA0_end.tmp"
-cfile.write "foo"
-cfile.setFilePos 0
-assert readAll(cfile) == "foo"
-close cfile
-assert readFile(path) == "foo"
-```
-
-
-## User-defined literals
-
-Custom numeric literals (e.g. `-128'bignum`) are now supported.
-Additionally, the unary minus in `-1` is now part of the integer literal, i.e.
-it is now parsed as a single token.
-This implies that edge cases like `-128'i8` finally work correctly.
-Example:
-```nim
-func `'big`*(num: cstring): JsBigInt {.importjs: "BigInt(#)".}
-assert 0xffffffffffffffff'big == (1'big shl 64'big) - 1'big
-```
-
-
-## Dot-like operators
-
-With `-d:nimPreviewDotLikeOps`, dot-like operators (operators starting with `.`,
-but not with `..`) now have the same precedence as `.`, so that `a.?b.c` is now
-parsed as `(a.?b).c` instead of `a.?(b.c)`.
-A warning is generated when a dot-like operator is used without `-d:nimPreviewDotLikeOps`.
-
-An important use case is to enable dynamic fields without affecting the
-built-in `.` operator, e.g. for `std/jsffi`, `std/json`, `pkg/nimpy`. Example:
-```nim
-import std/json
-template `.?`(a: JsonNode, b: untyped{ident}): JsonNode =
-  a[astToStr(b)]
-let j = %*{"a1": {"a2": 10}}
-assert j.?a1.?a2.getInt == 10
-```
-
-
-## Block arguments now support optional parameters
-
-This solves a major pain point for routines accepting block parameters,
-see PR [#18631](https://github.com/nim-lang/Nim/pull/18631) for details:
-
-```nim
-template fn(a = 1, b = 2, body) = discard
-fn(1, 2): # already works
-  bar
-fn(a = 1): # now works
-  bar
-```
-
-Likewise with multiple block arguments via `do`:
-```nim
-template fn(a = 1, b = 2, body1, body2) = discard
-fn(a = 1): # now works
-  bar1
-do:
-  bar2
-```
-
-
-# Other new features
-
-## New and deprecated modules
-
-The following modules were added (they are discussed in the rest of the text):
-- `std/enumutils`
-- `std/genasts`
-- `std/importutils`
-- `std/jsbigints`
-- `std/jsfetch`
-- `std/jsformdata`
-- `std/jsheaders`
-- `std/packedsets`
-- `std/setutils`
-- `std/socketstreams`
-- `std/strbasics`
-- `std/sysrand`
-- `std/tasks`
-- `std/tempfiles`
-- `std/vmutils`
-
-- Deprecated `std/mersenne`.
-- Removed deprecated `std/iup` module from stdlib; it has already moved to
-  [nimble](https://github.com/nim-lang/iup).
-
-
-## New `std/jsfetch` module
-
-Provides a wrapper for JS Fetch API.
-Example:
-```nim
-# requires -d:nimExperimentalAsyncjsThen
-import std/[jsfetch, asyncjs, jsconsole, jsffi, sugar]
-proc fn {.async.} =
-  await fetch("https://api.github.com/users/torvalds".cstring)
-    .then((response: Response) => response.json())
-    .then((json: JsObject) => console.log(json))
-    .catch((err: Error) => console.log("Request Failed", err))
-discard fn()
-```
-
-
-## New `std/tasks` module
-
-Provides basic primitives for creating parallel programs.
-Example:
-```nim
-import std/tasks
-var num = 0
-proc hello(a: int) = num+=a
-
-let b = toTask hello(13) # arguments must be isolated, see `std/isolation`
-b.invoke()
-assert num == 13
-b.invoke() # can be called again
-assert num == 26
-```
-
-
-## New module: `std/genasts`
-
-Provides an API `genAst` that avoids the problems inherent with `quote do` and can
-be used as a replacement.
-Example showing how this could be used for writing a simplified version of `unittest.check`:
-```nim
-import std/[genasts, macros, strutils]
-macro check2(cond: bool): untyped =
-  assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind
-  result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]):
-    # each local symbol we access must be explicitly captured
-    if not cond:
-      doAssert false, "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs]
-let a = 3
-check2 a*2 == a+3
-if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4'
-```
-
-See PR [#17426](https://github.com/nim-lang/Nim/pull/17426) for details.
-
-
-## New module: `std/setutils`
-
-- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`,
-  if the iterable yields a built-in settable type.
-- Added `setutils.fullSet` which returns a full built-in `set` for a valid type.
-- Added `setutils.complement` which returns the complement of a built-in `set`.
-- Added `setutils.[]=`.
-
-
-## New module: `std/enumutils`
-
-- Added `genEnumCaseStmt` macro that generates
-  case statement to parse string to enum.
-- Added `items` for enums with holes.
-- Added `symbolName` to return the `enum` symbol name ignoring the human-readable name.
-- Added `symbolRank` to return the index in which an `enum` member is listed in an enum.
-
-
-## `system`
-
-- Added `system.prepareMutation` for better support of low
-  level `moveMem`, `copyMem` operations for `gc:orc`'s copy-on-write string
-  implementation.
-- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible
-  with most other languages.
-- Added `cmpMem` to `system`.
-- `doAssertRaises` now correctly handles foreign exceptions.
-- `addInt` now supports unsigned integers.
-
-Compatibility notes:
-- `system.delete` had surprising behavior when the index passed to it was out of
-  bounds (it would delete the last entry then). Compile with `-d:nimStrictDelete` so
-  that an index error is produced instead. Be aware however that your code might depend on
-  this quirky behavior so a review process is required on your part before you can
-  use `-d:nimStrictDelete`. To make this review easier, use the `-d:nimAuditDelete`
-  switch, which pretends that `system.delete` is deprecated so that it is easier
-  to see where it was used in your code.
-  `-d:nimStrictDelete` will become the default in upcoming versions.
-- `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`.
-  Please use `char` or `uint8` instead.
-- `repr` now doesn't insert trailing newlines; the previous behavior was very inconsistent,
-  see [#16034](https://github.com/nim-lang/Nim/pull/16034).
-  Use `-d:nimLegacyReprWithNewline` for the previous behavior.
-  `repr` now also renders ASTs correctly for user defined literals, setters, `do`, etc.
-- Deprecated `any`. See RFC [#281](https://github.com/nim-lang/RFCs/issues/281).
-- The unary slice `..b` was deprecated, use `0..b` instead.
-
-
-## `std/math`
-
-- Added `almostEqual` for comparing two float values using a machine epsilon.
-- Added `clamp` which allows using a `Slice` to clamp to a value.
-- Added `ceilDiv` for integer division that rounds up.
-- Added `isNaN`.
-- Added `copySign`.
-- Added `euclDiv` and `euclMod`.
-- Added `signbit`.
-- Added `frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead.
-
-Compatibility notes:
-- `math.round` now rounds "away from zero" in the JS backend, which is consistent
-  with other backends. See [#9125](https://github.com/nim-lang/Nim/pull/9125).
-  Use `-d:nimLegacyJsRound` for the previous behavior.
-
-
-## Random number generators: `std/random`, `std/sysrand`, `std/oids`
-
-- Added `std/sysrand` module (see details above).
-- Added `randState` template that exposes the default random number generator.
-  Useful for library authors.
-- Added `initRand()` overload with no argument which uses the current time as a seed.
-- `initRand(seed)` now allows `seed == 0`.
-- Fixed overflow bugs.
-- Fix `initRand` to avoid random number sequences overlapping, refs
-  [#18744](https://github.com/nim-lang/Nim/pull/18744).
-- `std/oids` now uses `std/random`.
-
-Compatibility notes:
-- Deprecated `std/mersenne`.
-- `random.initRand(seed)` now produces non-skewed values for the first call to
-  `rand()` after initialization with a small (< 30000) seed.
-  Use `-d:nimLegacyRandomInitRand` to restore previous behavior for a transition
-  time, see PR [#17467](https://github.com/nim-lang/Nim/pull/17467).
-
-
-## `std/json`, `std/jsonutils`
-
-- With `-d:nimPreviewJsonutilsHoleyEnum`, `jsonutils` now can serialize/deserialize
-  holey enums as regular enums (via `ord`) instead of as strings.
-  It is expected that this behavior becomes the new default in upcoming versions.
- `toJson` now serializes `JsonNode` as is via reference (without a deep copy)
-   instead of treating `JsonNode` as a regular ref object,
-  this can be customized via `jsonNodeMode`.
-- `std/json` and `std/jsonutils` now serialize `NaN`, `Inf`, `-Inf` as strings,
-  so that `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]`
-  which was invalid JSON.
-- `std/json` can now handle integer literals and floating point literals of
-  arbitrary length and precision.
-  Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are
-  kept as string literals and one can use external BigNum libraries to handle these.
-  The `parseFloat` family of functions also has now optional `rawIntegers` and
-  `rawFloats` parameters that can be used to enforce that all integer or float
-  literals remain in the "raw" string form so that client code can easily treat
-  small and large numbers uniformly.
-- Added `BackwardsIndex` overload for `JsonNode`.
-- `json.%`,`json.to`, `jsonutils.fromJson`,`jsonutils.toJson` now work with `uint|uint64`
-  instead of raising (as in 1.4) or giving wrong results (as in 1.2).
-- `std/jsonutils` now handles `cstring` (including as Table key), and `set`.
-- Added `jsonutils.jsonTo` overload with `opt = Joptions()` param.
-- `jsonutils.toJson` now supports customization via `ToJsonOptions`.
-- `std/json`, `std/jsonutils` now support round-trip serialization when
-  `-d:nimPreviewFloatRoundtrip` is used.
-
-
-## `std/typetraits`, `std/compilesettings`
-
-- `distinctBase` now is identity instead of error for non distinct types.
-- `distinctBase` now allows controlling whether to be recursive or not.
-- Added `enumLen` to return the number of elements in an enum.
-- Added `HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes.
-- Added `hasClosure`.
-- Added `pointerBase` to return `T` for `ref T | ptr T`.
-- Added `compilesettings.SingleValueSetting.libPath`.
-
-
-## networking: `std/net`, `std/asyncnet`, `std/htmlgen`, `std/httpclient`, `std/asyncdispatch`, `std/asynchttpserver`, `std/httpcore`
-
-- Fixed buffer overflow bugs in `std/net`.
-- Exported `sslHandle` from `std/net` and `std/asyncnet`.
-- Added `hasDataBuffered` to `std/asyncnet`.
-- Various functions in `std/httpclient` now accept `url` of type `Uri`.
-  Moreover, the `request` function's `httpMethod` argument of type `string` was
-  deprecated in favor of `HttpMethod` `enum` type; see
-  [#15919](https://github.com/nim-lang/Nim/pull/15919).
-- Added `asyncdispatch.activeDescriptors` that returns the number of currently
-  active async event handles/file descriptors.
-- Added `getPort` to `std/asynchttpserver` to resolve OS-assigned `Port(0)`;
-  this is usually recommended instead of hardcoding ports which can lead to
-  "Address already in use" errors.
-- Fixed premature garbage collection in `std/asyncdispatch`, when a stacktrace
-  override is in place.
-- Added `httpcore.is1xx` and missing HTTP codes.
-- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals).
-
-Compatibility notes:
-- On Windows, the SSL library now checks for valid certificates.
-  For this purpose it uses the `cacert.pem` file, which was extracted
-  from `https://curl.se/ca/cacert.pem`. Besides
-  the OpenSSL DLLs (e.g. `libssl-1_1-x64.dll`, `libcrypto-1_1-x64.dll`) you
-  now also need to ship `cacert.pem` with your `.exe` file.
-
-
-## `std/hashes`
-
-- `hashes.hash` can now support `object` and `ref` (can be overloaded in user code),
-  if `-d:nimPreviewHashRef` is used. It is expected that this behavior
-  becomes the new default in upcoming versions.
-- `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`.
-  `hashes.hash(closure)` has also been improved.
-
-
-## OS: `std/os`, `std/io`, `std/socketstream`, `std/linenoise`, `std/tempfiles`
-
-- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`,
-  determining preferred I/O block size for this file object.
-- Added `os.getCacheDir()` to return platform specific cache directory.
-- Improved `os.getTempDir()`, see PR [#16914](https://github.com/nim-lang/Nim/pull/16914).
-- Added `os.isAdmin` to tell whether the caller's process is a member of the
-  Administrators local group (on Windows) or a root (on POSIX).
-- Added optional `options` argument to `copyFile`, `copyFileToDir`, and
-  `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are
-  followed (copy files symlinks point to); on Windows, `options` argument is
-  ignored and symlinks are skipped.
-- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as
-  symlinks (instead of skipping them as it was before); on Windows symlinks are
-  skipped.
-- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks
-  (instead of skipping them sometimes as it was before).
-- Added optional `followSymlinks` argument to `setFilePermissions`.
-- Added a simpler to use `io.readChars` overload.
-- Added `socketstream` module that wraps sockets in the stream interface.
-- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C).
-
-
-## Environment variable handling
-
-- Empty environment variable values are now supported across OS's and backends.
-- Environment variable APIs now work in multithreaded scenarios, by delegating to
-  direct OS calls instead of trying to keep track of the environment.
-- `putEnv`, `delEnv` now work at compile time.
-- NodeJS backend now supports osenv: `getEnv`, `putEnv`, `envPairs`, `delEnv`, `existsEnv`.
-
-Compatibility notes:
-- `std/os`: `putEnv` now raises if the first argument contains a `=`.
-
-
-## POSIX
-
-- On POSIX systems, the default signal handlers used for Nim programs (it's
-  used for printing the stacktrace on fatal signals) will now re-raise the
-  signal for the OS default handlers to handle.
-  This lets the OS perform its default actions, which might include core
-  dumping (on select signals) and notifying the parent process about the cause
-  of termination.
-- On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler`
-  for previous behavior.
-- Added `posix_utils.osReleaseFile` to get system identification from `os-release`
-  file on Linux and the BSDs.
-  ([link](https://www.freedesktop.org/software/systemd/man/os-release.html))
-- Removed undefined behavior for `posix.open`.
-
-
-## `std/prelude`
-
-- `std/strformat` is now part of `include std/prelude`.
-- Added `std/sequtils` import to `std/prelude`.
-- `std/prelude` now works with the JS target.
-- `std/prelude` can now be used via `include std/prelude`, but `include prelude` still works.
-
-
-## String manipulation: `std/strformat`, `std/strbasics`
-
-- Added support for parenthesized expressions.
-- Added support for const strings instead of just string literals.
-- Added `std/strbasics` for high-performance string operations.
-- Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`.
-
-
-## `std/wrapnils`
-
-- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding
-  issues like bug [#13063](https://github.com/nim-lang/Nim/issues/13063)
-  (which affected error messages) for modules importing `std/wrapnils`.
-- Added `??.` macro which returns an `Option`.
-- `std/wrapnils` can now be used to protect against `FieldDefect` errors in
-  case objects, generates optimal code (no overhead compared to manual
-  if-else branches), and preserves lvalue semantics which allows modifying
-  an expression.
-
-
-## Containers: `std/algorithm`, `std/lists`, `std/sequtils`, `std/options`, `std/packedsets`
-
-- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix`
-  iterators (it never worked reliably).
-- Added `algorithm.merge`.
-- In `std/lists`: renamed `append` to `add` and retained `append` as an alias;
-  added `prepend` and `prependMoved` analogously to `add` and `addMoved`;
-  added `remove` for `SinglyLinkedList`s.
-- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList`
-  and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements
-  shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes
-  its argument, `addMoved`, is also supplied.
-  See PRs [#16362](https://github.com/nim-lang/Nim/pull/16362),
-  [#16536](https://github.com/nim-lang/Nim/pull/16536).
-- New module: `std/packedsets`.
-  Generalizes `std/intsets`, see PR [#15564](https://github.com/nim-lang/Nim/pull/15564).
-
-Compatibility notes:
-- Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises
-  a defect if the slice is out of bounds, likewise with `strutils.delete`.
-- Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]`
-  in `std/algorithm`.
-- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"`
-  and `$none(int)` to `"none(int)"` instead of `"None[int]"`.
-
-
-## `std/times`
-
-- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time
-  zone offsets without colons, e.g. `UTC+7 -> +0700`.
-- Added `dateTime` and deprecated `initDateTime`.
-
-
-## `std/macros` and AST
-
-- New module `std/genasts`, see description above.
-- The required name of case statement macros for the experimental
-  `caseStmtMacros` feature has changed from `match` to `` `case` ``.
-- Tuple expressions are now parsed consistently as
-  `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`.
-- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes
-  in a collapsed way.
-  Use `-d:nimLegacyMacrosCollapseSymChoice` to get the previous behavior.
-- Made custom op in `macros.quote` work for all statements.
-
-
-## `std/sugar`
-
-- Added `sugar.dumpToString` which improves on `sugar.dump`.
-- Added an overload for the `collect` macro that infers the container type based
-  on the syntax of the last expression. Works with std seqs, tables and sets.
-
-Compatibility notes:
-- Removed support for named procs in `sugar.=>`.
-
-
-## Parsing: `std/parsecfg`, `std/strscans`, `std/uri`
-
-- Added `sections` iterator in `parsecfg`.
-- `strscans.scanf` now supports parsing single characters.
-- Added `strscans.scanTuple` which uses `strscans.scanf` internally,
-  returning a tuple which can be unpacked for easier usage of `scanf`.
-- Added `decodeQuery` to `std/uri`.
-- `parseopt.initOptParser` has been made available and `parseopt` has been
-  added back to `std/prelude` for all backends. Previously `initOptParser` was
-  unavailable if the `std/os` module did not have `paramCount` or `paramStr`,
-  but the use of these in `initOptParser` were conditionally to the runtime
-  arguments passed to it, so `initOptParser` has been changed to raise
-  `ValueError` when the real command line is not available. `parseopt` was
-  previously excluded from `std/prelude` for JS, as it could not be imported.
-
-Compatibility notes:
-- Changed the behavior of `uri.decodeQuery` when there are unencoded `=`
-  characters in the decoded values. Prior versions would raise an error. This is
-  no longer the case to comply with the HTML spec and other languages'
-  implementations. Old behavior can be obtained with
-  `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same
-  underlying code is also updated the same way.
-
-
-## JS stdlib changes
-
-- Added `std/jsbigints` module, which provides arbitrary precision integers for the JS target.
-- Added `setCurrentException` for the JS backend.
-- `writeStackTrace` is available in the JS backend now.
-- Added `then`, `catch` to `std/asyncjs` for promise pipelining, for now hidden
-  behind `-d:nimExperimentalAsyncjsThen`.
-- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API)
-  wrapper for the JS target.
-- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers)
-  wrapper for the JS target.
-- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData)
-  wrapper for the JS target.
-- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger).
-- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`.
-- Added dollar `$` and `len` for `jsre.RegExp`.
-- Added `jsconsole.jsAssert` for the JS target.
-- Added `**` to `std/jsffi`.
-- Added `copyWithin` [for `seq` and `array` for JS targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin).
-- In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`.
-  Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated.
-- Added `dom.scrollIntoView` proc with options.
-- Added `dom.setInterval`, `dom.clearInterval` overloads.
-- Merged `std/dom_extensions` into the `std/dom` module,
-  as it was a module with a single line, see RFC [#413](https://github.com/nim-lang/RFCs/issues/413).
-- `$` now gives more correct results on the JS backend.
-
-
-## JS compiler changes
-
-- `cstring` doesn't support the `[]=` operator anymore in the JS backend.
-- Array literals now use JS typed arrays when the corresponding JS typed array exists,
-  for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`.
-
-
-## VM and nimscript backend
-
-- VM now supports `addr(mystring[ind])` (index + index assignment).
-- `nimscript` now handles `except Exception as e`.
-- `nil` dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time.
-- `static[T]` now works better, refs [#17590](https://github.com/nim-lang/Nim/pull/17590),
-  [#15853](https://github.com/nim-lang/Nim/pull/15853).
-- `distinct T` conversions now work in VM.
-- `items(cstring)` now works in VM.
-- Fix `addr`, `len`, `high` in VM ([#16002](https://github.com/nim-lang/Nim/pull/16002),
-  [#16610](https://github.com/nim-lang/Nim/pull/16610)).
-- `std/cstrutils` now works in VM.
-
-
-## OS/platform-specific notes
-
-- Support for Apple silicon/M1.
-- Support for 32-bit RISC-V, refs [#16231](https://github.com/nim-lang/Nim/pull/16231).
-- Support for armv8l, refs [#18901](https://github.com/nim-lang/Nim/pull/18901).
-- Support for CROSSOS, refs [#18889](https://github.com/nim-lang/Nim/pull/18889).
-- The allocator for Nintendo Switch, which was nonfunctional because
-  of breaking changes in libnx, was removed, in favor of the new `-d:nimAllocPagesViaMalloc` option.
-- Allow reading parameters when compiling for Nintendo Switch.
-- `--nimcache` now correctly works in a cross-compilation setting.
-- Cross compilation targeting Windows was improved.
-- This now works from macOS/Linux: `nim r -d:mingw main`
-
-
-
-## Performance / memory optimizations
-
-- The comment field in PNode AST was moved to a side channel, reducing overall
-  memory usage during compilation by a factor 1.25x
-- `std/jsonutils` deserialization is now up to 20x faster.
-- `os.copyFile` is now 2.5x faster on macOS, by using `copyfile` from `copyfile.h`;
-  use `-d:nimLegacyCopyFile` for macOS < 10.5.
-- Float to string conversion is now 10x faster thanks to the Dragonbox algorithm,
-  with `-d:nimPreviewFloatRoundtrip`.
-- `newSeqWith` is 3x faster.
-- CI now supports batching (making Windows CI 2.3X faster).
-- Sets now uses the optimized `countSetBits` proc, see PR
-  [#17334](https://github.com/nim-lang/Nim/pull/17334).
-
-
-## Debugging
-
-- You can now enable/disable VM tracing in user code via `vmutils.vmTrace`.
-- `koch tools` now builds `bin/nim_dbg` which allows easy access to a debug version
-  of Nim without recompiling.
-- Added new module `compiler/debugutils` to help with debugging Nim compiler.
-- Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints` and
-  used it in more contexts; this flag which works in tandem with `--stackTraceMsgs`
-  to show user code context in compiler stacktraces.
-
-
-## Type system
-
-- `typeof(voidStmt)` now works and returns `void`.
-- `enum` values can now be overloaded. This needs to be enabled
-  via `{.experimental: "overloadableEnums".}`. We hope that this feature allows
-  for the development of more fluent (less ugly) APIs.
-  See RFC [#373](https://github.com/nim-lang/RFCs/issues/373) for more details.
-- A type conversion from one `enum` type to another now produces an `[EnumConv]` warning.
-  You should use `ord` (or `cast`, but the compiler won't help, if you misuse it) instead.
-  ```nim
-  type A = enum a1, a2
-  type B = enum b1, b2
-  echo a1.B # produces a warning
-  echo a1.ord.B # produces no warning
-  ```
-- A dangerous implicit conversion to `cstring` now triggers a `[CStringConv]` warning.
-  This warning will become an error in future versions! Use an explicit conversion
-  like `cstring(x)` in order to silence the warning.
-- There is a new warning for *any* type conversion to `enum` that can be enabled via
-  `.warning[AnyEnumConv]:on` or `--warning:AnyEnumConv:on`.
-- Reusing a type name in a different scope now works, refs [#17710](https://github.com/nim-lang/Nim/pull/17710).
-- Fixed implicit and explicit generics in procedures, refs [#18808](https://github.com/nim-lang/Nim/pull/18808).
-
-
-## New-style concepts
-
-Example:
-```nim
-type
-  Comparable = concept # no T, an atom
-    proc cmp(a, b: Self): int
-```
-The new design does not rely on `system.compiles` and may compile faster.
-See PR [#15251](https://github.com/nim-lang/Nim/pull/15251)
-and RFC [#168](https://github.com/nim-lang/RFCs/issues/168) for details.
-
-
-## Lexical / syntactic
-
-- Nim now supports a small subset of Unicode operators as operator symbols.
-  The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔".
-  To enable this feature, use `--experimental:unicodeOperators`. Note that due
-  to parser limitations you **cannot** enable this feature via a
-  pragma `{.experimental: "unicodeOperators".}` reliably, you need to enable
-  it via the command line or in a configuration file.
-- `var a {.foo.} = expr` now works inside templates (except when `foo` is overloaded).
-
-
-## Compiler messages, error messages, hints, warnings
-
-- Significant improvement to error messages involving effect mismatches,
-  see PRs [#18384](https://github.com/nim-lang/Nim/pull/18384),
-  [#18418](https://github.com/nim-lang/Nim/pull/18418).
-- Added `--declaredLocs` to show symbol declaration location in error messages.
-- Added `--spellSuggest` to show spelling suggestions on typos.
-- Added `--processing:dots|filenames|off` which customizes `hintProcessing`;
-  `--processing:filenames` shows which include/import modules are being compiled as an import stack.
-- `FieldDefect` messages now shows discriminant value + lineinfo, in all backends (C, JS, VM)
-- Added `--hintAsError` with similar semantics as `--warningAsError`.
-- Added `--unitsep:on|off` to control whether to add ASCII unit separator `\31`
-  before a newline for every generated message (potentially multiline),
-  so tooling can tell when messages start and end.
-- Added `--filenames:abs|canonical|legacyRelProj` which replaces `--listFullPaths:on|off`
-- `--hint:all:on|off` is now supported to select or deselect all hints; it
-  differs from `--hints:on|off` which acts as a (reversible) gate.
-  Likewise with `--warning:all:on|off`.
-- The style checking of the compiler now supports a `--styleCheck:usages` switch.
-  This switch enforces that every symbol is written as it was declared, not enforcing
-  the official Nim style guide. To be enabled, this has to be combined either
-  with `--styleCheck:error` or `--styleCheck:hint`.
-- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for
-  previous behavior.
-- `typedesc[Foo]` now renders as such instead of `type Foo` in compiler messages.
-- `runnableExamples` now show originating location in stacktraces on failure.
-- `SuccessX` message now shows more useful information.
-- New `DuplicateModuleImport` warning; improved `UnusedImport` and
-  `XDeclaredButNotUsed` accuracy.
-
-Compatibility notes:
-- `--hint:CC` now prints to stderr (like all other hints) instead of stdout.
-
-
-## Building and running Nim programs, configuration system
-
-- JSON build instructions are now generated in `$nimcache/outFileBasename.json`
-  instead of `$nimcache/projectName.json`. This allows avoiding recompiling a
-  given project compiled with different options if the output file differs.
-- `--usenimcache` (implied by `nim r main`) now generates an output file that includes
-  a hash of some of the compilation options, which allows caching generated binaries:
-  ```bash
-    nim r main # recompiles
-    nim r -d:foo main # recompiles
-    nim r main # uses cached binary
-    nim r main arg1 arg2 # likewise (runtime arguments are irrelevant)
-  ```
-- `nim r` now supports cross compilation from unix to windows when specifying
-  `-d:mingw` by using Wine, e.g.:
-  `nim r --eval:'import os; echo "a" / "b"'` prints `a\b`.
-- `nim` can compile version 1.4.0 as follows:
-  `nim c --lib:lib --stylecheck:off -d:nimVersion140 compiler/nim`.
-  `-d:nimVersion140` is not needed for bootstrapping, only for building 1.4.0 from devel.
-- `nim e` now accepts arbitrary file extensions for the nimscript file,
-  although `.nims` is still the preferred extension in general.
-- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected.
-  The downside is that these defines now have custom logic that doesn't apply for
-  other defines.
-
-
-## Multithreading
-
-- TLS: macOS now uses native TLS (`--tlsEmulation:off`). TLS now works with
-  `importcpp` non-POD types; such types must use `.cppNonPod` and
-  `--tlsEmulation:off`should be used.
-- Added `unsafeIsolate` and `extract` to `std/isolation`.
-- Added `std/tasks`, see description above.
-
-
-## Memory management
-
-- `--gc:arc` now bootstraps (PR [#17342](https://github.com/nim-lang/Nim/pull/17342)).
-- Lots of improvements to `gc:arc`, `gc:orc`, see PR
-  [#15697](https://github.com/nim-lang/Nim/pull/15697),
-  [#16849](https://github.com/nim-lang/Nim/pull/16849),
-  [#17993](https://github.com/nim-lang/Nim/pull/17993).
-- `--gc:orc` is now 10% faster than previously for common workloads.
-  If you have trouble with its changed behavior, compile with `-d:nimOldOrc`.
-- The `--gc:orc` algorithm was refined so that custom container types can participate in the
-  cycle collection process. See the documentation of `=trace` for more details.
-- On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`.
-  This is only supported for `--gc:orc` or `--gc:arc`.
-
-Compatibility notes:
-- `--newruntime` and `--refchecks` are deprecated,
-  use `--gc:arc`, `--gc:orc`, or `--gc:none` as appropriate instead.
-
-
-## Docgen
-
-- docgen: RST files can now use single backticks instead of double backticks and
-  correctly render in both `nim rst2html` (as before) as well as common tools rendering
-  RST directly (e.g. GitHub).
-  This is done by adding the `default-role:: code` directive inside the RST file
-  (which is now handled by `nim rst2html`).
-- Source+Edit links now appear on top of every docgen'd page when
-  `nim doc --git.url:url ...` is given.
-- Latex doc generation is revised: output `.tex` files should be compiled
-  by `xelatex` (not by `pdflatex` as before). Now default Latex settings
-  provide support for Unicode and better avoid margin overflows.
-  The minimum required version is TeXLive 2018 (or an equivalent MikTeX version).
-- The RST parser now supports footnotes, citations, admonitions, and short style
-  references with symbols.
-- The RST parser now supports Markdown table syntax.
-  Known limitations:
-  - cell alignment is not supported, i.e. alignment annotations in a delimiter
-    row (`:---`, `:--:`, `---:`) are ignored
-  - every table row must start with `|`, e.g. `| cell 1 | cell 2 |`.
-- Implemented `doc2tex` compiler command which converts documentation in
-  `.nim` files to Latex.
-- docgen now supports syntax highlighting for inline code.
-- docgen now supports same-line doc comments:
-  ```nim
-  func fn*(a: int): int = 42  ## Doc comment
-  ```
-- docgen now renders `deprecated` and other pragmas.
-- `runnableExamples` now works with templates and nested templates.
-- `runnableExamples: "-r:off"` now works for examples that should compile but not run.
-- `runnableExamples` now renders code verbatim, and produces correct code in all cases.
-- docgen now shows correct, canonical import paths in docs.
-- docgen now shows all routines in sidebar, and the proc signature is now shown in sidebar.
-
-
-## Effects and checks
-
-- Significant improvement to error messages involving effect mismatches
-- There is a new `cast` section `{.cast(uncheckedAssign).}: body` that disables some
-  compiler checks regarding `case objects`. This allows serialization libraries
-  to avoid ugly, non-portable solutions. See RFC
-  [#407](https://github.com/nim-lang/RFCs/issues/407) for more details.
-
-Compatibility notes:
-- Fixed effect tracking for borrowed procs (see [#18882](https://github.com/nim-lang/Nim/pull/18882)).
-  One consequence is that, under some circumstances, Nim could previously permit a procedure with side effects to be written with `func` - you may need to change some occurrences of `func` to `proc`.
-  To illustrate, Nim versions before 1.6.0 compile the below without error
-  ```nim
-  proc print(s: string) =
-    echo s
-
-  type
-    MyString = distinct string
-
-  proc print(s: MyString) {.borrow.}
-
-  func foo(s: MyString) =
-    print(s)
-  ```
-  but Nim 1.6.0 produces the error
-  ```
-  Error: 'foo' can have side effects
-  ```
-  similar to how we expect that
-  ```nim
-  func print(s: string) =
-    echo s
-  ```
-  produces
-  ```
-  Error: 'print' can have side effects
-  ```
-
-
-## Tools
-
-- Major improvements to `nimgrep`, see PR [#15612
-](https://github.com/nim-lang/Nim/pull/15612).
-- `fusion` is now un-bundled from Nim, `./koch fusion` will
-  install it via Nimble at a fixed hash.
-- `testament`: added `nimoutFull: bool` spec to compare full output of compiler
-  instead of a subset; many bugfixes to testament.
-
-
-## Misc/cleanups
-
-- Deprecated `TaintedString` and `--taintmode`.
-- Deprecated `--nilseqs` which is now a noop.
-- Added `-d:nimStrictMode` in CI in several places to ensure code doesn't have
-  certain hints/warnings.
-- Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`.
-- `[skip ci]` now works in azure and CI pipelines, see detail in PR
-  [#17561](https://github.com/nim-lang/Nim/pull/17561).

changelogs/changelog_X_XX_X.md

@@ -3,16 +3,13 @@
 This is an example file.
 The changes should go to changelog.md!
 
-## Changes affecting backward compatibility
-
-- `foo` now behaves differently, use `-d:nimLegacyFoo` for previous behavior.
 
 ## Standard library additions and changes
 
 - Added `example.exampleProc`.
-
 - Changed `example.foo` to take additional `bar` parameter.
 
+
 ## Language changes
 
 

ci/build.bat

@@ -0,0 +1,14 @@
+REM Some debug info
+echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%."
+gcc -v
+
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
+cd csources
+call build64.bat
+cd ..
+set PATH=%CD%\bin;%PATH%
+nim -v
+nim c koch
+koch.exe boot
+copy bin/nim bin/nimd
+koch.exe boot -d:release

ci/build.sh

@@ -0,0 +1,15 @@
+sh ci/deps.sh
+
+# Build from C sources.
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
+cd csources
+sh build.sh
+cd ..
+# Add Nim to the PATH
+export PATH=$(pwd)/bin${PATH:+:$PATH}
+# Bootstrap.
+nim -v
+nim c koch
+./koch boot
+cp bin/nim bin/nimd
+./koch boot -d:release

ci/build_autogen.bat

@@ -1,26 +0,0 @@
-@echo off
-rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
-rem Build development version of the compiler; can be rerun safely
-rem bare bones version of ci/funs.sh adapted for windows.
-
-rem Read in some common shared variables (shared with other tools),
-rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file
-for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H
-SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe
-echo "building from csources: %nim_csources%"
-
-if not exist %nim_csourcesDir% (
-  git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir%
-)
-
-if not exist %nim_csources% (
-  cd %nim_csourcesDir%
-  git checkout %nim_csourcesHash%
-  echo "%PROCESSOR_ARCHITECTURE%"
-  if "%PROCESSOR_ARCHITECTURE%"=="AMD64" (
-    SET ARCH=64
-  )
-  CALL build.bat
-  cd ..
-  copy /y bin\nim.exe  %nim_csources%
-)

ci/deps.bat

@@ -0,0 +1,4 @@
+nim e install_nimble.nims
+nim e tests/test_nimscript.nims
+nimble update
+nimble install -y zip opengl sdl1 jester@#head niminst

ci/deps.sh

@@ -0,0 +1,16 @@
+# Some debug info
+echo "Running on $CI_RUNNER_ID ($CI_RUNNER_DESCRIPTION) with tags $CI_RUNNER_TAGS."
+
+# Packages
+apt-get update -qq
+apt-get install -y -qq build-essential git libcurl4-openssl-dev libsdl1.2-dev libgc-dev nodejs
+
+gcc -v
+
+export PATH=$(pwd)/bin${PATH:+:$PATH}
+
+# Nimble deps
+nim e install_nimble.nims
+nim e tests/test_nimscript.nims
+nimble update
+nimble install zip opengl sdl1 jester@#head niminst

ci/funs.sh

@@ -1,145 +0,0 @@
-# Utilities used in CI pipelines and tooling to avoid duplication.
-# Avoid top-level statements.
-# Prefer nim scripts whenever possible.
-# functions starting with `_` are considered internal, less stable.
-
-echo_run () {
-  # echo's a command before running it, which helps understanding logs
-  echo ""
-  echo "cmd: $@" # in azure we could also use this: echo '##[section]"$@"'
-  "$@"
-}
-
-nimGetLastCommit() {
-  git log --no-merges -1 --pretty=format:"%s"
-}
-
-nimIsCiSkip(){
-  # D20210329T004830:here refs https://github.com/microsoft/azure-pipelines-agent/issues/2944
-  # `--no-merges` is needed to avoid merge commits which occur for PR's.
-  # $(Build.SourceVersionMessage) is not helpful
-  # nor is `github.event.head_commit.message` for github actions.
-  # Note: `[skip ci]` is now handled automatically for github actions, see https://github.blog/changelog/2021-02-08-github-actions-skip-pull-request-and-push-workflows-with-skip-ci/
-  commitMsg=$(nimGetLastCommit)
-  echo commitMsg: "$commitMsg"
-  if [[ $commitMsg == *"[skip ci]"* ]]; then
-    echo "skipci: true"
-    return 0
-  else
-    echo "skipci: false"
-    return 1
-  fi
-}
-
-nimInternalInstallDepsWindows(){
-  echo_run mkdir dist
-  echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
-  echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
-  echo_run 7z x dist/mingw64.7z -odist
-  echo_run 7z x dist/dlls.zip -obin
-}
-
-nimInternalBuildKochAndRunCI(){
-  echo_run nim c koch
-  if ! echo_run ./koch runCI; then
-    echo_run echo "runCI failed"
-    echo_run nim r tools/ci_testresults.nim
-    return 1
-  fi
-}
-
-nimDefineVars(){
-  . config/build_config.txt
-  nim_csources=bin/nim_csources_$nim_csourcesHash
-}
-
-_nimNumCpu(){
-  # linux: $(nproc)
-  # FreeBSD | macOS: $(sysctl -n hw.ncpu)
-  # OpenBSD: $(sysctl -n hw.ncpuonline)
-  # windows: $NUMBER_OF_PROCESSORS ?
-  if env | grep -q '^NIMCORES='; then
-    echo $NIMCORES
-  else
-    echo $(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
-  fi
-}
-
-_nimBuildCsourcesIfNeeded(){
-  # if some systems cannot use make or gmake, we could add support for calling `build.sh`
-  # but this is slower (not parallel jobs) and would require making build.sh
-  # understand the arguments passed to the makefile (e.g. `CC=gcc ucpu=amd64 uos=darwin`),
-  # instead of `--cpu amd64 --os darwin`.
-  unamestr=$(uname)
-  # uname values: https://en.wikipedia.org/wiki/Uname
-  if [ "$unamestr" = 'FreeBSD' ]; then
-    makeX=gmake
-  elif [ "$unamestr" = 'OpenBSD' ]; then
-    makeX=gmake
-  elif [ "$unamestr" = 'NetBSD' ]; then
-    makeX=gmake
-  elif [ "$unamestr" = 'CROSSOS' ]; then
-    makeX=gmake
-  else
-    makeX=make
-  fi
-  nCPU=$(_nimNumCpu)
-  echo_run which $makeX
-  # parallel jobs (5X faster on 16 cores: 10s instead of 50s)
-  echo_run $makeX -C $nim_csourcesDir -j $((nCPU + 2)) -l $nCPU "$@"
-  # keep $nim_csources in case needed to investigate bootstrap issues
-  # without having to rebuild
-  echo_run cp bin/nim $nim_csources
-}
-
-nimCiSystemInfo(){
-  nimDefineVars
-  echo_run eval echo '$'nim_csources
-  echo_run pwd
-  echo_run date
-  echo_run uname -a
-  echo_run git log --no-merges -1 --pretty=oneline
-  echo_run eval echo '$'PATH
-  echo_run gcc -v
-  echo_run node -v
-  echo_run make -v
-}
-
-nimCsourcesHash(){
-  nimDefineVars
-  echo $nim_csourcesHash
-}
-
-nimBuildCsourcesIfNeeded(){
-  # goal: allow cachine each tagged version independently
-  # to avoid rebuilding, so that tools like `git bisect`
-  # can grab a cached past version without rebuilding.
-  nimDefineVars
-  (
-    set -e
-    # avoid polluting caller scope with internal variable definitions.
-    if test -f "$nim_csources"; then
-      echo "$nim_csources exists."
-    else
-      if test -d "$nim_csourcesDir"; then
-        echo "$nim_csourcesDir exists."
-      else
-        # Note: using git tags would allow fetching just what's needed, unlike git hashes, e.g.
-        # via `git clone -q --depth 1 --branch $tag $nim_csourcesUrl`.
-        echo_run git clone -q --depth 1 -b $nim_csourcesBranch \
-            $nim_csourcesUrl "$nim_csourcesDir"
-        # old `git` versions don't support -C option, using `cd` explicitly:
-        echo_run cd "$nim_csourcesDir"
-        echo_run git checkout $nim_csourcesHash
-        echo_run cd "$OLDPWD"
-        # if needed we could also add: `git reset --hard $nim_csourcesHash`
-      fi
-      _nimBuildCsourcesIfNeeded "$@"
-    fi
-
-    echo_run rm -f bin/nim
-      # fixes bug #17913, but it's unclear why it's needed, maybe specific to MacOS Big Sur 11.3 on M1 arch?
-    echo_run cp $nim_csources bin/nim
-    echo_run $nim_csources -v
-  )
-}

ci/nsis_build.bat

@@ -0,0 +1,59 @@
+REM - Run the full testsuite;  testament\tester all
+
+REM - Uncomment the list of changes in news.txt
+REM - write a news ticker entry
+REM - Update the version
+
+REM - Generate the full docs;  koch web0
+REM - Generate the installers;
+REM - Update the version in system.nim
+REM - Test the installers
+REM - Tag the release
+REM - Merge devel into master
+REM - Update csources
+
+set NIMVER=%1
+
+Rem Build -docs file:
+koch web0
+cd web\upload
+7z a -tzip docs-%NIMVER%.zip *.html
+move /y docs-%NIMVER%.zip download
+cd ..\..
+
+Rem Build csources
+koch csources -d:release || exit /b
+
+rem Grab C sources and nimsuggest
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
+
+set PATH=%CD%\bin;%PATH%
+
+ReM Build Win32 version:
+
+set PATH=C:\Users\araq\projects\mingw32\bin;%PATH%
+cd csources
+call build.bat
+cd ..
+ReM Rebuilding koch is necessary because it uses its pointer size to determine
+ReM which mingw link to put in the NSIS installer.
+nim c --out:koch_temp koch || exit /b
+koch_temp boot -d:release || exit /b
+koch_temp nsis -d:release || exit /b
+koch_temp zip -d:release || exit /b
+dir build
+move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x32.exe || exit /b
+move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x32.zip || exit /b
+
+
+ReM Build Win64 version:
+set PATH=C:\Users\araq\projects\mingw64\bin;%PATH%
+cd csources
+call build64.bat
+cd ..
+nim c --out:koch_temp koch || exit /b
+koch_temp boot -d:release || exit /b
+koch_temp nsis -d:release || exit /b
+koch_temp zip -d:release || exit /b
+move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x64.exe || exit /b
+move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x64.zip || exit /b

compiler.nimble

@@ -0,0 +1,9 @@
+
+version = system.NimVersion
+author = "Andreas Rumpf"
+description = "Compiler package providing the compiler sources as a library."
+license = "MIT"
+
+installDirs = @["compiler", "nimsuggest"]
+
+requires "nim >= 0.14.0"

compiler/asciitables.nim

@@ -1,6 +1,6 @@
 #[
-move to std/asciitables.nim once stable, or to a fusion package
-once compiler can depend on fusion
+move to std/asciitables.nim once stable, or to a nimble paackage
+once compiler can depend on nimble
 ]#
 
 type Cell* = object
@@ -8,7 +8,7 @@ type Cell* = object
   width*, row*, col*, ncols*, nrows*: int
 
 iterator parseTableCells*(s: string, delim = '\t'): Cell =
-  ## Iterates over all cells in a `delim`-delimited `s`, after a 1st
+  ## iterates over all cells in a `delim`-delimited `s`, after a 1st
   ## pass that computes number of rows, columns, and width of each column.
   var widths: seq[int]
   var cell: Cell
@@ -69,7 +69,7 @@ iterator parseTableCells*(s: string, delim = '\t'): Cell =
     finishRow()
 
 proc alignTable*(s: string, delim = '\t', fill = ' ', sep = " "): string =
-  ## Formats a `delim`-delimited `s` representing a table; each cell is aligned
+  ## formats a `delim`-delimited `s` representing a table; each cell is aligned
   ## to a width that's computed for each column; consecutive columns are
   ## delimited by `sep`, and alignment space is filled using `fill`.
   ## More customized formatting can be done by calling `parseTableCells` directly.

compiler/ast.nim

@@ -10,7 +10,7 @@
 # abstract syntax tree + symbol table
 
 import
-  lineinfos, hashes, options, ropes, idents, int128, tables
+  lineinfos, hashes, options, ropes, idents, idgen, int128
 from strutils import toLowerAscii
 
 export int128
@@ -221,15 +221,11 @@ type
     nkBreakState,         # special break statement for easier code generation
     nkFuncDef,            # a func
     nkTupleConstr         # a tuple constructor
-    nkError               # erroneous AST node
-    nkModuleRef           # for .rod file support: A (moduleId, itemId) pair
-    nkReplayAction        # for .rod file support: A replay action
-    nkNilRodNode          # for .rod file support: a 'nil' PNode
 
   TNodeKinds* = set[TNodeKind]
 
 type
-  TSymFlag* = enum    # 49 flags!
+  TSymFlag* = enum    # 46 flags!
     sfUsed,           # read access of sym (for warnings) or simply used
     sfExported,       # symbol is exported from module
     sfFromGeneric,    # symbol is instantiation of a generic; this is needed
@@ -266,10 +262,6 @@ type
     sfShadowed,       # a symbol that was shadowed in some inner scope
     sfThread,         # proc will run as a thread
                       # variable is a thread variable
-    sfCppNonPod,      # tells compiler to treat such types as non-pod's, so that
-                      # `thread_local` is used instead of `__thread` for
-                      # {.threadvar.} + `--threads`. Only makes sense for importcpp types.
-                      # This has a performance impact so isn't set by default.
     sfCompileTime,    # proc can be evaluated at compile time
     sfConstructor,    # proc is a C++ constructor
     sfDispatcher,     # copied method symbol is the dispatcher
@@ -299,13 +291,6 @@ type
     sfUsedInFinallyOrExcept  # symbol is used inside an 'except' or 'finally'
     sfSingleUsedTemp  # For temporaries that we know will only be used once
     sfNoalias         # 'noalias' annotation, means C's 'restrict'
-    sfEffectsDelayed  # an 'effectsDelayed' parameter
-    sfGeneratedType   # A anonymous generic type that is generated by the compiler for
-                      # objects that do not have generic parameters in case one of the
-                      # object fields has one.
-                      #
-                      # This is disallowed but can cause the typechecking to go into
-                      # an infinite loop, this flag is used as a sentinel to stop it.
 
   TSymFlags* = set[TSymFlag]
 
@@ -386,7 +371,7 @@ type
     tySequence,
     tyProc,
     tyPointer, tyOpenArray,
-    tyString, tyCstring, tyForward,
+    tyString, tyCString, tyForward,
     tyInt, tyInt8, tyInt16, tyInt32, tyInt64, # signed integers
     tyFloat, tyFloat32, tyFloat64, tyFloat128,
     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64,
@@ -442,17 +427,16 @@ type
       # instantiation and prior to this it has the potential to
       # be any type.
 
-    tyConcept
-      # new style concept.
+    tyOptDeprecated
+      # 'out' parameter. Comparable to a 'var' parameter but every
+      # path must assign a value to it before it can be read from.
 
     tyVoid
       # now different from tyEmpty, hurray!
-    tyIterable
 
 static:
   # remind us when TTypeKind stops to fit in a single 64-bit word
-  # assert TTypeKind.high.ord <= 63
-  discard
+  assert TTypeKind.high.ord <= 63
 
 const
   tyPureObject* = tyTuple
@@ -471,8 +455,6 @@ const
   # consider renaming as `tyAbstractVarRange`
   abstractVarRange* = {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal,
                        tyTypeDesc, tyAlias, tyInferred, tySink, tyOwned}
-  abstractInst* = {tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias,
-                   tyInferred, tySink, tyOwned} # xxx what about tyStatic?
 
 type
   TTypeKinds* = set[TTypeKind]
@@ -502,13 +484,9 @@ type
     nfDefaultRefsParam # a default param value references another parameter
                        # the flag is applied to proc default values and to calls
     nfExecuteOnReload  # A top-level statement that will be executed during reloads
-    nfLastRead  # this node is a last read
-    nfFirstWrite# this node is a first write
-    nfHasComment # node has a comment
-    nfSkipFieldChecking # node skips field visable checking
 
   TNodeFlags* = set[TNodeFlag]
-  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 45)
+  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: ~40)
     tfVarargs,        # procedure has C styled varargs
                       # tyArray type represeting a varargs list
     tfNoSideEffect,   # procedure type does not allow side effects
@@ -575,8 +553,6 @@ type
       # (for importc types); type is fully specified, allowing to compute
       # sizeof, alignof, offsetof at CT
     tfExplicitCallConv
-    tfIsConstructor
-    tfEffectSystemWorkaround
 
   TTypeFlags* = set[TTypeFlag]
 
@@ -618,8 +594,6 @@ type
 const
   routineKinds* = {skProc, skFunc, skMethod, skIterator,
                    skConverter, skMacro, skTemplate}
-  ExportableSymKinds* = {skVar, skLet, skConst, skType, skEnumField, skStub, skAlias} + routineKinds
-
   tfUnion* = tfNoSideEffect
   tfGcSafe* = tfThread
   tfObjHasKids* = tfEnumHasHoles
@@ -662,9 +636,7 @@ type
     mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot,
     mUnaryPlusI, mBitnotI,
     mUnaryPlusF64, mUnaryMinusF64,
-    mCharToStr, mBoolToStr,
-    mIntToStr, mInt64ToStr, mFloatToStr, # for compiling nimStdlibVersion < 1.5.1 (not bootstrapping)
-    mCStrToStr,
+    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
     mStrToStr, mEnumToStr,
     mAnd, mOr,
     mImplies, mIff, mExists, mForall, mOld,
@@ -677,13 +649,13 @@ type
     mInSet, mRepr, mExit,
     mSetLengthStr, mSetLengthSeq,
     mIsPartOf, mAstToStr, mParallel,
-    mSwap, mIsNil, mArrToSeq, mOpenArrayToSeq,
+    mSwap, mIsNil, mArrToSeq,
     mNewString, mNewStringOfCap, mParseBiggestFloat,
-    mMove, mWasMoved, mDestroy, mTrace,
-    mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mAccessTypeField, mReset,
+    mMove, mWasMoved, mDestroy,
+    mDefault, mUnown, mIsolate, mAccessEnv, mReset,
     mArray, mOpenArray, mRange, mSet, mSeq, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
-    mOrdinal, mIterableType,
+    mOrdinal,
     mInt, mInt8, mInt16, mInt32, mInt64,
     mUInt, mUInt8, mUInt16, mUInt32, mUInt64,
     mFloat, mFloat32, mFloat64, mFloat128,
@@ -708,11 +680,11 @@ type
     mInstantiationInfo, mGetTypeInfo, mGetTypeInfoV2,
     mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples,
     mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf,
-    mSymIsInstantiationOf, mNodeId, mPrivateAccess
+    mSymIsInstantiationOf, mNodeId
 
 
+# things that we can evaluate safely at compile time, even if not asked for it:
 const
-  # things that we can evaluate safely at compile time, even if not asked for it:
   ctfeWhitelist* = {mNone, mSucc,
     mPred, mInc, mDec, mOrd, mLengthOpenArray,
     mLengthStr, mLengthArray, mLengthSeq,
@@ -732,38 +704,15 @@ const
     mEqRef, mEqProc, mLePtr, mLtPtr, mEqCString, mXor,
     mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI,
     mUnaryPlusF64, mUnaryMinusF64,
-    mCharToStr, mBoolToStr,
-    mIntToStr, mInt64ToStr, mFloatToStr,
-    mCStrToStr,
+    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
     mStrToStr, mEnumToStr,
     mAnd, mOr,
     mEqStr, mLeStr, mLtStr,
     mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet,
     mConStrStr, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
     mInSet, mRepr}
-  
-  generatedMagics* = {mNone, mIsolate, mFinished, mOpenArrayToSeq}
-    ## magics that are generated as normal procs in the backend
 
 type
-  ItemId* = object
-    module*: int32
-    item*: int32
-
-proc `==`*(a, b: ItemId): bool {.inline.} =
-  a.item == b.item and a.module == b.module
-
-proc hash*(x: ItemId): Hash =
-  var h: Hash = hash(x.module)
-  h = h !& hash(x.item)
-  result = !$h
-
-
-type
-  TIdObj* {.acyclic.} = object of RootObj
-    itemId*: ItemId
-  PIdObj* = ref TIdObj
-
   PNode* = ref TNode
   TNodeSeq* = seq[PNode]
   PType* = ref TType
@@ -787,8 +736,7 @@ type
       ident*: PIdent
     else:
       sons*: TNodeSeq
-    when defined(nimsuggest):
-      endInfo*: TLineInfo
+    comment*: string
 
   TStrTable* = object         # a table[PIdent] of PSym
     counter*: int
@@ -809,7 +757,7 @@ type
     locOther                  # location is something other
   TLocFlag* = enum
     lfIndirect,               # backend introduced a pointer
-    lfFullExternalName, # only used when 'conf.cmd == cmdNimfix': Indicates
+    lfFullExternalName, # only used when 'conf.cmd == cmdPretty': Indicates
       # that the symbol has been imported via 'importc: "fullname"' and
       # no format string.
     lfNoDeepCopy,             # no need for a deep copy
@@ -843,7 +791,6 @@ type
     libHeader, libDynamic
 
   TLib* = object              # also misused for headers!
-                              # keep in sync with PackedLib
     kind*: TLibKind
     generated*: bool          # needed for the backends:
     isOverriden*: bool
@@ -864,18 +811,31 @@ type
     depthLevel*: int
     symbols*: TStrTable
     parent*: PScope
-    allowPrivateAccess*: seq[PSym] #  # enable access to private fields
 
   PScope* = ref TScope
 
   PLib* = ref TLib
-  TSym* {.acyclic.} = object of TIdObj # Keep in sync with PackedSym
+  TSym* {.acyclic.} = object of TIdObj
     # proc and type instantiations are cached in the generic symbol
     case kind*: TSymKind
+    of skType, skGenericParam:
+      typeInstCache*: seq[PType]
     of routineKinds:
-      #procInstCache*: seq[PInstantiation]
-      gcUnsafetyReason*: PSym  # for better error messages regarding gcsafe
+      procInstCache*: seq[PInstantiation]
+      gcUnsafetyReason*: PSym  # for better error messages wrt gcsafe
       transformedBody*: PNode  # cached body after transf pass
+    of skModule, skPackage:
+      # modules keep track of the generic symbols they use from other modules.
+      # this is because in incremental compilation, when a module is about to
+      # be replaced with a newer version, we must decrement the usage count
+      # of all previously used generics.
+      # For 'import as' we copy the module symbol but shallowCopy the 'tab'
+      # and set the 'usedGenerics' to ... XXX gah! Better set module.name
+      # instead? But this doesn't work either. --> We need an skModuleAlias?
+      # No need, just leave it as skModule but set the owner accordingly and
+      # check for the owner when touching 'usedGenerics'.
+      usedGenerics*: seq[PInstantiation]
+      tab*: TStrTable         # interface table for modules
     of skLet, skVar, skField, skForVar:
       guard*: PSym
       bitsize*: int
@@ -885,9 +845,6 @@ type
     typ*: PType
     name*: PIdent
     info*: TLineInfo
-    when defined(nimsuggest):
-      endInfo*: TLineInfo
-      hasUserSpecifiedType*: bool  # used for determining whether to display inlay type hints
     owner*: PSym
     flags*: TSymFlags
     ast*: PNode               # syntax tree of proc, iterator, etc.:
@@ -932,13 +889,13 @@ type
     attachedAsgn,
     attachedSink,
     attachedTrace,
+    attachedDispose,
     attachedDeepCopy
 
   TType* {.acyclic.} = object of TIdObj # \
                               # types are identical iff they have the
                               # same id; there may be multiple copies of a type
                               # in memory!
-                              # Keep in sync with PackedType
     kind*: TTypeKind          # kind of type
     callConv*: TCallingConvention # for procs
     flags*: TTypeFlags        # flags of the type
@@ -955,6 +912,8 @@ type
     owner*: PSym              # the 'owner' of the type
     sym*: PSym                # types have the sym associated with them
                               # it is used for converting types to strings
+    attachedOps*: array[TTypeAttachedOp, PSym] # destructors, etc.
+    methods*: seq[(int,PSym)] # attached methods
     size*: BiggestInt         # the size of the type in bytes
                               # -1 means that the size is unkwown
     align*: int16             # the type's alignment requirements
@@ -963,8 +922,8 @@ type
     loc*: TLoc
     typeInst*: PType          # for generic instantiations the tyGenericInst that led to this
                               # type.
-    uniqueId*: ItemId         # due to a design mistake, we need to keep the real ID here as it
-                              # is required by the --incremental:on mode.
+    uniqueId*: int            # due to a design mistake, we need to keep the real ID here as it
+                              # required by the --incremental:on mode.
 
   TPair* = object
     key*, val*: RootRef
@@ -1008,45 +967,13 @@ type
   TImplication* = enum
     impUnknown, impNo, impYes
 
-template nodeId(n: PNode): int = cast[int](n)
-
-type Gconfig = object
-  # we put comments in a side channel to avoid increasing `sizeof(TNode)`, which
-  # reduces memory usage given that `PNode` is the most allocated type by far.
-  comments: Table[int, string] # nodeId => comment
-  useIc*: bool
-
-var gconfig {.threadvar.}: Gconfig
-
-proc setUseIc*(useIc: bool) = gconfig.useIc = useIc
-
-proc comment*(n: PNode): string =
-  if nfHasComment in n.flags and not gconfig.useIc:
-    # IC doesn't track comments, see `packed_ast`, so this could fail
-    result = gconfig.comments[n.nodeId]
-
-proc `comment=`*(n: PNode, a: string) =
-  let id = n.nodeId
-  if a.len > 0:
-    # if needed, we could periodically cleanup gconfig.comments when its size increases,
-    # to ensure only live nodes (and with nfHasComment) have an entry in gconfig.comments;
-    # for compiling compiler, the waste is very small:
-    # num calls to newNodeImpl: 14984160 (num of PNode allocations)
-    # size of gconfig.comments: 33585
-    # num of nodes with comments that were deleted and hence wasted: 3081
-    n.flags.incl nfHasComment
-    gconfig.comments[id] = a
-  elif nfHasComment in n.flags:
-    n.flags.excl nfHasComment
-    gconfig.comments.del(id)
-
 # BUGFIX: a module is overloadable so that a proc can have the
 # same name as an imported module. This is necessary because of
 # the poor naming choices in the standard library.
 
 const
   OverloadableSyms* = {skProc, skFunc, skMethod, skIterator,
-    skConverter, skModule, skTemplate, skMacro, skEnumField}
+    skConverter, skModule, skTemplate, skMacro}
 
   GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody,
     tyGenericParam}
@@ -1060,21 +987,23 @@ const
     tyBool, tyChar, tyEnum, tyArray, tyObject,
     tySet, tyTuple, tyRange, tyPtr, tyRef, tyVar, tyLent, tySequence, tyProc,
     tyPointer,
-    tyOpenArray, tyString, tyCstring, tyInt..tyInt64, tyFloat..tyFloat128,
+    tyOpenArray, tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128,
     tyUInt..tyUInt64}
   IntegralTypes* = {tyBool, tyChar, tyEnum, tyInt..tyInt64,
     tyFloat..tyFloat128, tyUInt..tyUInt64} # weird name because it contains tyFloat
   ConstantDataTypes*: TTypeKinds = {tyArray, tySet,
                                     tyTuple, tySequence}
-  NilableTypes*: TTypeKinds = {tyPointer, tyCstring, tyRef, tyPtr,
-    tyProc, tyError} # TODO
+  NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr,
+    tyProc, tyError}
   PtrLikeKinds*: TTypeKinds = {tyPointer, tyPtr} # for VM
+  ExportableSymKinds* = {skVar, skConst, skProc, skFunc, skMethod, skType,
+    skIterator,
+    skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
                                       nfDotSetter, nfDotField,
                                       nfIsRef, nfIsPtr, nfPreventCg, nfLL,
                                       nfFromTemplate, nfDefaultRefsParam,
-                                      nfExecuteOnReload, nfLastRead,
-                                      nfFirstWrite, nfSkipFieldChecking}
+                                      nfExecuteOnReload}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2
@@ -1099,7 +1028,6 @@ const
   declarativeDefs* = {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef}
   routineDefs* = declarativeDefs + {nkMacroDef, nkTemplateDef}
   procDefs* = nkLambdaKinds + declarativeDefs
-  callableDefs* = nkLambdaKinds + routineDefs
 
   nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice}
   nkStrKinds* = {nkStrLit..nkTripleStrLit}
@@ -1110,61 +1038,31 @@ const
 
   defaultSize = -1
   defaultAlignment = -1
-  defaultOffset* = -1
+  defaultOffset = -1
 
-proc getPIdent*(a: PNode): PIdent {.inline.} =
-  ## Returns underlying `PIdent` for `{nkSym, nkIdent}`, or `nil`.
-  case a.kind
-  of nkSym: a.sym.name
-  of nkIdent: a.ident
-  of nkOpenSymChoice, nkClosedSymChoice: a.sons[0].sym.name
-  else: nil
 
-const
-  moduleShift = when defined(cpu32): 20 else: 24
+proc getnimblePkg*(a: PSym): PSym =
+  result = a
+  while result != nil:
+    case result.kind
+    of skModule:
+      result = result.owner
+      assert result.kind == skPackage
+    of skPackage:
+      if result.owner == nil:
+        break
+      else:
+        result = result.owner
+    else:
+      assert false, $result.kind
 
-template id*(a: PIdObj): int =
-  let x = a
-  (x.itemId.module.int shl moduleShift) + x.itemId.item.int
-
-type
-  IdGenerator* = ref object # unfortunately, we really need the 'shared mutable' aspect here.
-    module*: int32
-    symId*: int32
-    typeId*: int32
-    sealed*: bool
-
-const
-  PackageModuleId* = -3'i32
-
-proc idGeneratorFromModule*(m: PSym): IdGenerator =
-  assert m.kind == skModule
-  result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0)
-
-proc nextSymId*(x: IdGenerator): ItemId {.inline.} =
-  assert(not x.sealed)
-  inc x.symId
-  result = ItemId(module: x.module, item: x.symId)
-
-proc nextTypeId*(x: IdGenerator): ItemId {.inline.} =
-  assert(not x.sealed)
-  inc x.typeId
-  result = ItemId(module: x.module, item: x.typeId)
-
-when false:
-  proc nextId*(x: IdGenerator): ItemId {.inline.} =
-    inc x.item
-    result = x[]
-
-when false:
-  proc storeBack*(dest: var IdGenerator; src: IdGenerator) {.inline.} =
-    assert dest.ItemId.module == src.ItemId.module
-    if dest.ItemId.item > src.ItemId.item:
-      echo dest.ItemId.item, " ", src.ItemId.item, " ", src.ItemId.module
-    assert dest.ItemId.item <= src.ItemId.item
-    dest = src
+proc getnimblePkgId*(a: PSym): int =
+  let b = a.getnimblePkg
+  result = if b == nil: -1 else: b.id
 
 var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things
+#var
+#  gMainPackageId*: int
 
 proc isCallExpr*(n: PNode): bool =
   result = n.kind in nkCallKinds
@@ -1174,7 +1072,11 @@ proc discardSons*(father: PNode)
 type Indexable = PNode | PType
 
 proc len*(n: Indexable): int {.inline.} =
-  result = n.sons.len
+  when defined(nimNoNilSeqs):
+    result = n.sons.len
+  else:
+    if isNil(n.sons): result = 0
+    else: result = n.sons.len
 
 proc safeLen*(n: PNode): int {.inline.} =
   ## works even for leaves.
@@ -1189,9 +1091,8 @@ proc safeArrLen*(n: PNode): int {.inline.} =
 
 proc add*(father, son: Indexable) =
   assert son != nil
-  father.sons.add(son)
-
-proc addAllowNil*(father, son: Indexable) {.inline.} =
+  when not defined(nimNoNilSeqs):
+    if isNil(father.sons): father.sons = @[]
   father.sons.add(son)
 
 template `[]`*(n: Indexable, i: int): Indexable = n.sons[i]
@@ -1200,52 +1101,12 @@ template `[]=`*(n: Indexable, i: int; x: Indexable) = n.sons[i] = x
 template `[]`*(n: Indexable, i: BackwardsIndex): Indexable = n[n.len - i.int]
 template `[]=`*(n: Indexable, i: BackwardsIndex; x: Indexable) = n[n.len - i.int] = x
 
-proc getDeclPragma*(n: PNode): PNode =
-  ## return the `nkPragma` node for declaration `n`, or `nil` if no pragma was found.
-  ## Currently only supports routineDefs + {nkTypeDef}.
-  case n.kind
-  of routineDefs:
-    if n[pragmasPos].kind != nkEmpty: result = n[pragmasPos]
-  of nkTypeDef:
-    #[
-    type F3*{.deprecated: "x3".} = int
-
-    TypeSection
-      TypeDef
-        PragmaExpr
-          Postfix
-            Ident "*"
-            Ident "F3"
-          Pragma
-            ExprColonExpr
-              Ident "deprecated"
-              StrLit "x3"
-        Empty
-        Ident "int"
-    ]#
-    if n[0].kind == nkPragmaExpr:
-      result = n[0][1]
-  else:
-    # support as needed for `nkIdentDefs` etc.
-    result = nil
-  if result != nil:
-    assert result.kind == nkPragma, $(result.kind, n.kind)
-
 when defined(useNodeIds):
   const nodeIdToDebug* = -1 # 2322968
   var gNodeId: int
 
-template newNodeImpl(info2) =
-  result = PNode(kind: kind, info: info2)
-  when false:
-    # this would add overhead, so we skip it; it results in a small amount of leaked entries
-    # for old PNode that gets re-allocated at the same address as a PNode that
-    # has `nfHasComment` set (and an entry in that table). Only `nfHasComment`
-    # should be used to test whether a PNode has a comment; gconfig.comments
-    # can contain extra entries for deleted PNode's with comments.
-    gconfig.comments.del(cast[int](result))
-
-template setIdMaybe() =
+proc newNode*(kind: TNodeKind): PNode =
+  result = PNode(kind: kind, info: unknownLineInfo)
   when defined(useNodeIds):
     result.id = gNodeId
     if result.id == nodeIdToDebug:
@@ -1253,25 +1114,27 @@ template setIdMaybe() =
       writeStackTrace()
     inc gNodeId
 
-proc newNode*(kind: TNodeKind): PNode =
-  ## new node with unknown line info, no type, and no children
-  newNodeImpl(unknownLineInfo)
-  setIdMaybe()
-
 proc newNodeI*(kind: TNodeKind, info: TLineInfo): PNode =
-  ## new node with line info, no type, and no children
-  newNodeImpl(info)
-  setIdMaybe()
+  result = PNode(kind: kind, info: info)
+  when defined(useNodeIds):
+    result.id = gNodeId
+    if result.id == nodeIdToDebug:
+      echo "KIND ", result.kind
+      writeStackTrace()
+    inc gNodeId
 
 proc newNodeI*(kind: TNodeKind, info: TLineInfo, children: int): PNode =
-  ## new node with line info, type, and children
-  newNodeImpl(info)
+  result = PNode(kind: kind, info: info)
   if children > 0:
     newSeq(result.sons, children)
-  setIdMaybe()
+  when defined(useNodeIds):
+    result.id = gNodeId
+    if result.id == nodeIdToDebug:
+      echo "KIND ", result.kind
+      writeStackTrace()
+    inc gNodeId
 
 proc newNodeIT*(kind: TNodeKind, info: TLineInfo, typ: PType): PNode =
-  ## new node with line info, type, and no children
   result = newNode(kind)
   result.info = info
   result.typ = typ
@@ -1297,25 +1160,13 @@ proc newTreeIT*(kind: TNodeKind; info: TLineInfo; typ: PType; children: varargs[
 template previouslyInferred*(t: PType): PType =
   if t.sons.len > 1: t.lastSon else: nil
 
-when false:
-  import tables, strutils
-  var x: CountTable[string]
-
-  addQuitProc proc () {.noconv.} =
-    for k, v in pairs(x):
-      echo k
-      echo v
-
-proc newSym*(symKind: TSymKind, name: PIdent, id: ItemId, owner: PSym,
+proc newSym*(symKind: TSymKind, name: PIdent, owner: PSym,
              info: TLineInfo; options: TOptions = {}): PSym =
   # generates a symbol and initializes the hash field too
-  result = PSym(name: name, kind: symKind, flags: {}, info: info, itemId: id,
+  result = PSym(name: name, kind: symKind, flags: {}, info: info, id: getID(),
                 options: options, owner: owner, offset: defaultOffset)
-  when false:
-    if id.module == 48 and id.item == 39:
-      writeStackTrace()
-      echo "kind ", symKind, " ", name.s
-      if owner != nil: echo owner.name.s
+  when debugIds:
+    registerId(result)
 
 proc astdef*(s: PSym): PNode =
   # get only the definition (initializer) portion of the ast
@@ -1380,7 +1231,10 @@ proc copyObjectSet*(dest: var TObjectSet, src: TObjectSet) =
   for i in 0..high(src.data): dest.data[i] = src.data[i]
 
 proc discardSons*(father: PNode) =
-  father.sons = @[]
+  when defined(nimNoNilSeqs):
+    father.sons = @[]
+  else:
+    father.sons = nil
 
 proc withInfo*(n: PNode, info: TLineInfo): PNode =
   n.info = info
@@ -1473,7 +1327,7 @@ const
   MaxLockLevel* = 1000'i16
   UnknownLockLevel* = TLockLevel(1001'i16)
   AttachedOpToStr*: array[TTypeAttachedOp, string] = [
-    "=destroy", "=copy", "=sink", "=trace", "=deepcopy"]
+    "=destroy", "=copy", "=sink", "=trace", "=dispose", "=deepcopy"]
 
 proc `$`*(x: TLockLevel): string =
   if x.ord == UnspecifiedLockLevel.ord: result = "<unspecified>"
@@ -1486,17 +1340,18 @@ proc `$`*(s: PSym): string =
   else:
     result = "<nil>"
 
-proc newType*(kind: TTypeKind, id: ItemId; owner: PSym): PType =
+proc newType*(kind: TTypeKind, owner: PSym): PType =
+  let id = getID()
   result = PType(kind: kind, owner: owner, size: defaultSize,
-                 align: defaultAlignment, itemId: id,
-                 lockLevel: UnspecifiedLockLevel,
-                 uniqueId: id)
+                 align: defaultAlignment, id: id, uniqueId: id,
+                 lockLevel: UnspecifiedLockLevel)
+  when debugIds:
+    registerId(result)
   when false:
-    if result.itemId.module == 55 and result.itemId.item == 2:
+    if result.id == 76426:
       echo "KNID ", kind
       writeStackTrace()
 
-
 proc mergeLoc(a: var TLoc, b: TLoc) =
   if a.k == low(typeof(a.k)): a.k = b.k
   if a.storage == low(typeof(a.storage)): a.storage = b.storage
@@ -1505,7 +1360,13 @@ proc mergeLoc(a: var TLoc, b: TLoc) =
   if a.r == nil: a.r = b.r
 
 proc newSons*(father: Indexable, length: int) =
-  setLen(father.sons, length)
+  when defined(nimNoNilSeqs):
+    setLen(father.sons, length)
+  else:
+    if isNil(father.sons):
+      newSeq(father.sons, length)
+    else:
+      setLen(father.sons, length)
 
 proc assignType*(dest, src: PType) =
   dest.kind = src.kind
@@ -1514,11 +1375,12 @@ proc assignType*(dest, src: PType) =
   dest.n = src.n
   dest.size = src.size
   dest.align = src.align
+  dest.attachedOps = src.attachedOps
   dest.lockLevel = src.lockLevel
   # this fixes 'type TLock = TSysLock':
   if src.sym != nil:
     if dest.sym != nil:
-      dest.sym.flags.incl src.sym.flags-{sfUsed, sfExported}
+      dest.sym.flags.incl src.sym.flags-{sfExported}
       if dest.sym.annex == nil: dest.sym.annex = src.sym.annex
       mergeLoc(dest.sym.loc, src.sym.loc)
     else:
@@ -1526,20 +1388,26 @@ proc assignType*(dest, src: PType) =
   newSons(dest, src.len)
   for i in 0..<src.len: dest[i] = src[i]
 
-proc copyType*(t: PType, id: ItemId, owner: PSym): PType =
-  result = newType(t.kind, id, owner)
+proc copyType*(t: PType, owner: PSym, keepId: bool): PType =
+  result = newType(t.kind, owner)
   assignType(result, t)
+  if keepId:
+    result.id = t.id
+  else:
+    when debugIds: registerId(result)
   result.sym = t.sym          # backend-info should not be copied
 
-proc exactReplica*(t: PType): PType =
-  result = copyType(t, t.itemId, t.owner)
+proc exactReplica*(t: PType): PType = copyType(t, t.owner, true)
 
-proc copySym*(s: PSym; id: ItemId): PSym =
-  result = newSym(s.kind, s.name, id, s.owner, s.info, s.options)
+proc copySym*(s: PSym): PSym =
+  result = newSym(s.kind, s.name, s.owner, s.info, s.options)
   #result.ast = nil            # BUGFIX; was: s.ast which made problems
   result.typ = s.typ
+  when debugIds: registerId(result)
   result.flags = s.flags
   result.magic = s.magic
+  if s.kind == skModule:
+    copyStrTable(result.tab, s.tab)
   result.options = s.options
   result.position = s.position
   result.loc = s.loc
@@ -1550,17 +1418,20 @@ proc copySym*(s: PSym; id: ItemId): PSym =
     result.bitsize = s.bitsize
     result.alignment = s.alignment
 
-proc createModuleAlias*(s: PSym, id: ItemId, newIdent: PIdent, info: TLineInfo;
+proc createModuleAlias*(s: PSym, newIdent: PIdent, info: TLineInfo;
                         options: TOptions): PSym =
-  result = newSym(s.kind, newIdent, id, s.owner, info, options)
+  result = newSym(s.kind, newIdent, s.owner, info, options)
   # keep ID!
   result.ast = s.ast
-  #result.id = s.id # XXX figure out what to do with the ID.
+  result.id = s.id
   result.flags = s.flags
+  system.shallowCopy(result.tab, s.tab)
   result.options = s.options
   result.position = s.position
   result.loc = s.loc
   result.annex = s.annex
+  # XXX once usedGenerics is used, ensure module aliases keep working!
+  assert s.usedGenerics.len == 0
 
 proc initStrTable*(x: var TStrTable) =
   x.counter = 0
@@ -1626,7 +1497,7 @@ proc propagateToOwner*(owner, elem: PType; propagateHasAsgn = true) =
   if mask != {} and propagateHasAsgn:
     let o2 = owner.skipTypes({tyGenericInst, tyAlias, tySink})
     if o2.kind in {tyTuple, tyObject, tyArray,
-                   tySequence, tySet, tyDistinct}:
+                   tySequence, tySet, tyDistinct, tyOpenArray, tyVarargs}:
       o2.flags.incl mask
       owner.flags.incl mask
 
@@ -1639,17 +1510,26 @@ proc propagateToOwner*(owner, elem: PType; propagateHasAsgn = true) =
       owner.flags.incl tfHasGCedMem
 
 proc rawAddSon*(father, son: PType; propagateHasAsgn = true) =
+  when not defined(nimNoNilSeqs):
+    if isNil(father.sons): father.sons = @[]
   father.sons.add(son)
   if not son.isNil: propagateToOwner(father, son, propagateHasAsgn)
 
 proc rawAddSonNoPropagationOfTypeFlags*(father, son: PType) =
+  when not defined(nimNoNilSeqs):
+    if isNil(father.sons): father.sons = @[]
   father.sons.add(son)
 
 proc addSonNilAllowed*(father, son: PNode) =
+  when not defined(nimNoNilSeqs):
+    if isNil(father.sons): father.sons = @[]
   father.sons.add(son)
 
 proc delSon*(father: PNode, idx: int) =
-  if father.len == 0: return
+  when defined(nimNoNilSeqs):
+    if father.len == 0: return
+  else:
+    if isNil(father.sons): return
   for i in idx..<father.len - 1: father[i] = father[i + 1]
   father.sons.setLen(father.len - 1)
 
@@ -1672,13 +1552,11 @@ proc copyNode*(src: PNode): PNode =
   of nkIdent: result.ident = src.ident
   of nkStrLit..nkTripleStrLit: result.strVal = src.strVal
   else: discard
-  when defined(nimsuggest):
-    result.endInfo = src.endInfo
 
 template transitionNodeKindCommon(k: TNodeKind) =
   let obj {.inject.} = n[]
-  n[] = TNode(kind: k, typ: obj.typ, info: obj.info, flags: obj.flags)
-  # n.comment = obj.comment # shouldn't be needed, the address doesnt' change
+  n[] = TNode(kind: k, typ: obj.typ, info: obj.info, flags: obj.flags,
+              comment: obj.comment)
   when defined(useNodeIds):
     n.id = obj.id
 
@@ -1690,16 +1568,12 @@ proc transitionIntKind*(n: PNode, kind: range[nkCharLit..nkUInt64Lit]) =
   transitionNodeKindCommon(kind)
   n.intVal = obj.intVal
 
-proc transitionIntToFloatKind*(n: PNode, kind: range[nkFloatLit..nkFloat128Lit]) =
-  transitionNodeKindCommon(kind)
-  n.floatVal = BiggestFloat(obj.intVal)
-
 proc transitionNoneToSym*(n: PNode) =
   transitionNodeKindCommon(nkSym)
 
 template transitionSymKindCommon*(k: TSymKind) =
   let obj {.inject.} = s[]
-  s[] = TSym(kind: k, itemId: obj.itemId, magic: obj.magic, typ: obj.typ, name: obj.name,
+  s[] = TSym(kind: k, id: obj.id, magic: obj.magic, typ: obj.typ, name: obj.name,
              info: obj.info, owner: obj.owner, flags: obj.flags, ast: obj.ast,
              options: obj.options, position: obj.position, offset: obj.offset,
              loc: obj.loc, annex: obj.annex, constraint: obj.constraint)
@@ -1710,9 +1584,11 @@ template transitionSymKindCommon*(k: TSymKind) =
 
 proc transitionGenericParamToType*(s: PSym) =
   transitionSymKindCommon(skType)
+  s.typeInstCache = obj.typeInstCache
 
 proc transitionRoutineSymKind*(s: PSym, kind: range[skProc..skTemplate]) =
   transitionSymKindCommon(kind)
+  s.procInstCache = obj.procInstCache
   s.gcUnsafetyReason = obj.gcUnsafetyReason
   s.transformedBody = obj.transformedBody
 
@@ -1726,8 +1602,6 @@ template copyNodeImpl(dst, src, processSonsStmt) =
   if src == nil: return
   dst = newNode(src.kind)
   dst.info = src.info
-  when defined(nimsuggest):
-    result.endInfo = src.endInfo
   dst.typ = src.typ
   dst.flags = src.flags * PersistentNodeFlags
   dst.comment = src.comment
@@ -1785,7 +1659,7 @@ proc containsNode*(n: PNode, kinds: TNodeKinds): bool =
 
 proc hasSubnodeWith*(n: PNode, kind: TNodeKind): bool =
   case n.kind
-  of nkEmpty..nkNilLit, nkFormalParams: result = n.kind == kind
+  of nkEmpty..nkNilLit: result = n.kind == kind
   else:
     for i in 0..<n.len:
       if (n[i].kind == kind) or hasSubnodeWith(n[i], kind):
@@ -1828,7 +1702,10 @@ proc getStr*(a: PNode): string =
   of nkStrLit..nkTripleStrLit: result = a.strVal
   of nkNilLit:
     # let's hope this fixes more problems than it creates:
-    result = ""
+    when defined(nimNoNilSeqs):
+      result = ""
+    else:
+      result = nil
   else:
     raiseRecoverableError("cannot extract string from invalid AST node")
     #doAssert false, "getStr"
@@ -1845,32 +1722,12 @@ proc getStrOrChar*(a: PNode): string =
     #internalError(a.info, "getStrOrChar")
     #result = ""
 
-proc isGenericParams*(n: PNode): bool {.inline.} =
-  ## used to judge whether a node is generic params.
-  n != nil and n.kind == nkGenericParams
-
-proc isGenericRoutine*(n: PNode): bool  {.inline.} =
-  n != nil and n.kind in callableDefs and n[genericParamsPos].isGenericParams
-
-proc isGenericRoutineStrict*(s: PSym): bool {.inline.} =
-  ## determines if this symbol represents a generic routine
-  ## the unusual name is so it doesn't collide and eventually replaces
-  ## `isGenericRoutine`
-  s.kind in skProcKinds and s.ast.isGenericRoutine
-
-proc isGenericRoutine*(s: PSym): bool {.inline.} =
-  ## determines if this symbol represents a generic routine or an instance of
-  ## one. This should be renamed accordingly and `isGenericRoutineStrict`
-  ## should take this name instead.
-  ##
-  ## Warning/XXX: Unfortunately, it considers a proc kind symbol flagged with
-  ## sfFromGeneric as a generic routine. Instead this should likely not be the
-  ## case and the concepts should be teased apart:
-  ## - generic definition
-  ## - generic instance
-  ## - either generic definition or instance
-  s.kind in skProcKinds and (sfFromGeneric in s.flags or
-                             s.ast.isGenericRoutine)
+proc isGenericRoutine*(s: PSym): bool =
+  case s.kind
+  of skProcKinds:
+    result = sfFromGeneric in s.flags or
+             (s.ast != nil and s.ast[genericParamsPos].kind != nkEmpty)
+  else: discard
 
 proc skipGenericOwner*(s: PSym): PSym =
   ## Generic instantiations are owned by their originating generic
@@ -1937,20 +1794,20 @@ proc skipStmtList*(n: PNode): PNode =
   else:
     result = n
 
-proc toVar*(typ: PType; kind: TTypeKind; idgen: IdGenerator): PType =
+proc toVar*(typ: PType; kind: TTypeKind): PType =
   ## If ``typ`` is not a tyVar then it is converted into a `var <typ>` and
   ## returned. Otherwise ``typ`` is simply returned as-is.
   result = typ
   if typ.kind != kind:
-    result = newType(kind, nextTypeId(idgen), typ.owner)
+    result = newType(kind, typ.owner)
     rawAddSon(result, typ)
 
-proc toRef*(typ: PType; idgen: IdGenerator): PType =
+proc toRef*(typ: PType): PType =
   ## If ``typ`` is a tyObject then it is converted into a `ref <typ>` and
   ## returned. Otherwise ``typ`` is simply returned as-is.
   result = typ
   if typ.skipTypes({tyAlias, tyGenericInst}).kind == tyObject:
-    result = newType(tyRef, nextTypeId(idgen), typ.owner)
+    result = newType(tyRef, typ.owner)
     rawAddSon(result, typ)
 
 proc toObject*(typ: PType): PType =
@@ -1961,26 +1818,6 @@ proc toObject*(typ: PType): PType =
   if t.kind == tyRef: t.lastSon
   else: typ
 
-proc toObjectFromRefPtrGeneric*(typ: PType): PType =
-  #[
-  See also `toObject`.
-  Finds the underlying `object`, even in cases like these:
-  type
-    B[T] = object f0: int
-    A1[T] = ref B[T]
-    A2[T] = ref object f1: int
-    A3 = ref object f2: int
-    A4 = object f3: int
-  ]#
-  result = typ
-  while true:
-    case result.kind
-    of tyGenericBody: result = result.lastSon
-    of tyRef, tyPtr, tyGenericInst, tyGenericInvocation, tyAlias: result = result[0]
-      # automatic dereferencing is deep, refs #18298.
-    else: break
-  assert result.sym != nil
-
 proc isImportedException*(t: PType; conf: ConfigRef): bool =
   assert t != nil
 
@@ -2018,24 +1855,25 @@ when false:
     for i in 0..<n.safeLen:
       if n[i].containsNil: return true
 
-
 template hasDestructor*(t: PType): bool = {tfHasAsgn, tfHasOwned} * t.flags != {}
 
+proc hasDisabledAsgn*(t: PType): bool =
+  t.attachedOps[attachedAsgn] != nil and sfError in t.attachedOps[attachedAsgn].flags
+
 template incompleteType*(t: PType): bool =
   t.sym != nil and {sfForward, sfNoForward} * t.sym.flags == {sfForward}
 
 template typeCompleted*(s: PSym) =
   incl s.flags, sfNoForward
 
+template getBody*(s: PSym): PNode = s.ast[bodyPos]
+
 template detailedInfo*(sym: PSym): string =
   sym.name.s
 
 proc isInlineIterator*(typ: PType): bool {.inline.} =
   typ.kind == tyProc and tfIterator in typ.flags and typ.callConv != ccClosure
 
-proc isIterator*(typ: PType): bool {.inline.} =
-  typ.kind == tyProc and tfIterator in typ.flags
-
 proc isClosureIterator*(typ: PType): bool {.inline.} =
   typ.kind == tyProc and tfIterator in typ.flags and typ.callConv == ccClosure
 
@@ -2048,8 +1886,8 @@ proc isSinkParam*(s: PSym): bool {.inline.} =
 proc isSinkType*(t: PType): bool {.inline.} =
   t.kind == tySink or tfHasOwned in t.flags
 
-proc newProcType*(info: TLineInfo; id: ItemId; owner: PSym): PType =
-  result = newType(tyProc, id, owner)
+proc newProcType*(info: TLineInfo; owner: PSym): PType =
+  result = newType(tyProc, owner)
   result.n = newNodeI(nkFormalParams, info)
   rawAddSon(result, nil) # return type
   # result.n[0] used to be `nkType`, but now it's `nkEffectList` because
@@ -2062,6 +1900,10 @@ proc addParam*(procType: PType; param: PSym) =
   procType.n.add newSymNode(param)
   rawAddSon(procType, param.typ)
 
+template destructor*(t: PType): PSym = t.attachedOps[attachedDestructor]
+template assignment*(t: PType): PSym = t.attachedOps[attachedAsgn]
+template asink*(t: PType): PSym = t.attachedOps[attachedSink]
+
 const magicsThatCanRaise = {
   mNone, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst, mEcho}
 
@@ -2079,13 +1921,9 @@ proc canRaise*(fn: PNode): bool =
   elif fn.kind == nkSym and fn.sym.magic == mEcho:
     result = true
   else:
-    # TODO check for n having sons? or just return false for now if not
-    if fn.typ != nil and fn.typ.n != nil and fn.typ.n[0].kind == nkSym:
-      result = false
-    else:
-      result = fn.typ != nil and fn.typ.n != nil and ((fn.typ.n[0].len < effectListLen) or
-        (fn.typ.n[0][exceptionEffects] != nil and
-        fn.typ.n[0][exceptionEffects].safeLen > 0))
+    result = fn.typ != nil and fn.typ.n != nil and ((fn.typ.n[0].len < effectListLen) or
+      (fn.typ.n[0][exceptionEffects] != nil and
+      fn.typ.n[0][exceptionEffects].safeLen > 0))
 
 proc toHumanStrImpl[T](kind: T, num: static int): string =
   result = $kind
@@ -2102,15 +1940,3 @@ proc toHumanStr*(kind: TTypeKind): string =
 
 proc skipAddr*(n: PNode): PNode {.inline.} =
   (if n.kind == nkHiddenAddr: n[0] else: n)
-
-proc isNewStyleConcept*(n: PNode): bool {.inline.} =
-  assert n.kind == nkTypeClassTy
-  result = n[0].kind == nkEmpty
-
-const
-  nodesToIgnoreSet* = {nkNone..pred(nkSym), succ(nkSym)..nkNilLit,
-    nkTypeSection, nkProcDef, nkConverterDef,
-    nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo,
-    nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt,
-    nkExportStmt, nkPragma, nkCommentStmt, nkBreakState,
-    nkTypeOfExpr, nkMixinStmt, nkBindStmt}

compiler/astalgo.nim

@@ -222,9 +222,9 @@ proc getNamedParamFromList*(list: PNode, ident: PIdent): PSym =
   ## gensym'ed and then they have '\`<number>' suffix that we need to
   ## ignore, see compiler / evaltempl.nim, snippet:
   ##
-  ## .. code-block:: nim
+  ##..code-block:: nim
   ##
-  ##   result.add newIdentNode(getIdent(c.ic, x.name.s & "\`gensym" & $x.id),
+  ##    result.add newIdentNode(getIdent(c.ic, x.name.s & "\`gensym" & $x.id),
   ##            if c.instLines: actual.info else: templ.info)
   for i in 1..<list.len:
     let it = list[i].sym
@@ -585,9 +585,6 @@ proc value(this: var DebugPrinter; value: PNode) =
   this.openCurly
   this.key "kind"
   this.value  value.kind
-  if value.comment.len > 0:
-    this.key "comment"
-    this.value  value.comment
   when defined(useNodeIds):
     this.key "id"
     this.value value.id
@@ -762,9 +759,9 @@ proc strTableAdd*(t: var TStrTable, n: PSym) =
 
 proc strTableInclReportConflict*(t: var TStrTable, n: PSym;
                                  onConflictKeepOld = false): PSym =
-  # if `t` has a conflicting symbol (same identifier as `n`), return it
-  # otherwise return `nil`. Incl `n` to `t` unless `onConflictKeepOld = true`
-  # and a conflict was found.
+  # returns true if n is already in the string table:
+  # It is essential that `n` is written nevertheless!
+  # This way the newest redefinition is picked by the semantic analyses!
   assert n.name != nil
   var h: Hash = n.name.h and high(t.data)
   var replaceSlot = -1
@@ -780,10 +777,9 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym;
       replaceSlot = h
     h = nextTry(h, high(t.data))
   if replaceSlot >= 0:
-    result = t.data[replaceSlot] # found it
     if not onConflictKeepOld:
       t.data[replaceSlot] = n # overwrite it with newer definition!
-    return result # but return the old one
+    return t.data[replaceSlot] # found it
   elif mustRehash(t.data.len, t.counter):
     strTableEnlarge(t)
     strTableRawInsert(t.data, n)

compiler/astmsgs.nim

@@ -1,39 +0,0 @@
-# this module avoids ast depending on msgs or vice versa
-import std/strutils
-import options, ast, msgs
-
-proc typSym*(t: PType): PSym =
-  result = t.sym
-  if result == nil and t.kind == tyGenericInst: # this might need to be refined
-    result = t[0].sym
-
-proc addDeclaredLoc*(result: var string, conf: ConfigRef; sym: PSym) =
-  result.add " [$1 declared in $2]" % [sym.kind.toHumanStr, toFileLineCol(conf, sym.info)]
-
-proc addDeclaredLocMaybe*(result: var string, conf: ConfigRef; sym: PSym) =
-  if optDeclaredLocs in conf.globalOptions and sym != nil:
-    addDeclaredLoc(result, conf, sym)
-
-proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) =
-  # xxx figure out how to resolve `tyGenericParam`, e.g. for
-  # proc fn[T](a: T, b: T) = discard
-  # fn(1.1, "a")
-  let typ = typ.skipTypes(abstractInst + {tyStatic, tySequence, tyArray, tySet, tyUserTypeClassInst, tyVar, tyRef, tyPtr} - {tyRange})
-  result.add " [$1" % typ.kind.toHumanStr
-  if typ.sym != nil:
-    result.add " declared in " & toFileLineCol(conf, typ.sym.info)
-  result.add "]"
-
-proc addDeclaredLocMaybe*(result: var string, conf: ConfigRef; typ: PType) =
-  if optDeclaredLocs in conf.globalOptions: addDeclaredLoc(result, conf, typ)
-
-template quoteExpr*(a: string): untyped =
-  ## can be used for quoting expressions in error msgs.
-  "'" & a & "'"
-
-proc genFieldDefect*(conf: ConfigRef, field: string, disc: PSym): string =
-  let obj = disc.owner.name.s # `types.typeToString` might be better, eg for generics
-  result = "field '$#' is not accessible for type '$#'" % [field, obj]
-  if optDeclaredLocs in conf.globalOptions:
-    result.add " [discriminant declared in $#]" % toFileLineCol(conf, disc.info)
-  result.add " using '$# = " % disc.name.s

compiler/canonicalizer.nim

@@ -0,0 +1,421 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements the canonalization for the various caching mechanisms.
+
+import strutils, db_sqlite, md5
+
+var db: DbConn
+
+# We *hash* the relevant information into 128 bit hashes. This should be good
+# enough to prevent any collisions.
+
+type
+  TUid = distinct MD5Digest
+
+# For name mangling we encode these hashes via a variant of base64 (called
+# 'base64a') and prepend the *primary* identifier to ease the debugging pain.
+# So a signature like:
+#
+#   proc gABI(c: PCtx; n: PNode; opc: TOpcode; a, b: TRegister; imm: BiggestInt)
+#
+# is mangled into:
+#   gABI_MTdmOWY5MTQ1MDcyNGQ3ZA
+#
+# This is a good compromise between correctness and brevity. ;-)
+
+const
+  cb64 = [
+    "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N",
+    "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
+    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n",
+    "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
+    "0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
+    "_A", "_B"]
+
+proc toBase64a(s: cstring, len: int): string =
+  ## encodes `s` into base64 representation. After `lineLen` characters, a
+  ## `newline` is added.
+  result = newStringOfCap(((len + 2) div 3) * 4)
+  var i = 0
+  while i < s.len - 2:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    let c = ord(s[i+2])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2) or ((c and 0xC0) shr 6)]
+    result.add cb64[c and 0x3F]
+    inc(i, 3)
+  if i < s.len-1:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2)]
+  elif i < s.len:
+    let a = ord(s[i])
+    result.add cb64[a shr 2]
+    result.add cb64[(a and 3) shl 4]
+
+proc toBase64a(u: TUid): string = toBase64a(cast[cstring](u), sizeof(u))
+
+proc `&=`(c: var MD5Context, s: string) = md5Update(c, s, s.len)
+
+proc hashSym(c: var MD5Context, s: PSym) =
+  if sfAnon in s.flags or s.kind == skGenericParam:
+    c &= ":anon"
+  else:
+    var it = s.owner
+    while it != nil:
+      hashSym(c, it)
+      c &= "."
+      it = s.owner
+    c &= s.name.s
+
+proc hashTree(c: var MD5Context, n: PNode) =
+  if n == nil:
+    c &= "\255"
+    return
+  var k = n.kind
+  md5Update(c, cast[cstring](addr(k)), 1)
+  # we really must not hash line information. 'n.typ' is debatable but
+  # shouldn't be necessary for now and avoids potential infinite recursions.
+  case n.kind
+  of nkEmpty, nkNilLit, nkType: discard
+  of nkIdent:
+    c &= n.ident.s
+  of nkSym:
+    hashSym(c, n.sym)
+  of nkCharLit..nkUInt64Lit:
+    var v = n.intVal
+    md5Update(c, cast[cstring](addr(v)), sizeof(v))
+  of nkFloatLit..nkFloat64Lit:
+    var v = n.floatVal
+    md5Update(c, cast[cstring](addr(v)), sizeof(v))
+  of nkStrLit..nkTripleStrLit:
+    c &= n.strVal
+  else:
+    for i in 0..<n.len: hashTree(c, n[i])
+
+proc hashType(c: var MD5Context, t: PType) =
+  # modelled after 'typeToString'
+  if t == nil:
+    c &= "\254"
+    return
+
+  var k = t.kind
+  md5Update(c, cast[cstring](addr(k)), 1)
+
+  if t.sym != nil and sfAnon notin t.sym.flags:
+    # t.n for literals, but not for e.g. objects!
+    if t.kind in {tyFloat, tyInt}: c.hashNode(t.n)
+    c.hashSym(t.sym)
+
+  case t.kind
+  of tyGenericBody, tyGenericInst, tyGenericInvocation:
+    for i in 0..<t.len-ord(t.kind != tyGenericInvocation):
+      c.hashType t[i]
+  of tyUserTypeClass:
+    internalAssert t.sym != nil and t.sym.owner != nil
+    c &= t.sym.owner.name.s
+  of tyUserTypeClassInst:
+    let body = t.base
+    c.hashSym body.sym
+    for i in 1..<t.len-1:
+      c.hashType t[i]
+  of tyFromExpr:
+    c.hashTree(t.n)
+  of tyArray:
+    c.hashTree(t[0].n)
+    c.hashType(t[1])
+  of tyTuple:
+    if t.n != nil:
+      assert(t.n.len == t.len)
+      for i in 0..<t.n.len:
+        assert(t.n[i].kind == nkSym)
+        c &= t.n[i].sym.name.s
+        c &= ":"
+        c.hashType(t[i])
+        c &= ","
+    else:
+      for i in 0..<t.len: c.hashType t[i]
+  of tyRange:
+    c.hashTree(t.n)
+    c.hashType(t[0])
+  of tyProc:
+    c &= (if tfIterator in t.flags: "iterator " else: "proc ")
+    for i in 0..<t.len: c.hashType(t[i])
+    md5Update(c, cast[cstring](addr(t.callConv)), 1)
+
+    if tfNoSideEffect in t.flags: c &= ".noSideEffect"
+    if tfThread in t.flags: c &= ".thread"
+  else:
+    for i in 0..<t.len: c.hashType(t[i])
+  if tfNotNil in t.flags: c &= "not nil"
+
+proc canonConst(n: PNode): TUid =
+  var c: MD5Context
+  md5Init(c)
+  c.hashTree(n)
+  c.hashType(n.typ)
+  md5Final(c, MD5Digest(result))
+
+proc canonSym(s: PSym): TUid =
+  var c: MD5Context
+  md5Init(c)
+  c.hashSym(s)
+  md5Final(c, MD5Digest(result))
+
+proc pushType(w: PRodWriter, t: PType) =
+  # check so that the stack does not grow too large:
+  if iiTableGet(w.index.tab, t.id) == InvalidKey:
+    w.tstack.add(t)
+
+proc pushSym(w: PRodWriter, s: PSym) =
+  # check so that the stack does not grow too large:
+  if iiTableGet(w.index.tab, s.id) == InvalidKey:
+    w.sstack.add(s)
+
+proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
+                result: var string) =
+  if n == nil:
+    # nil nodes have to be stored too:
+    result.add("()")
+    return
+  result.add('(')
+  encodeVInt(ord(n.kind), result)
+  # we do not write comments for now
+  # Line information takes easily 20% or more of the filesize! Therefore we
+  # omit line information if it is the same as the father's line information:
+  if fInfo.fileIndex != n.info.fileIndex:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+    result.add(',')
+    encodeVInt(n.info.line, result)
+    result.add(',')
+    encodeVInt(fileIdx(w, toFilename(n.info)), result)
+  elif fInfo.line != n.info.line:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+    result.add(',')
+    encodeVInt(n.info.line, result)
+  elif fInfo.col != n.info.col:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+  var f = n.flags * PersistentNodeFlags
+  if f != {}:
+    result.add('$')
+    encodeVInt(cast[int32](f), result)
+  if n.typ != nil:
+    result.add('^')
+    encodeVInt(n.typ.id, result)
+    pushType(w, n.typ)
+  case n.kind
+  of nkCharLit..nkInt64Lit:
+    if n.intVal != 0:
+      result.add('!')
+      encodeVBiggestInt(n.intVal, result)
+  of nkFloatLit..nkFloat64Lit:
+    if n.floatVal != 0.0:
+      result.add('!')
+      encodeStr($n.floatVal, result)
+  of nkStrLit..nkTripleStrLit:
+    if n.strVal != "":
+      result.add('!')
+      encodeStr(n.strVal, result)
+  of nkIdent:
+    result.add('!')
+    encodeStr(n.ident.s, result)
+  of nkSym:
+    result.add('!')
+    encodeVInt(n.sym.id, result)
+    pushSym(w, n.sym)
+  else:
+    for i in 0..<n.len:
+      encodeNode(w, n.info, n[i], result)
+  result.add(')')
+
+proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
+  var oldLen = result.len
+  result.add('<')
+  if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
+  if loc.s != low(loc.s):
+    result.add('*')
+    encodeVInt(ord(loc.s), result)
+  if loc.flags != {}:
+    result.add('$')
+    encodeVInt(cast[int32](loc.flags), result)
+  if loc.t != nil:
+    result.add('^')
+    encodeVInt(cast[int32](loc.t.id), result)
+    pushType(w, loc.t)
+  if loc.r != nil:
+    result.add('!')
+    encodeStr($loc.r, result)
+  if loc.a != 0:
+    result.add('?')
+    encodeVInt(loc.a, result)
+  if oldLen + 1 == result.len:
+    # no data was necessary, so remove the '<' again:
+    setLen(result, oldLen)
+  else:
+    result.add('>')
+
+proc encodeType(w: PRodWriter, t: PType, result: var string) =
+  if t == nil:
+    # nil nodes have to be stored too:
+    result.add("[]")
+    return
+  # we need no surrounding [] here because the type is in a line of its own
+  if t.kind == tyForward: internalError("encodeType: tyForward")
+  # for the new rodfile viewer we use a preceding [ so that the data section
+  # can easily be disambiguated:
+  result.add('[')
+  encodeVInt(ord(t.kind), result)
+  result.add('+')
+  encodeVInt(t.id, result)
+  if t.n != nil:
+    encodeNode(w, unknownLineInfo, t.n, result)
+  if t.flags != {}:
+    result.add('$')
+    encodeVInt(cast[int32](t.flags), result)
+  if t.callConv != low(t.callConv):
+    result.add('?')
+    encodeVInt(ord(t.callConv), result)
+  if t.owner != nil:
+    result.add('*')
+    encodeVInt(t.owner.id, result)
+    pushSym(w, t.owner)
+  if t.sym != nil:
+    result.add('&')
+    encodeVInt(t.sym.id, result)
+    pushSym(w, t.sym)
+  if t.size != - 1:
+    result.add('/')
+    encodeVBiggestInt(t.size, result)
+  if t.align != - 1:
+    result.add('=')
+    encodeVInt(t.align, result)
+  encodeLoc(w, t.loc, result)
+  for i in 0..<t.len:
+    if t[i] == nil:
+      result.add("^()")
+    else:
+      result.add('^')
+      encodeVInt(t[i].id, result)
+      pushType(w, t[i])
+
+proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) =
+  result.add('|')
+  encodeVInt(ord(lib.kind), result)
+  result.add('|')
+  encodeStr($lib.name, result)
+  result.add('|')
+  encodeNode(w, info, lib.path, result)
+
+proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
+  if s == nil:
+    # nil nodes have to be stored too:
+    result.add("{}")
+    return
+  # we need no surrounding {} here because the symbol is in a line of its own
+  encodeVInt(ord(s.kind), result)
+  result.add('+')
+  encodeVInt(s.id, result)
+  result.add('&')
+  encodeStr(s.name.s, result)
+  if s.typ != nil:
+    result.add('^')
+    encodeVInt(s.typ.id, result)
+    pushType(w, s.typ)
+  result.add('?')
+  if s.info.col != -1'i16: encodeVInt(s.info.col, result)
+  result.add(',')
+  if s.info.line != -1'i16: encodeVInt(s.info.line, result)
+  result.add(',')
+  encodeVInt(fileIdx(w, toFilename(s.info)), result)
+  if s.owner != nil:
+    result.add('*')
+    encodeVInt(s.owner.id, result)
+    pushSym(w, s.owner)
+  if s.flags != {}:
+    result.add('$')
+    encodeVInt(cast[int32](s.flags), result)
+  if s.magic != mNone:
+    result.add('@')
+    encodeVInt(ord(s.magic), result)
+  if s.options != w.options:
+    result.add('!')
+    encodeVInt(cast[int32](s.options), result)
+  if s.position != 0:
+    result.add('%')
+    encodeVInt(s.position, result)
+  if s.offset != - 1:
+    result.add('`')
+    encodeVInt(s.offset, result)
+  encodeLoc(w, s.loc, result)
+  if s.annex != nil: encodeLib(w, s.annex, s.info, result)
+  if s.constraint != nil:
+    result.add('#')
+    encodeNode(w, unknownLineInfo, s.constraint, result)
+  # lazy loading will soon reload the ast lazily, so the ast needs to be
+  # the last entry of a symbol:
+  if s.ast != nil:
+    # we used to attempt to save space here by only storing a dummy AST if
+    # it is not necessary, but Nim's heavy compile-time evaluation features
+    # make that unfeasible nowadays:
+    encodeNode(w, s.info, s.ast, result)
+
+
+proc createDb() =
+  db.exec(sql"""
+    create table if not exists Module(
+      id integer primary key,
+      name varchar(256) not null,
+      fullpath varchar(256) not null,
+      interfHash varchar(256) not null,
+      fullHash varchar(256) not null,
+
+      created timestamp not null default (DATETIME('now'))
+    );""")
+
+  db.exec(sql"""
+    create table if not exists Backend(
+      id integer primary key,
+      strongdeps varchar(max) not null,
+      weakdeps varchar(max) not null,
+      header varchar(max) not null,
+      code varchar(max) not null
+    )
+
+    create table if not exists Symbol(
+      id integer primary key,
+      module integer not null,
+      backend integer not null,
+      name varchar(max) not null,
+      data varchar(max) not null,
+      created timestamp not null default (DATETIME('now')),
+
+      foreign key (module) references Module(id),
+      foreign key (backend) references Backend(id)
+    );""")
+
+  db.exec(sql"""
+    create table if not exists Type(
+      id integer primary key,
+      module integer not null,
+      name varchar(max) not null,
+      data varchar(max) not null,
+      created timestamp not null default (DATETIME('now')),
+
+      foreign key (module) references module(id)
+    );""")
+
+

compiler/ccgcalls.nim

@@ -21,7 +21,7 @@ proc canRaiseDisp(p: BProc; n: PNode): bool =
     # we have to be *very* conservative:
     result = canRaiseConservative(n)
 
-proc preventNrvo(p: BProc; dest, le, ri: PNode): bool =
+proc preventNrvo(p: BProc; le, ri: PNode): bool =
   proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
     var n = le
     while true:
@@ -54,11 +54,6 @@ proc preventNrvo(p: BProc; dest, le, ri: PNode): bool =
     if canRaise(ri[0]) and
         locationEscapes(p, le, p.nestedTryStmts.len > 0):
       message(p.config, le.info, warnObservableStores, $le)
-  # bug #19613 prevent dangerous aliasing too:
-  if dest != nil and dest != le:
-    for i in 1..<ri.len:
-      let r = ri[i]
-      if isPartOf(dest, r) != arNo: return true
 
 proc hasNoInit(call: PNode): bool {.inline.} =
   result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
@@ -81,10 +76,10 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
   # getUniqueType() is too expensive here:
   var typ = skipTypes(ri[0].typ, abstractInst)
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ):
+    if isInvalidReturnType(p.config, typ[0]):
       if params != nil: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not preventNrvo(p, d.lode, le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
@@ -155,7 +150,7 @@ proc reifiedOpenArray(n: PNode): bool {.inline.} =
   else:
     result = true
 
-proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareForMutation = false): (Rope, Rope) =
+proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope, Rope) =
   var a, b, c: TLoc
   initLocExpr(p, q[1], a)
   initLocExpr(p, q[2], b)
@@ -163,8 +158,6 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareF
   # but first produce the required index checks:
   if optBoundsCheck in p.options:
     genBoundsCheck(p, a, b, c)
-  if prepareForMutation:
-    linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
   let ty = skipTypes(a.t, abstractVar+{tyPtr})
   let dest = getTypeDesc(p.module, destType)
   let lengthExpr = "($1)-($2)+1" % [rdLoc(c), rdLoc(b)]
@@ -185,7 +178,7 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareF
     else:
       result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest],
                 lengthExpr)
-  of tyUncheckedArray, tyCstring:
+  of tyUncheckedArray, tyCString:
     result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest],
               lengthExpr)
   of tyString, tySequence:
@@ -194,12 +187,10 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareF
         optSeqDestructors in p.config.globalOptions:
       linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
     if atyp.kind in {tyVar} and not compileToCpp(p.module):
-      result = ("(($5) ? (($4*)(*$1)$3+($2)) : NIM_NIL)" %
-                  [rdLoc(a), rdLoc(b), dataField(p), dest, dataFieldAccessor(p, "*" & rdLoc(a))],
+      result = ("($4*)(*$1)$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
                 lengthExpr)
     else:
-      result = ("(($5) ? (($4*)$1$3+($2)) : NIM_NIL)" %
-                  [rdLoc(a), rdLoc(b), dataField(p), dest, dataFieldAccessor(p, rdLoc(a))],
+      result = ("($4*)$1$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
                 lengthExpr)
   else:
     internalError(p.config, "openArrayLoc: " & typeToString(a.t))
@@ -240,12 +231,9 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
       if ntyp.kind in {tyVar} and not compileToCpp(p.module):
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
-        result = "($4) ? ((*$1)$3) : NIM_NIL, $2" %
-                     [a.rdLoc, lenExpr(p, t), dataField(p),
-                      dataFieldAccessor(p, "*" & a.rdLoc)]
+        result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
       else:
-        result = "($4) ? ($1$3) : NIM_NIL, $2" %
-                     [a.rdLoc, lenExpr(p, a), dataField(p), dataFieldAccessor(p, a.rdLoc)]
+        result = "$1$3, $2" % [a.rdLoc, lenExpr(p, a), dataField(p)]
     of tyArray:
       result = "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, a.t))]
     of tyPtr, tyRef:
@@ -253,9 +241,7 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
       of tyString, tySequence:
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
-        result = "($4) ? ((*$1)$3) : NIM_NIL, $2" %
-                     [a.rdLoc, lenExpr(p, t), dataField(p),
-                      dataFieldAccessor(p, "*" & a.rdLoc)]
+        result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
       of tyArray:
         result = "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, lastSon(a.t)))]
       else:
@@ -273,10 +259,6 @@ proc withTmpIfNeeded(p: BProc, a: TLoc, needsTmp: bool): TLoc =
   else:
     result = a
 
-proc literalsNeedsTmp(p: BProc, a: TLoc): TLoc =
-  getTemp(p, a.lode.typ, result, needsInit=false)
-  genAssignment(p, result, a, {})
-
 proc genArgStringToCString(p: BProc, n: PNode, needsTmp: bool): Rope {.inline.} =
   var a: TLoc
   initLocExpr(p, n[0], a)
@@ -291,10 +273,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode, needsTmp = false): Rop
     result = openArrayLoc(p, param.typ, n)
   elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
-    if n.kind in {nkCharLit..nkNilLit}:
-      result = addrLoc(p.config, literalsNeedsTmp(p, a))
-    else:
-      result = addrLoc(p.config, withTmpIfNeeded(p, a, needsTmp))
+    result = addrLoc(p.config, withTmpIfNeeded(p, a, needsTmp))
   elif p.module.compileToCpp and param.typ.kind in {tyVar} and
       n.kind == nkHiddenAddr:
     initLocExprSingleUse(p, n[0], a)
@@ -310,7 +289,6 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode, needsTmp = false): Rop
   else:
     initLocExprSingleUse(p, n, a)
     result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
-  #assert result != nil
 
 proc genArgNoParam(p: BProc, n: PNode, needsTmp = false): Rope =
   var a: TLoc
@@ -320,59 +298,47 @@ proc genArgNoParam(p: BProc, n: PNode, needsTmp = false): Rope =
     initLocExprSingleUse(p, n, a)
     result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
 
-from dfa import aliases, AliasKind
+from dfa import instrTargets, InstrTargetKind
 
 proc potentialAlias(n: PNode, potentialWrites: seq[PNode]): bool =
   for p in potentialWrites:
-    if p.aliases(n) != no or n.aliases(p) != no:
+    if instrTargets(p, n) != None:
       return true
 
 proc skipTrivialIndirections(n: PNode): PNode =
   result = n
   while true:
     case result.kind
-    of nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv:
+    of {nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv}:
       result = result[0]
-    of nkHiddenStdConv, nkHiddenSubConv:
+    of {nkHiddenStdConv, nkHiddenSubConv}:
       result = result[1]
     else: break
 
-proc getPotentialWrites(n: PNode; mutate: bool; result: var seq[PNode]) =
+proc getPotentialWrites(n: PNode, mutate = false): seq[PNode] =
   case n.kind:
-  of nkLiterals, nkIdent, nkFormalParams: discard
+  of nkLiterals, nkIdent: discard
   of nkSym:
     if mutate: result.add n
   of nkAsgn, nkFastAsgn:
-    getPotentialWrites(n[0], true, result)
-    getPotentialWrites(n[1], mutate, result)
+    result.add getPotentialWrites(n[0], true)
+    result.add getPotentialWrites(n[1], mutate)
   of nkAddr, nkHiddenAddr:
-    getPotentialWrites(n[0], true, result)
-  of nkBracketExpr, nkDotExpr, nkCheckedFieldExpr:
-    getPotentialWrites(n[0], mutate, result)
-  of nkCallKinds:
-    case n.getMagic:
-    of mIncl, mExcl, mInc, mDec, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
-        mAddr, mNew, mNewFinalize, mWasMoved, mDestroy, mReset:
-      getPotentialWrites(n[1], true, result)
-      for i in 2..<n.len:
-        getPotentialWrites(n[i], mutate, result)
-    of mSwap:
-      for i in 1..<n.len:
-        getPotentialWrites(n[i], true, result)
-    else:
-      for i in 1..<n.len:
-        getPotentialWrites(n[i], mutate, result)
+    result.add getPotentialWrites(n[0], true)
+  of nkCallKinds: #TODO: Find out why in f += 1, f is a nkSym and not a nkHiddenAddr
+    for s in n.sons:
+      result.add getPotentialWrites(s, true)
   else:
-    for s in n:
-      getPotentialWrites(s, mutate, result)
+    for s in n.sons:
+      result.add getPotentialWrites(s, mutate)
 
-proc getPotentialReads(n: PNode; result: var seq[PNode]) =
+proc getPotentialReads(n: PNode): seq[PNode] =
   case n.kind:
-  of nkLiterals, nkIdent, nkFormalParams: discard
+  of nkLiterals, nkIdent: discard
   of nkSym: result.add n
   else:
-    for s in n:
-      getPotentialReads(s, result)
+    for s in n.sons:
+      result.add getPotentialReads(s)
 
 proc genParams(p: BProc, ri: PNode, typ: PType): Rope =
   # We must generate temporaries in cases like #14396
@@ -383,15 +349,12 @@ proc genParams(p: BProc, ri: PNode, typ: PType): Rope =
     if ri[i].skipTrivialIndirections.kind == nkSym:
       needTmp[i - 1] = potentialAlias(ri[i], potentialWrites)
     else:
-      #if not ri[i].typ.isCompileTimeOnly:
-      var potentialReads: seq[PNode]
-      getPotentialReads(ri[i], potentialReads)
-      for n in potentialReads:
+      for n in getPotentialReads(ri[i]):
         if not needTmp[i - 1]:
           needTmp[i - 1] = potentialAlias(n, potentialWrites)
-      getPotentialWrites(ri[i], false, potentialWrites)
-    if ri[i].kind in {nkHiddenAddr, nkAddr}:
-      # Optimization: don't use a temp, if we would only take the address anyway
+      potentialWrites.add getPotentialWrites(ri[i])
+    if ri[i].kind == nkHiddenAddr:
+      # Optimization: don't use a temp, if we would only take the adress anyway
       needTmp[i - 1] = false
 
   for i in 1..<ri.len:
@@ -453,10 +416,10 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
   let rawProc = getClosureType(p.module, typ, clHalf)
   let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ):
+    if isInvalidReturnType(p.config, typ[0]):
       if ri.len > 1: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not preventNrvo(p, d.lode, le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
           getTemp(p, typ[0], d, needsInit=true)
@@ -751,7 +714,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
     pl.add(~": ")
     pl.add(genArg(p, ri[i], param, ri))
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ):
+    if isInvalidReturnType(p.config, typ[0]):
       if ri.len > 1: pl.add(~" ")
       # beware of 'result = p(result)'. We always allocate a temporary:
       if d.k in {locTemp, locNone}:
@@ -815,5 +778,6 @@ proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
     genNamedParamCall(p, ri, d)
   else:
     genPrefixCall(p, le, ri, d)
+  postStmtActions(p)
 
 proc genCall(p: BProc, e: PNode, d: var TLoc) = genAsgnCall(p, nil, e, d)

compiler/ccgexprs.nim

@@ -9,17 +9,12 @@
 
 # included from cgen.nim
 
-when defined(nimCompilerStacktraceHints):
-  import std/stackframes
-
 proc getNullValueAuxT(p: BProc; orig, t: PType; obj, constOrNil: PNode,
                       result: var Rope; count: var int;
                       isConst: bool, info: TLineInfo)
 
 # -------------------------- constant expressions ------------------------
 
-proc rdSetElemLoc(conf: ConfigRef; a: TLoc, typ: PType): Rope
-
 proc int64Literal(i: BiggestInt): Rope =
   if i > low(int64):
     result = "IL64($1)" % [rope(i)]
@@ -87,21 +82,19 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope =
     of tyNil:
       result = genNilStringLiteral(p.module, n.info)
     of tyString:
-      # with the new semantics for not 'nil' strings, we can map "" to nil and
+      # with the new semantics for 'nil' strings, we can map "" to nil and
       # save tons of allocations:
-      if n.strVal.len == 0 and optSeqDestructors notin p.config.globalOptions:
+      if n.strVal.len == 0 and optNilSeqs notin p.options and
+          optSeqDestructors notin p.config.globalOptions:
         result = genNilStringLiteral(p.module, n.info)
       else:
         result = genStringLiteral(p.module, n)
     else:
       result = makeCString(n.strVal)
   of nkFloatLit, nkFloat64Lit:
-    if ty.kind == tyFloat32:
-      result = rope(n.floatVal.float32.toStrMaxPrecision)
-    else:
-      result = rope(n.floatVal.toStrMaxPrecision)
+    result = rope(n.floatVal.toStrMaxPrecision)
   of nkFloat32Lit:
-    result = rope(n.floatVal.float32.toStrMaxPrecision)
+    result = rope(n.floatVal.toStrMaxPrecision("f"))
   else:
     internalError(p.config, n.info, "genLiteral(" & $n.kind & ')')
     result = nil
@@ -180,7 +173,8 @@ proc canMove(p: BProc, n: PNode; dest: TLoc): bool =
     if not isDeepConstExpr(n) or n.len == 0:
       if skipTypes(n.typ, abstractVarRange).kind == tySequence:
         return true
-  elif n.kind in nkStrKinds and n.strVal.len == 0:
+  elif optNilSeqs notin p.options and
+    n.kind in nkStrKinds and n.strVal.len == 0:
     # Empty strings are codegen'd as NIM_NIL so it's just a pointer copy
     return true
   result = n.kind in nkCallKinds
@@ -292,8 +286,8 @@ proc genOpenArrayConv(p: BProc; d: TLoc; a: TLoc) =
       linefmt(p, cpsStmts, "$1.Field0 = $2; $1.Field1 = $2Len_0;$n",
         [rdLoc(d), a.rdLoc])
   of tySequence:
-    linefmt(p, cpsStmts, "$1.Field0 = ($5) ? ($2$3) : NIM_NIL; $1.Field1 = $4;$n",
-      [rdLoc(d), a.rdLoc, dataField(p), lenExpr(p, a), dataFieldAccessor(p, a.rdLoc)])
+    linefmt(p, cpsStmts, "$1.Field0 = $2$3; $1.Field1 = $4;$n",
+      [rdLoc(d), a.rdLoc, dataField(p), lenExpr(p, a)])
   of tyArray:
     linefmt(p, cpsStmts, "$1.Field0 = $2; $1.Field1 = $3;$n",
       [rdLoc(d), rdLoc(a), rope(lengthOrd(p.config, a.t))])
@@ -302,8 +296,8 @@ proc genOpenArrayConv(p: BProc; d: TLoc; a: TLoc) =
     if etyp.kind in {tyVar} and optSeqDestructors in p.config.globalOptions:
       linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
 
-    linefmt(p, cpsStmts, "$1.Field0 = ($5) ? ($2$3) : NIM_NIL; $1.Field1 = $4;$n",
-      [rdLoc(d), a.rdLoc, dataField(p), lenExpr(p, a), dataFieldAccessor(p, a.rdLoc)])
+    linefmt(p, cpsStmts, "$1.Field0 = $2$3; $1.Field1 = $4;$n",
+      [rdLoc(d), a.rdLoc, dataField(p), lenExpr(p, a)])
   else:
     internalError(p.config, a.lode.info, "cannot handle " & $a.t.kind)
 
@@ -404,7 +398,7 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
               [rdLoc(dest), rdLoc(src), getSize(p.config, dest.t)])
     else:
       linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)])
-  of tyPtr, tyPointer, tyChar, tyBool, tyEnum, tyCstring,
+  of tyPtr, tyPointer, tyChar, tyBool, tyEnum, tyCString,
      tyInt..tyUInt64, tyRange, tyVar, tyLent, tyNil:
     linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)])
   else: internalError(p.config, "genAssignment: " & $ty.kind)
@@ -444,10 +438,9 @@ proc genDeepCopy(p: BProc; dest, src: TLoc) =
               [addrLoc(p.config, dest), rdLoc(src),
               genTypeInfoV1(p.module, dest.t, dest.lode.info)])
   of tyOpenArray, tyVarargs:
-    let source = addrLocOrTemp(src)
     linefmt(p, cpsStmts,
-         "#genericDeepCopyOpenArray((void*)$1, (void*)$2, $2->Field1, $3);$n",
-         [addrLoc(p.config, dest), source,
+         "#genericDeepCopyOpenArray((void*)$1, (void*)$2, $1Len_0, $3);$n",
+         [addrLoc(p.config, dest), addrLocOrTemp(src),
          genTypeInfoV1(p.module, dest.t, dest.lode.info)])
   of tySet:
     if mapSetType(p.config, ty) == ctArray:
@@ -455,7 +448,7 @@ proc genDeepCopy(p: BProc; dest, src: TLoc) =
               [rdLoc(dest), rdLoc(src), getSize(p.config, dest.t)])
     else:
       linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)])
-  of tyPointer, tyChar, tyBool, tyEnum, tyCstring,
+  of tyPointer, tyChar, tyBool, tyEnum, tyCString,
      tyInt..tyUInt64, tyRange, tyVar, tyLent:
     linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)])
   else: internalError(p.config, "genDeepCopy: " & $ty.kind)
@@ -585,23 +578,13 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
   else:
     # we handle div by zero here so that we know that the compilerproc's
     # result is only for overflows.
-    var needsOverflowCheck = true
     if m in {mDivI, mModI}:
-      var canBeZero = true
-      if e[2].kind in {nkIntLit..nkUInt64Lit}:
-        canBeZero = e[2].intVal == 0
-      if e[2].kind in {nkIntLit..nkInt64Lit}:
-        needsOverflowCheck = e[2].intVal == -1
-      if canBeZero:
-        linefmt(p, cpsStmts, "if ($1 == 0){ #raiseDivByZero(); $2}$n",
-                [rdLoc(b), raiseInstr(p)])
-    if needsOverflowCheck:
-      let res = binaryArithOverflowRaw(p, t, a, b,
-        if t.kind == tyInt64: prc64[m] else: prc[m])
-      putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res])
-    else:
-      let res = "($1)(($2) $3 ($4))" % [getTypeDesc(p.module, e.typ), rdLoc(a), rope(opr[m]), rdLoc(b)]
-      putIntoDest(p, d, e, res)
+      linefmt(p, cpsStmts, "if ($1 == 0){ #raiseDivByZero(); $2}$n",
+              [rdLoc(b), raiseInstr(p)])
+
+    let res = binaryArithOverflowRaw(p, t, a, b,
+      if t.kind == tyInt64: prc64[m] else: prc[m])
+    putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res])
 
 proc unaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
   var
@@ -732,22 +715,7 @@ proc isCppRef(p: BProc; typ: PType): bool {.inline.} =
       skipTypes(typ, abstractInstOwned).kind in {tyVar} and
       tfVarIsPtr notin skipTypes(typ, abstractInstOwned).flags
 
-proc derefBlock(p: BProc, e: PNode, d: var TLoc) =
-  # We transform (block: x)[] to (block: x[])
-  let e0 = e[0]
-  var n = shallowCopy(e0)
-  n.typ = e.typ
-  for i in 0 ..< e0.len - 1:
-    n[i] = e0[i]
-  n[e0.len-1] = newTreeIT(nkHiddenDeref, e.info, e.typ, e0[e0.len-1])
-  expr p, n, d
-
 proc genDeref(p: BProc, e: PNode, d: var TLoc) =
-  if e.kind == nkHiddenDeref and e[0].kind in {nkBlockExpr, nkBlockStmt}:
-    # bug #20107. Watch out to not deref the pointer too late.
-    derefBlock(p, e, d)
-    return
-
   let mt = mapType(p.config, e[0].typ, mapTypeChooser(e[0]))
   if mt in {ctArray, ctPtrToArray} and lfEnforceDeref notin d.flags:
     # XXX the amount of hacks for C's arrays is incredible, maybe we should
@@ -901,42 +869,16 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) =
     v.r.add(".")
     v.r.add(disc.sym.loc.r)
     genInExprAux(p, it, u, v, test)
-    var msg = ""
-    if optDeclaredLocs in p.config.globalOptions:
-      # xxx this should be controlled by a separate flag, and
-      # used for other similar defects so that location information is shown
-      # even without the expensive `--stacktrace`; binary size could be optimized
-      # by encoding the file names separately from `file(line:col)`, essentially
-      # passing around `TLineInfo` + the set of files in the project.
-      msg.add toFileLineCol(p.config, e.info) & " "
-    msg.add genFieldDefect(p.config, field.name.s, disc.sym)
+    let msg = genFieldDefect(field, disc.sym)
     let strLit = genStringLiteral(p.module, newStrNode(nkStrLit, msg))
-
-    ## discriminant check
-    template fun(code) = linefmt(p, cpsStmts, code, [rdLoc(test)])
-    if op.magic == mNot: fun("if ($1) ") else: fun("if (!($1)) ")
-
-    ## call raiseFieldError2 on failure
-    let discIndex = rdSetElemLoc(p.config, v, u.t)
-    if optTinyRtti in p.config.globalOptions:
-      # not sure how to use `genEnumToStr` here
-      if p.config.getStdlibVersion < (1,5,1):
-        const code = "{ #raiseFieldError($1); $2} $n"
-        linefmt(p, cpsStmts, code, [strLit, raiseInstr(p)])
-      else:
-        const code = "{ #raiseFieldError2($1, (NI)$3); $2} $n"
-        linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex])
+    if op.magic == mNot:
+      linefmt(p, cpsStmts,
+              "if ($1){ #raiseFieldError($2); $3}$n",
+              [rdLoc(test), strLit, raiseInstr(p)])
     else:
-      # complication needed for signed types
-      let first = p.config.firstOrd(disc.sym.typ)
-      let firstLit = int64Literal(cast[int](first))
-      let discName = genTypeInfo(p.config, p.module, disc.sym.typ, e.info)
-      if p.config.getStdlibVersion < (1,5,1):
-        const code = "{ #raiseFieldError($1); $2} $n"
-        linefmt(p, cpsStmts, code, [strLit, raiseInstr(p)])
-      else:
-        const code = "{ #raiseFieldError2($1, #reprDiscriminant(((NI)$3) + (NI)$4, $5)); $2} $n"
-        linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex, firstLit, discName])
+      linefmt(p, cpsStmts,
+              "if (!($1)){ #raiseFieldError($2); $3}$n",
+              [rdLoc(test), strLit, raiseInstr(p)])
 
 proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) =
   assert e[0].kind == nkDotExpr
@@ -1004,12 +946,12 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) =
     if reifiedOpenArray(arr.lode):
       linefmt(p, cpsStmts,
         "if ($2-$1 != -1 && " &
-        "($1 < 0 || $1 >= $3.Field1 || $2 < 0 || $2 >= $3.Field1)){ #raiseIndexError4($1, $2, $3.Field1); $4}$n",
+        "((NU)($1) >= (NU)($3.Field1) || (NU)($2) >= (NU)($3.Field1))){ #raiseIndexError(); $4}$n",
         [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)])
     else:
       linefmt(p, cpsStmts,
-        "if ($2-$1 != -1 && ($1 < 0 || $1 >= $3Len_0 || $2 < 0 || $2 >= $3Len_0))" &
-        "{ #raiseIndexError4($1, $2, $3Len_0); $4}$n",
+        "if ($2-$1 != -1 && " &
+        "((NU)($1) >= (NU)($3Len_0) || (NU)($2) >= (NU)($3Len_0))){ #raiseIndexError(); $4}$n",
         [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)])
   of tyArray:
     let first = intLiteral(firstOrd(p.config, ty))
@@ -1020,7 +962,7 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) =
   of tySequence, tyString:
     linefmt(p, cpsStmts,
       "if ($2-$1 != -1 && " &
-      "($1 < 0 || $1 >= $3 || $2 < 0 || $2 >= $3)){ #raiseIndexError4($1, $2, $3); $4}$n",
+      "((NU)($1) >= (NU)$3 || (NU)($2) >= (NU)$3)){ #raiseIndexError(); $4}$n",
       [rdLoc(a), rdLoc(b), lenExpr(p, arr), raiseInstr(p)])
   else: discard
 
@@ -1031,15 +973,15 @@ proc genOpenArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   if not reifiedOpenArray(x):
     # emit range check:
     if optBoundsCheck in p.options:
-      linefmt(p, cpsStmts, "if ($1 < 0 || $1 >= $2Len_0){ #raiseIndexError2($1,$2Len_0-1); $3}$n",
-              [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``!
+      linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)){ #raiseIndexError2($1,$2Len_0-1); $3}$n",
+              [rdLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``!
     inheritLocation(d, a)
     putIntoDest(p, d, n,
                 ropecg(p.module, "$1[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage)
   else:
     if optBoundsCheck in p.options:
-      linefmt(p, cpsStmts, "if ($1 < 0 || $1 >= $2.Field1){ #raiseIndexError2($1,$2.Field1-1); $3}$n",
-              [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``!
+      linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2.Field1)){ #raiseIndexError2($1,$2.Field1-1); $3}$n",
+              [rdLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``!
     inheritLocation(d, a)
     putIntoDest(p, d, n,
                 ropecg(p.module, "$1.Field0[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage)
@@ -1052,9 +994,14 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   if ty.kind in {tyRef, tyPtr}:
     ty = skipTypes(ty.lastSon, abstractVarRange) # emit range check:
   if optBoundsCheck in p.options:
-    linefmt(p, cpsStmts,
-            "if ($1 < 0 || $1 >= $2){ #raiseIndexError2($1,$2-1); $3}$n",
-            [rdCharLoc(b), lenExpr(p, a), raiseInstr(p)])
+    if ty.kind == tyString and not defined(nimNoZeroTerminator):
+      linefmt(p, cpsStmts,
+              "if ((NU)($1) > (NU)$2){ #raiseIndexError2($1,$2); $3}$n",
+              [rdLoc(b), lenExpr(p, a), raiseInstr(p)])
+    else:
+      linefmt(p, cpsStmts,
+              "if ((NU)($1) >= (NU)$2){ #raiseIndexError2($1,$2-1); $3}$n",
+              [rdLoc(b), lenExpr(p, a), raiseInstr(p)])
   if d.k == locNone: d.storage = OnHeap
   if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}:
     a.r = ropecg(p.module, "(*$1)", [a.r])
@@ -1073,7 +1020,7 @@ proc genBracketExpr(p: BProc; n: PNode; d: var TLoc) =
   of tyArray: genArrayElem(p, n, n[0], n[1], d)
   of tyOpenArray, tyVarargs: genOpenArrayElem(p, n, n[0], n[1], d)
   of tySequence, tyString: genSeqElem(p, n, n[0], n[1], d)
-  of tyCstring: genCStringElem(p, n, n[0], n[1], d)
+  of tyCString: genCStringElem(p, n, n[0], n[1], d)
   of tyTuple: genTupleElem(p, n, d)
   else: internalError(p.config, n.info, "expr(nkBracketExpr, " & $ty.kind & ')')
   discard getTypeDesc(p.module, n.typ)
@@ -1160,21 +1107,15 @@ proc genEcho(p: BProc, n: PNode) =
     # echo directly to the Genode LOG session
     var args: Rope = nil
     var a: TLoc
-    for i, it in n.sons:
+    for it in n.sons:
       if it.skipConv.kind == nkNilLit:
         args.add(", \"\"")
-      elif n.len != 0:
+      else:
         initLocExpr(p, it, a)
-        if i > 0:
-          args.add(", ")
-        case detectStrVersion(p.module)
-        of 2:
-          args.add(ropecg(p.module, "Genode::Cstring($1.p->data, $1.len)", [a.rdLoc]))
-        else:
-          args.add(ropecg(p.module, "Genode::Cstring($1->data, $1->len)", [a.rdLoc]))
+        args.add(ropecg(p.module, ", Genode::Cstring($1->data, $1->len)", [rdLoc(a)]))
     p.module.includeHeader("<base/log.h>")
     p.module.includeHeader("<util/string.h>")
-    linefmt(p, cpsStmts, """Genode::log($1);$n""", [args])
+    linefmt(p, cpsStmts, """Genode::log(""$1);$n""", [args])
   else:
     if n.len == 0:
       linefmt(p, cpsStmts, "#echoBinSafe(NIM_NIL, $1);$n", [n.len])
@@ -1348,21 +1289,20 @@ proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) =
     genAssignment(p, a, b, {})
   else:
     let ti = genTypeInfoV1(p.module, typ, a.lode.info)
-    let op = getAttachedOp(p.module.g.graph, bt, attachedDestructor)
-    if op != nil and not isTrivialProc(p.module.g.graph, op):
+    if bt.destructor != nil and not isTrivialProc(bt.destructor):
       # the prototype of a destructor is ``=destroy(x: var T)`` and that of a
       # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling
       # convention at least:
-      if op.typ == nil or op.typ.callConv != ccNimCall:
+      if bt.destructor.typ == nil or bt.destructor.typ.callConv != ccNimCall:
         localError(p.module.config, a.lode.info,
           "the destructor that is turned into a finalizer needs " &
           "to have the 'nimcall' calling convention")
       var f: TLoc
-      initLocExpr(p, newSymNode(op), f)
+      initLocExpr(p, newSymNode(bt.destructor), f)
       p.module.s[cfsTypeInit3].addf("$1->finalizer = (void*)$2;$n", [ti, rdLoc(f)])
 
     if a.storage == OnHeap and usesWriteBarrier(p.config):
-      if canFormAcycle(p.module.g.graph, a.t):
+      if canFormAcycle(a.t):
         linefmt(p, cpsStmts, "if ($1) { #nimGCunrefRC1($1); $1 = NIM_NIL; }$n", [a.rdLoc])
       else:
         linefmt(p, cpsStmts, "if ($1) { #nimGCunrefNoCycle($1); $1 = NIM_NIL; }$n", [a.rdLoc])
@@ -1399,7 +1339,7 @@ proc genNewSeqAux(p: BProc, dest: TLoc, length: Rope; lenIsZero: bool) =
   var call: TLoc
   initLoc(call, locExpr, dest.lode, OnHeap)
   if dest.storage == OnHeap and usesWriteBarrier(p.config):
-    if canFormAcycle(p.module.g.graph, dest.t):
+    if canFormAcycle(dest.t):
       linefmt(p, cpsStmts, "if ($1) { #nimGCunrefRC1($1); $1 = NIM_NIL; }$n", [dest.rdLoc])
     else:
       linefmt(p, cpsStmts, "if ($1) { #nimGCunrefNoCycle($1); $1 = NIM_NIL; }$n", [dest.rdLoc])
@@ -1432,7 +1372,8 @@ proc genNewSeq(p: BProc, e: PNode) =
        getTypeDesc(p.module, seqtype.lastSon),
        getSeqPayloadType(p.module, seqtype)])
   else:
-    let lenIsZero = e[2].kind == nkIntLit and e[2].intVal == 0
+    let lenIsZero = optNilSeqs notin p.options and
+      e[2].kind == nkIntLit and e[2].intVal == 0
     genNewSeqAux(p, a, b.rdLoc, lenIsZero)
     gcUsage(p.config, e)
 
@@ -1447,7 +1388,6 @@ proc genNewSeqOfCap(p: BProc; e: PNode; d: var TLoc) =
       getSeqPayloadType(p.module, seqtype),
     ])
   else:
-    if d.k == locNone: getTemp(p, e.typ, d, needsInit=false) # bug #22560
     putIntoDest(p, d, e, ropecg(p.module,
                 "($1)#nimNewSeqOfCap($2, $3)", [
                 getTypeDesc(p.module, seqtype),
@@ -1558,7 +1498,8 @@ proc genSeqConstr(p: BProc, n: PNode, d: var TLoc) =
       getSeqPayloadType(p.module, seqtype)])
   else:
     # generate call to newSeq before adding the elements per hand:
-    genNewSeqAux(p, dest[], l, n.len == 0)
+    genNewSeqAux(p, dest[], l,
+      optNilSeqs notin p.options and n.len == 0)
   for i in 0..<n.len:
     initLoc(arr, locExpr, n[i], OnHeap)
     arr.r = ropecg(p.module, "$1$3[$2]", [rdLoc(dest[]), intLiteral(i), dataField(p)])
@@ -1579,7 +1520,6 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) =
     return
   if d.k == locNone:
     getTemp(p, n.typ, d)
-  initLocExpr(p, n[1], a)
   # generate call to newSeq before adding the elements per hand:
   let L = toInt(lengthOrd(p.config, n[1].typ))
   if optSeqDestructors in p.config.globalOptions:
@@ -1588,7 +1528,8 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) =
       [rdLoc d, L, getTypeDesc(p.module, seqtype.lastSon),
       getSeqPayloadType(p.module, seqtype)])
   else:
-    genNewSeqAux(p, d, intLiteral(L), L == 0)
+    genNewSeqAux(p, d, intLiteral(L), optNilSeqs notin p.options and L == 0)
+  initLocExpr(p, n[1], a)
   # bug #5007; do not produce excessive C source code:
   if L < 10:
     for i in 0..<L:
@@ -1620,7 +1561,10 @@ proc genNewFinalize(p: BProc, e: PNode) =
   initLocExpr(p, e[1], a)
   initLocExpr(p, e[2], f)
   initLoc(b, locExpr, a.lode, OnHeap)
-  ti = genTypeInfo(p.config, p.module, refType, e.info)
+  if optTinyRtti in p.config.globalOptions:
+    ti = genTypeInfoV2(p.module, refType, e.info)
+  else:
+    ti = genTypeInfoV1(p.module, refType, e.info)
   p.module.s[cfsTypeInit3].addf("$1->finalizer = (void*)$2;$n", [ti, rdLoc(f)])
   b.r = ropecg(p.module, "($1) #newObj($2, sizeof($3))", [
       getTypeDesc(p.module, refType),
@@ -1632,11 +1576,8 @@ proc genNewFinalize(p: BProc, e: PNode) =
 
 proc genOfHelper(p: BProc; dest: PType; a: Rope; info: TLineInfo): Rope =
   if optTinyRtti in p.config.globalOptions:
-    let ti = genTypeInfo2Name(p.module, dest)
-    inc p.module.labels
-    let cache = "Nim_OfCheck_CACHE" & p.module.labels.rope
-    p.module.s[cfsVars].addf("static TNimTypeV2* $#[2];$n", [cache])
-    result = ropecg(p.module, "#isObjWithCache($#.m_type, $#, $#)", [a, ti, cache])
+    result = ropecg(p.module, "#isObj($1.m_type, $2)",
+      [a, genTypeInfo2Name(p.module, dest)])
   else:
     # unfortunately 'genTypeInfoV1' sets tfObjHasKids as a side effect, so we
     # have to call it here first:
@@ -1716,9 +1657,7 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) =
       putIntoDest(p, b, e, "$1, $1Len_0" % [rdLoc(a)], a.storage)
     of tyString, tySequence:
       putIntoDest(p, b, e,
-                  "($4) ? ($1$3) : NIM_NIL, $2" %
-                    [rdLoc(a), lenExpr(p, a), dataField(p), dataFieldAccessor(p, a.rdLoc)],
-                  a.storage)
+                  "$1$3, $2" % [rdLoc(a), lenExpr(p, a), dataField(p)], a.storage)
     of tyArray:
       putIntoDest(p, b, e,
                   "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, a.t))], a.storage)
@@ -1726,7 +1665,7 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) =
     putIntoDest(p, d, e,
         ropecg(p.module, "#reprOpenArray($1, $2)", [rdLoc(b),
         genTypeInfoV1(p.module, elemType(t), e.info)]), a.storage)
-  of tyCstring, tyArray, tyRef, tyPtr, tyPointer, tyNil, tySequence:
+  of tyCString, tyArray, tyRef, tyPtr, tyPointer, tyNil, tySequence:
     putIntoDest(p, d, e,
                 ropecg(p.module, "#reprAny($1, $2)", [
                 rdLoc(a), genTypeInfoV1(p.module, t, e.info)]), a.storage)
@@ -1773,13 +1712,6 @@ proc genGetTypeInfoV2(p: BProc, e: PNode, d: var TLoc) =
     # use the dynamic type stored at offset 0:
     putIntoDest(p, d, e, rdMType(p, a, nilCheck))
 
-proc genAccessTypeField(p: BProc; e: PNode; d: var TLoc) =
-  var a: TLoc
-  initLocExpr(p, e[1], a)
-  var nilCheck = Rope(nil)
-  # use the dynamic type stored at offset 0:
-  putIntoDest(p, d, e, rdMType(p, a, nilCheck))
-
 template genDollar(p: BProc, n: PNode, d: var TLoc, frmt: string) =
   var a: TLoc
   initLocExpr(p, n[1], a)
@@ -1798,34 +1730,21 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     # Bug #9279, len(toOpenArray()) has to work:
     if a.kind in nkCallKinds and a[0].kind == nkSym and a[0].sym.magic == mSlice:
       # magic: pass slice to openArray:
-      var m: TLoc
       var b, c: TLoc
-      initLocExpr(p, a[1], m)
       initLocExpr(p, a[2], b)
       initLocExpr(p, a[3], c)
-      if optBoundsCheck in p.options:
-        genBoundsCheck(p, m, b, c)
       if op == mHigh:
-        putIntoDest(p, d, e, ropecg(p.module, "(($2)-($1))", [rdLoc(b), rdLoc(c)]))
+        putIntoDest(p, d, e, ropecg(p.module, "($2)-($1)", [rdLoc(b), rdLoc(c)]))
       else:
-        putIntoDest(p, d, e, ropecg(p.module, "(($2)-($1)+1)", [rdLoc(b), rdLoc(c)]))
+        putIntoDest(p, d, e, ropecg(p.module, "($2)-($1)+1", [rdLoc(b), rdLoc(c)]))
     else:
       if not reifiedOpenArray(a):
         if op == mHigh: unaryExpr(p, e, d, "($1Len_0-1)")
         else: unaryExpr(p, e, d, "$1Len_0")
       else:
-        let isDeref = a.kind in {nkHiddenDeref, nkDerefExpr}
-        if op == mHigh:
-          if isDeref:
-            unaryExpr(p, e, d, "($1->Field1-1)")
-          else:
-            unaryExpr(p, e, d, "($1.Field1-1)")
-        else:
-          if isDeref:
-            unaryExpr(p, e, d, "$1->Field1")
-          else:
-            unaryExpr(p, e, d, "$1.Field1")
-  of tyCstring:
+        if op == mHigh: unaryExpr(p, e, d, "($1.Field1-1)")
+        else: unaryExpr(p, e, d, "$1.Field1")
+  of tyCString:
     if op == mHigh: unaryExpr(p, e, d, "($1 ? (#nimCStrLen($1)-1) : -1)")
     else: unaryExpr(p, e, d, "($1 ? #nimCStrLen($1) : 0)")
   of tyString:
@@ -1849,20 +1768,20 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     else: putIntoDest(p, d, e, rope(lengthOrd(p.config, typ)))
   else: internalError(p.config, e.info, "genArrayLen()")
 
-proc makePtrType(baseType: PType; idgen: IdGenerator): PType =
-  result = newType(tyPtr, nextTypeId idgen, baseType.owner)
-  addSonSkipIntLit(result, baseType, idgen)
+proc makePtrType(baseType: PType): PType =
+  result = newType(tyPtr, baseType.owner)
+  addSonSkipIntLit(result, baseType)
 
-proc makeAddr(n: PNode; idgen: IdGenerator): PNode =
+proc makeAddr(n: PNode): PNode =
   if n.kind == nkHiddenAddr:
     result = n
   else:
     result = newTree(nkHiddenAddr, n)
-    result.typ = makePtrType(n.typ, idgen)
+    result.typ = makePtrType(n.typ)
 
 proc genSetLengthSeq(p: BProc, e: PNode, d: var TLoc) =
   if optSeqDestructors in p.config.globalOptions:
-    e[1] = makeAddr(e[1], p.module.idgen)
+    e[1] = makeAddr(e[1])
     genCall(p, e, d)
     return
   var a, b, call: TLoc
@@ -1875,7 +1794,7 @@ proc genSetLengthSeq(p: BProc, e: PNode, d: var TLoc) =
 
   initLoc(call, locCall, e, OnHeap)
   if not p.module.compileToCpp:
-    const setLenPattern = "($3) #setLengthSeqV2(($1)?&($1)->Sup:NIM_NIL, $4, $2)"
+    const setLenPattern = "($3) #setLengthSeqV2(&($1)->Sup, $4, $2)"
     call.r = ropecg(p.module, setLenPattern, [
       rdLoc(a), rdLoc(b), getTypeDesc(p.module, t),
       genTypeInfoV1(p.module, t.skipTypes(abstractInst), e.info)])
@@ -2047,7 +1966,7 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     of mCard:
       var a: TLoc
       initLocExpr(p, e[1], a)
-      putIntoDest(p, d, e, ropecg(p.module, "#cardSet($1, $2)", [addrLoc(p.config, a), size]))
+      putIntoDest(p, d, e, ropecg(p.module, "#cardSet($1, $2)", [rdCharLoc(a), size]))
     of mLtSet, mLeSet:
       getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i) # our counter
       initLocExpr(p, e[1], a)
@@ -2109,11 +2028,6 @@ proc genSomeCast(p: BProc, e: PNode, d: var TLoc) =
     elif etyp.kind == tyBool and srcTyp.kind in IntegralTypes:
       putIntoDest(p, d, e, "(($1) != 0)" % [rdCharLoc(a)], a.storage)
     else:
-      if etyp.kind == tyPtr:
-        # generates the definition of structs for casts like cast[ptr object](addr x)[]
-        let internalType = etyp.skipTypes({tyPtr})
-        if internalType.kind == tyObject:
-          discard getTypeDesc(p.module, internalType)
       putIntoDest(p, d, e, "(($1) ($2))" %
           [getTypeDesc(p.module, e.typ), rdCharLoc(a)], a.storage)
 
@@ -2287,8 +2201,7 @@ proc genDestroy(p: BProc; n: PNode) =
     else: discard "nothing to do"
   else:
     let t = n[1].typ.skipTypes(abstractVar)
-    let op = getAttachedOp(p.module.g.graph, t, attachedDestructor)
-    if op != nil and getBody(p.module.g.graph, op).len != 0:
+    if t.destructor != nil and t.destructor.ast[bodyPos].len != 0:
       internalError(p.config, n.info, "destructor turned out to be not trivial")
     discard "ignore calls to the default destructor"
 
@@ -2310,10 +2223,7 @@ proc genDispose(p: BProc; n: PNode) =
       lineCg(p, cpsStmts, ["#nimDestroyAndDispose($#)", rdLoc(a)])
 
 proc genSlice(p: BProc; e: PNode; d: var TLoc) =
-  let (x, y) = genOpenArraySlice(p, e, e.typ, e.typ.lastSon,
-    prepareForMutation = e[1].kind == nkHiddenDeref and
-                         e[1].typ.skipTypes(abstractInst).kind == tyString and
-                         p.config.selectedGC in {gcArc, gcOrc})
+  let (x, y) = genOpenArraySlice(p, e, e.typ, e.typ.lastSon)
   if d.k == locNone: getTemp(p, e.typ, d)
   linefmt(p, cpsStmts, "$1.Field0 = $2; $1.Field1 = $3;$n", [rdLoc(d), x, y])
   when false:
@@ -2321,9 +2231,16 @@ proc genSlice(p: BProc; e: PNode; d: var TLoc) =
       "'toOpenArray' is only valid within a call expression")
 
 proc genEnumToStr(p: BProc, e: PNode, d: var TLoc) =
+  const ToStringProcSlot = -4
   let t = e[1].typ.skipTypes(abstractInst+{tyRange})
-  let toStrProc = getToStringProc(p.module.g.graph, t)
-  # XXX need to modify this logic for IC.
+  var toStrProc: PSym = nil
+  for idx, p in items(t.methods):
+    if idx == ToStringProcSlot:
+      toStrProc = p
+      break
+  if toStrProc == nil:
+    toStrProc = genEnumToStrProc(t, e.info, p.module.g.graph)
+    t.methods.add((ToStringProcSlot, toStrProc))
   var n = copyTree(e)
   n[0] = newSymNode(toStrProc)
   expr(p, n, d)
@@ -2355,7 +2272,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       initLocExpr(p, e[1], a)
       initLocExpr(p, e[2], b)
 
-      let ranged = skipTypes(e[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent, tyDistinct})
+      let ranged = skipTypes(e[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent})
       let res = binaryArithOverflowRaw(p, ranged, a, b,
         if underlying.kind == tyInt64: fun64[op] else: fun[op])
 
@@ -2376,7 +2293,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mAppendStrStr: genStrAppend(p, e, d)
   of mAppendSeqElem:
     if optSeqDestructors in p.config.globalOptions:
-      e[1] = makeAddr(e[1], p.module.idgen)
+      e[1] = makeAddr(e[1])
       genCall(p, e, d)
     else:
       genSeqElemAppend(p, e, d)
@@ -2388,14 +2305,10 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mInt64ToStr: genDollar(p, e, d, "#nimInt64ToStr($1)")
   of mBoolToStr: genDollar(p, e, d, "#nimBoolToStr($1)")
   of mCharToStr: genDollar(p, e, d, "#nimCharToStr($1)")
-  of mFloatToStr:
-    if e[1].typ.skipTypes(abstractInst).kind == tyFloat32:
-      genDollar(p, e, d, "#nimFloat32ToStr($1)")
-    else:
-      genDollar(p, e, d, "#nimFloatToStr($1)")
+  of mFloatToStr: genDollar(p, e, d, "#nimFloatToStr($1)")
   of mCStrToStr: genDollar(p, e, d, "#cstrToNimstr($1)")
   of mStrToStr, mUnown: expr(p, e[1], d)
-  of generatedMagics: genCall(p, e, d)
+  of mIsolate: genCall(p, e, d)
   of mEnumToStr:
     if optTinyRtti in p.config.globalOptions:
       genEnumToStr(p, e, d)
@@ -2413,7 +2326,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       genNewFinalize(p, e)
   of mNewSeq:
     if optSeqDestructors in p.config.globalOptions:
-      e[1] = makeAddr(e[1], p.module.idgen)
+      e[1] = makeAddr(e[1])
       genCall(p, e, d)
     else:
       genNewSeq(p, e)
@@ -2457,7 +2370,6 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     #   somehow forward-declared from some other usage, but it is *possible*
     if lfNoDecl notin opr.loc.flags:
       let prc = magicsys.getCompilerProc(p.module.g.graph, $opr.loc.r)
-      assert prc != nil, $opr.loc.r
       # HACK:
       # Explicitly add this proc as declared here so the cgsym call doesn't
       # add a forward declaration - without this we could end up with the same
@@ -2484,15 +2396,15 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     localError(p.config, e.info, strutils.`%`(errXMustBeCompileTime, e[0].sym.name.s))
   of mSpawn:
     when defined(leanCompiler):
-      p.config.quitOrRaise "compiler built without support for the 'spawn' statement"
+      quit "compiler built without support for the 'spawn' statement"
     else:
-      let n = spawn.wrapProcForSpawn(p.module.g.graph, p.module.idgen, p.module.module, e, e.typ, nil, nil)
+      let n = spawn.wrapProcForSpawn(p.module.g.graph, p.module.module, e, e.typ, nil, nil)
       expr(p, n, d)
   of mParallel:
     when defined(leanCompiler):
-      p.config.quitOrRaise "compiler built without support for the 'parallel' statement"
+      quit "compiler built without support for the 'parallel' statement"
     else:
-      let n = semparallel.liftParallel(p.module.g.graph, p.module.idgen, p.module.module, e)
+      let n = semparallel.liftParallel(p.module.g.graph, p.module.module, e)
       expr(p, n, d)
   of mDeepCopy:
     if p.config.selectedGC in {gcArc, gcOrc} and optEnableDeepCopy notin p.config.globalOptions:
@@ -2509,9 +2421,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mMove: genMove(p, e, d)
   of mDestroy: genDestroy(p, e)
   of mAccessEnv: unaryExpr(p, e, d, "$1.ClE_0")
-  of mAccessTypeField: genAccessTypeField(p, e, d)
   of mSlice: genSlice(p, e, d)
-  of mTrace: discard "no code to generate"
   else:
     when defined(debugMagics):
       echo p.prc.name.s, " ", p.prc.id, " ", p.prc.flags, " ", p.prc.ast[genericParamsPos].kind
@@ -2568,30 +2478,15 @@ proc genTupleConstr(p: BProc, n: PNode, d: var TLoc) =
   if not handleConstExpr(p, n, d):
     let t = n.typ
     discard getTypeDesc(p.module, t) # so that any fields are initialized
-
-    var tmp: TLoc
-    # bug #16331
-    let doesAlias = lhsDoesAlias(d.lode, n)
-    let dest = if doesAlias: addr(tmp) else: addr(d)
-    if doesAlias:
-      getTemp(p, n.typ, tmp)
-    elif d.k == locNone:
-      getTemp(p, n.typ, d)
-
+    if d.k == locNone: getTemp(p, t, d)
     for i in 0..<n.len:
       var it = n[i]
       if it.kind == nkExprColonExpr: it = it[1]
-      initLoc(rec, locExpr, it, dest[].storage)
-      rec.r = "$1.Field$2" % [rdLoc(dest[]), rope(i)]
+      initLoc(rec, locExpr, it, d.storage)
+      rec.r = "$1.Field$2" % [rdLoc(d), rope(i)]
       rec.flags.incl(lfEnforceDeref)
       expr(p, it, rec)
 
-    if doesAlias:
-      if d.k == locNone:
-        d = tmp
-      else:
-        genAssignment(p, d, tmp, {})
-
 proc isConstClosure(n: PNode): bool {.inline.} =
   result = n[0].kind == nkSym and isRoutine(n[0].sym) and
       n[1].kind == nkNilLit
@@ -2697,32 +2592,40 @@ proc upConv(p: BProc, n: PNode, d: var TLoc) =
                         [getTypeDesc(p.module, dest), addrLoc(p.config, a)], a.storage)
 
 proc downConv(p: BProc, n: PNode, d: var TLoc) =
-  var arg = n[0]
-  while arg.kind == nkObjDownConv: arg = arg[0]
-
-  let dest = skipTypes(n.typ, abstractPtrs)
-  let src = skipTypes(arg.typ, abstractPtrs)
-  discard getTypeDesc(p.module, src)
-  let isRef = skipTypes(arg.typ, abstractInstOwned).kind in {tyRef, tyPtr, tyVar, tyLent}
-  if isRef and d.k == locNone and n.typ.skipTypes(abstractInstOwned).kind in {tyRef, tyPtr} and n.isLValue:
-    # it can happen that we end up generating '&&x->Sup' here, so we pack
-    # the '&x->Sup' into a temporary and then those address is taken
-    # (see bug #837). However sometimes using a temporary is not correct:
-    # init(TFigure(my)) # where it is passed to a 'var TFigure'. We test
-    # this by ensuring the destination is also a pointer:
-    var a: TLoc
-    initLocExpr(p, arg, a)
-    putIntoDest(p, d, n,
-              "(*(($1*) (&($2))))" % [getTypeDesc(p.module, n.typ), rdLoc(a)], a.storage)
-  elif p.module.compileToCpp:
-    # C++ implicitly downcasts for us
-    expr(p, arg, d)
+  if p.module.compileToCpp:
+    discard getTypeDesc(p.module, skipTypes(n[0].typ, abstractPtrs))
+    expr(p, n[0], d)     # downcast does C++ for us
   else:
+    var dest = skipTypes(n.typ, abstractPtrs)
+
+    var arg = n[0]
+    while arg.kind == nkObjDownConv: arg = arg[0]
+
+    var src = skipTypes(arg.typ, abstractPtrs)
+    discard getTypeDesc(p.module, src)
     var a: TLoc
     initLocExpr(p, arg, a)
-    var r = rdLoc(a) & (if isRef: "->Sup" else: ".Sup")
+    var r = rdLoc(a)
+    let isRef = skipTypes(arg.typ, abstractInstOwned).kind in {tyRef, tyPtr, tyVar, tyLent}
+    if isRef:
+      r.add("->Sup")
+    else:
+      r.add(".Sup")
     for i in 2..abs(inheritanceDiff(dest, src)): r.add(".Sup")
-    putIntoDest(p, d, n, if isRef: "&" & r else: r, a.storage)
+    if isRef:
+      # it can happen that we end up generating '&&x->Sup' here, so we pack
+      # the '&x->Sup' into a temporary and then those address is taken
+      # (see bug #837). However sometimes using a temporary is not correct:
+      # init(TFigure(my)) # where it is passed to a 'var TFigure'. We test
+      # this by ensuring the destination is also a pointer:
+      if d.k == locNone and skipTypes(n.typ, abstractInstOwned).kind in {tyRef, tyPtr, tyVar, tyLent}:
+        getTemp(p, n.typ, d)
+        linefmt(p, cpsStmts, "$1 = &$2;$n", [rdLoc(d), r])
+      else:
+        r = "&" & r
+        putIntoDest(p, d, n, r, a.storage)
+    else:
+      putIntoDest(p, d, n, r, a.storage)
 
 proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) =
   let t = n.typ
@@ -2745,62 +2648,7 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) =
     if t.kind notin {tySequence, tyString}:
       d.storage = OnStatic
 
-proc genConstSetup(p: BProc; sym: PSym): bool =
-  let m = p.module
-  useHeader(m, sym)
-  if sym.loc.k == locNone:
-    fillLoc(sym.loc, locData, sym.ast, mangleName(p.module, sym), OnStatic)
-  if m.hcrOn: incl(sym.loc.flags, lfIndirect)
-  result = lfNoDecl notin sym.loc.flags
-
-proc genConstHeader(m, q: BModule; p: BProc, sym: PSym) =
-  if sym.loc.r == nil:
-    if not genConstSetup(p, sym): return
-  assert(sym.loc.r != nil, $sym.name.s & $sym.itemId)
-  if m.hcrOn:
-    m.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]);
-    m.initProc.procSec(cpsLocals).addf(
-      "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r,
-      getTypeDesc(m, sym.loc.t, skVar), getModuleDllPath(q, sym)])
-  else:
-    let headerDecl = "extern NIM_CONST $1 $2;$n" %
-        [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]
-    m.s[cfsData].add(headerDecl)
-    if sfExportc in sym.flags and p.module.g.generatedHeader != nil:
-      p.module.g.generatedHeader.s[cfsData].add(headerDecl)
-
-proc genConstDefinition(q: BModule; p: BProc; sym: PSym) =
-  # add a suffix for hcr - will later init the global pointer with this data
-  let actualConstName = if q.hcrOn: sym.loc.r & "_const" else: sym.loc.r
-  q.s[cfsData].addf("N_LIB_PRIVATE NIM_CONST $1 $2 = $3;$n",
-      [getTypeDesc(q, sym.typ), actualConstName,
-      genBracedInit(q.initProc, sym.ast, isConst = true, sym.typ)])
-  if q.hcrOn:
-    # generate the global pointer with the real name
-    q.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(q, sym.loc.t, skVar), sym.loc.r])
-    # register it (but ignore the boolean result of hcrRegisterGlobal)
-    q.initProc.procSec(cpsLocals).addf(
-      "\thcrRegisterGlobal($1, \"$2\", sizeof($3), NULL, (void**)&$2);$n",
-      [getModuleDllPath(q, sym), sym.loc.r, rdLoc(sym.loc)])
-    # always copy over the contents of the actual constant with the _const
-    # suffix ==> this means that the constant is reloadable & updatable!
-    q.initProc.procSec(cpsLocals).add(ropecg(q,
-      "\t#nimCopyMem((void*)$1, (NIM_CONST void*)&$2, sizeof($3));$n",
-      [sym.loc.r, actualConstName, rdLoc(sym.loc)]))
-
-proc genConstStmt(p: BProc, n: PNode) =
-  # This code is only used in the new DCE implementation.
-  assert useAliveDataFromDce in p.module.flags
-  let m = p.module
-  for it in n:
-    if it[0].kind == nkSym:
-      let sym = it[0].sym
-      if not isSimpleConst(sym.typ) and sym.itemId.item in m.alive and genConstSetup(p, sym):
-        genConstDefinition(m, p, sym)
-
 proc expr(p: BProc, n: PNode, d: var TLoc) =
-  when defined(nimCompilerStacktraceHints):
-    setFrameMsg p.config$n.info & " " & $n.kind
   p.currLineInfo = n.info
 
   case n.kind
@@ -2808,7 +2656,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
     var sym = n.sym
     case sym.kind
     of skMethod:
-      if useAliveDataFromDce in p.module.flags or {sfDispatcher, sfForward} * sym.flags != {}:
+      if {sfDispatcher, sfForward} * sym.flags != {}:
         # we cannot produce code for the dispatcher yet:
         fillProcLoc(p.module, n)
         genProcPrototype(p.module, sym)
@@ -2821,21 +2669,13 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       if sfCompileTime in sym.flags:
         localError(p.config, n.info, "request to generate code for .compileTime proc: " &
            sym.name.s)
-      if useAliveDataFromDce in p.module.flags and sym.typ.callConv != ccInline:
-        fillProcLoc(p.module, n)
-        genProcPrototype(p.module, sym)
-      else:
-        genProc(p.module, sym)
+      genProc(p.module, sym)
       if sym.loc.r == nil or sym.loc.lode == nil:
         internalError(p.config, n.info, "expr: proc not init " & sym.name.s)
       putLocIntoDest(p, d, sym.loc)
     of skConst:
       if isSimpleConst(sym.typ):
         putIntoDest(p, d, n, genLiteral(p, sym.ast, sym.typ), OnStatic)
-      elif useAliveDataFromDce in p.module.flags:
-        genConstHeader(p.module, p.module, p, sym)
-        assert((sym.loc.r != nil) and (sym.loc.t != nil))
-        putLocIntoDest(p, d, sym.loc)
       else:
         genComplexConst(p, sym, d)
     of skEnumField:
@@ -2956,10 +2796,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
   of nkEmpty: discard
   of nkWhileStmt: genWhileStmt(p, n)
   of nkVarSection, nkLetSection: genVarStmt(p, n)
-  of nkConstSection:
-    if useAliveDataFromDce in p.module.flags:
-      genConstStmt(p, n)
-    # else: consts generated lazily on use
+  of nkConstSection: discard  # consts generated lazily on use
   of nkForStmt: internalError(p.config, n.info, "for statement not eliminated")
   of nkCaseStmt: genCase(p, n, d)
   of nkReturnStmt: genReturnStmt(p, n)
@@ -3004,17 +2841,13 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
   of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef:
     if n[genericParamsPos].kind == nkEmpty:
       var prc = n[namePos].sym
-      if useAliveDataFromDce in p.module.flags:
-        if p.module.alive.contains(prc.itemId.item) and
-            prc.magic in generatedMagics:
-          genProc(p.module, prc)
-      elif prc.skipGenericOwner.kind == skModule and sfCompileTime notin prc.flags:
+      # due to a bug/limitation in the lambda lifting, unused inner procs
+      # are not transformed correctly. We work around this issue (#411) here
+      # by ensuring it's no inner proc (owner is a module):
+      if prc.skipGenericOwner.kind == skModule and sfCompileTime notin prc.flags:
         if ({sfExportc, sfCompilerProc} * prc.flags == {sfExportc}) or
             (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or
             (prc.kind == skMethod):
-          # due to a bug/limitation in the lambda lifting, unused inner procs
-          # are not transformed correctly. We work around this issue (#411) here
-          # by ensuring it's no inner proc (owner is a module).
           # Generate proc even if empty body, bugfix #11651.
           genProc(p.module, prc)
   of nkParForStmt: genParForStmt(p, n)
@@ -3037,7 +2870,7 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo): Rope =
   of tyBool: result = rope"NIM_FALSE"
   of tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64: result = rope"0"
   of tyFloat..tyFloat128: result = rope"0.0"
-  of tyCstring, tyVar, tyLent, tyPointer, tyPtr, tyUntyped,
+  of tyCString, tyVar, tyLent, tyPointer, tyPtr, tyUntyped,
      tyTyped, tyTypeDesc, tyStatic, tyRef, tyNil:
     result = rope"NIM_NIL"
   of tyString, tySequence:
@@ -3263,9 +3096,7 @@ proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType): Rope
       else:
         result = genConstSeq(p, n, typ, isConst)
     of tyProc:
-      if typ.callConv == ccClosure and n.safeLen > 1 and n[1].kind == nkNilLit:
-        # n.kind could be: nkClosure, nkTupleConstr and maybe others; `n.safeLen`
-        # guards against the case of `nkSym`, refs bug #14340.
+      if typ.callConv == ccClosure and n.len > 1 and n[1].kind == nkNilLit:
         # Conversion: nimcall -> closure.
         # this hack fixes issue that nkNilLit is expanded to {NIM_NIL,NIM_NIL}
         # this behaviour is needed since closure_var = nil must be
@@ -3304,7 +3135,7 @@ proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType): Rope
 
     of tyObject:
       result = genConstObjConstr(p, n, isConst)
-    of tyString, tyCstring:
+    of tyString, tyCString:
       if optSeqDestructors in p.config.globalOptions and n.kind != nkNilLit and ty == tyString:
         result = genStringLiteralV2Const(p.module, n, isConst)
       else:

compiler/ccgmerge.nim

@@ -180,6 +180,10 @@ proc readKey(L: var TBaseLexer, result: var string) =
   if L.buf[pos] != ':': doAssert(false, "ccgmerge: ':' expected")
   L.bufpos = pos + 1 # skip ':'
 
+proc newFakeType(id: int): PType =
+  new(result)
+  result.id = id
+
 proc readTypeCache(L: var TBaseLexer, result: var TypeCache) =
   if ^L.bufpos != '{': doAssert(false, "ccgmerge: '{' expected")
   inc L.bufpos
@@ -188,7 +192,10 @@ proc readTypeCache(L: var TBaseLexer, result: var TypeCache) =
     var key = decodeStr(L.buf, L.bufpos)
     if ^L.bufpos != ':': doAssert(false, "ccgmerge: ':' expected")
     inc L.bufpos
-    discard decodeStr(L.buf, L.bufpos)
+    var value = decodeStr(L.buf, L.bufpos)
+    # XXX implement me
+    when false:
+      idTablePut(result, newFakeType(key), value.rope)
   inc L.bufpos
 
 proc readIntSet(L: var TBaseLexer, result: var IntSet) =
@@ -218,6 +225,9 @@ proc processMergeInfo(L: var TBaseLexer, m: BModule) =
       m.flags = cast[set[CodegenFlag]](decodeVInt(L.buf, L.bufpos) != 0)
     else: doAssert(false, "ccgmerge: unknown key: " & k)
 
+when not defined(nimhygiene):
+  {.pragma: inject.}
+
 template withCFile(cfilename: AbsoluteFile, body: untyped) =
   var s = llStreamOpen(cfilename, fmRead)
   if s == nil: return

compiler/ccgreset.nim

@@ -83,24 +83,11 @@ proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
       lineCg(p, cpsStmts, "$1.ClP_0 = NIM_NIL;$n", [accessor])
     else:
       lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
-  of tyChar, tyBool, tyEnum, tyRange, tyInt..tyUInt64:
+  of tyChar, tyBool, tyEnum, tyInt..tyUInt64:
     lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
-  of tyCstring, tyPointer, tyPtr, tyVar, tyLent:
+  of tyCString, tyPointer, tyPtr, tyVar, tyLent:
     lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
-  of tySet:
-    case mapSetType(p.config, typ)
-    of ctArray:
-      lineCg(p, cpsStmts, "#nimZeroMem($1, sizeof($2));$n",
-          [accessor, getTypeDesc(p.module, typ)])
-    of ctInt8, ctInt16, ctInt32, ctInt64:
-      lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
-    else:
-      doAssert false, "unexpected set type kind"
-  of {tyNone, tyEmpty, tyNil, tyUntyped, tyTyped, tyGenericInvocation,
-      tyGenericParam, tyOrdinal, tyOpenArray, tyForward, tyVarargs,
-      tyUncheckedArray, tyProxy, tyBuiltInTypeClass, tyUserTypeClass,
-      tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot,
-      tyAnything, tyStatic, tyFromExpr, tyConcept, tyVoid, tyIterable}:
+  else:
     discard
 
 proc specializeReset(p: BProc, a: TLoc) =

compiler/ccgstmts.nim

@@ -32,20 +32,13 @@ proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) =
       "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc])
 
 proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} =
-  if n.kind == nkEmpty:
-    result = false
-  elif n.kind in nkCallKinds and n[0] != nil and n[0].typ != nil and n[0].typ.skipTypes(abstractInst).kind == tyProc:
-    if isInvalidReturnType(conf, n[0].typ, true):
-      # var v = f()
-      # is transformed into: var v;  f(addr v)
-      # where 'f' **does not** initialize the result!
-      result = false
-    else:
-      result = true
-  elif isInvalidReturnType(conf, n.typ, false):
-    result = false
-  else:
-    result = true
+  if n.kind == nkEmpty: return false
+  if isInvalidReturnType(conf, n.typ):
+    # var v = f()
+    # is transformed into: var v;  f(addr v)
+    # where 'f' **does not** initialize the result!
+    return false
+  result = true
 
 proc inExceptBlockLen(p: BProc): int =
   for x in p.nestedTryStmts:
@@ -73,7 +66,7 @@ proc genVarTuple(p: BProc, n: PNode) =
   # if we have a something that's been captured, use the lowering instead:
   for i in 0..<n.len-2:
     if n[i].kind != nkSym:
-      genStmts(p, lowerTupleUnpacking(p.module.g.graph, n, p.module.idgen, p.prc))
+      genStmts(p, lowerTupleUnpacking(p.module.g.graph, n, p.prc))
       return
 
   # check only the first son
@@ -405,7 +398,7 @@ proc genClosureVar(p: BProc, a: PNode) =
   genLineDir(p, a)
   if immediateAsgn:
     loadInto(p, a[0], a[2], v)
-  elif sfNoInit notin a[0][1].sym.flags:
+  else:
     constructLoc(p, v)
 
 proc genVarStmt(p: BProc, n: PNode) =
@@ -660,19 +653,12 @@ proc genParForStmt(p: BProc, t: PNode) =
     initLocExpr(p, call[2], rangeB)
 
     # $n at the beginning because of #9710
-    if call.len == 4: # procName(a, b, annotation)
-      if call[0].sym.name.s == "||":  # `||`(a, b, annotation)
-        lineF(p, cpsStmts, "$n#pragma omp $4$n" &
-                            "for ($1 = $2; $1 <= $3; ++$1)",
-                            [forLoopVar.loc.rdLoc,
-                            rangeA.rdLoc, rangeB.rdLoc,
-                            call[3].getStr.rope])
-      else:
-        lineF(p, cpsStmts, "$n#pragma $4$n" &
-                    "for ($1 = $2; $1 <= $3; ++$1)",
-                    [forLoopVar.loc.rdLoc,
-                    rangeA.rdLoc, rangeB.rdLoc,
-                    call[3].getStr.rope])
+    if call.len == 4: # `||`(a, b, annotation)
+      lineF(p, cpsStmts, "$n#pragma omp $4$n" &
+                          "for ($1 = $2; $1 <= $3; ++$1)",
+                          [forLoopVar.loc.rdLoc,
+                          rangeA.rdLoc, rangeB.rdLoc,
+                          call[3].getStr.rope])
     else: # `||`(a, b, step, annotation)
       var step: TLoc
       initLocExpr(p, call[3], step)
@@ -1367,18 +1353,13 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
       linefmt(p, cpsStmts, "$1.status = __builtin_setjmp($1.context);$n", [safePoint])
     elif isDefined(p.config, "nimRawSetjmp"):
       if isDefined(p.config, "mswindows"):
-        if isDefined(p.config, "vcc") or isDefined(p.config, "clangcl"):
-          # For the vcc compiler, use `setjmp()` with one argument.
-          # See https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setjmp?view=msvc-170
-          linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])        
-        else:
-          # The Windows `_setjmp()` takes two arguments, with the second being an
-          # undocumented buffer used by the SEH mechanism for stack unwinding.
-          # Mingw-w64 has been trying to get it right for years, but it's still
-          # prone to stack corruption during unwinding, so we disable that by setting
-          # it to NULL.
-          # More details: https://github.com/status-im/nimbus-eth2/issues/3121
-          linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint])
+        # The Windows `_setjmp()` takes two arguments, with the second being an
+        # undocumented buffer used by the SEH mechanism for stack unwinding.
+        # Mingw-w64 has been trying to get it right for years, but it's still
+        # prone to stack corruption during unwinding, so we disable that by setting
+        # it to NULL.
+        # More details: https://github.com/status-im/nimbus-eth2/issues/3121
+        linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint])
       else:
         linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint])
     else:
@@ -1532,6 +1513,11 @@ proc genPragma(p: BProc, n: PNode) =
   for it in n.sons:
     case whichPragma(it)
     of wEmit: genEmit(p, it)
+    of wInjectStmt:
+      var p = newProc(nil, p.module)
+      p.options.excl {optLineTrace, optStackTrace}
+      genStmts(p, it[1])
+      p.module.injectStmt = p.s(cpsStmts)
     else: discard
 
 
@@ -1548,21 +1534,20 @@ proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
         [rdLoc(a), rdLoc(tmp), discriminatorTableName(p.module, t, field),
          intLiteral(toInt64(lengthOrd(p.config, field.typ))+1)])
 
-when false:
-  proc genCaseObjDiscMapping(p: BProc, e: PNode, t: PType, field: PSym; d: var TLoc) =
-    const ObjDiscMappingProcSlot = -5
-    var theProc: PSym = nil
-    for idx, p in items(t.methods):
-      if idx == ObjDiscMappingProcSlot:
-        theProc = p
-        break
-    if theProc == nil:
-      theProc = genCaseObjDiscMapping(t, field, e.info, p.module.g.graph, p.module.idgen)
-      t.methods.add((ObjDiscMappingProcSlot, theProc))
-    var call = newNodeIT(nkCall, e.info, getSysType(p.module.g.graph, e.info, tyUInt8))
-    call.add newSymNode(theProc)
-    call.add e
-    expr(p, call, d)
+proc genCaseObjDiscMapping(p: BProc, e: PNode, t: PType, field: PSym; d: var TLoc) =
+  const ObjDiscMappingProcSlot = -5
+  var theProc: PSym = nil
+  for idx, p in items(t.methods):
+    if idx == ObjDiscMappingProcSlot:
+      theProc = p
+      break
+  if theProc == nil:
+    theProc = genCaseObjDiscMapping(t, field, e.info, p.module.g.graph)
+    t.methods.add((ObjDiscMappingProcSlot, theProc))
+  var call = newNodeIT(nkCall, e.info, getSysType(p.module.g.graph, e.info, tyUInt8))
+  call.add newSymNode(theProc)
+  call.add e
+  expr(p, call, d)
 
 proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   var a, tmp: TLoc
@@ -1592,7 +1577,6 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
     initLoc(a, locNone, le, OnUnknown)
     a.flags.incl(lfEnforceDeref)
     a.flags.incl(lfPrepareForMutation)
-    genLineDir(p, le) # it can be a nkBracketExpr, which may raise
     expr(p, le, a)
     a.flags.excl(lfPrepareForMutation)
     if fastAsgn: incl(a.flags, lfNoDeepCopy)

compiler/ccgthreadvars.nim

@@ -7,8 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
-## Thread var support for architectures that lack native support for
-## thread local storage.
+## Thread var support for crappy architectures that lack native support for
+## thread local storage. (**Thank you Mac OS X!**)
 
 # included from cgen.nim
 
@@ -35,11 +35,7 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
     if isExtern: m.s[cfsVars].add("extern ")
     elif lfExportLib in s.loc.flags: m.s[cfsVars].add("N_LIB_EXPORT_VAR ")
     else: m.s[cfsVars].add("N_LIB_PRIVATE ")
-    if optThreads in m.config.globalOptions:
-      let sym = s.typ.sym
-      if sym != nil and sfCppNonPod in sym.flags:
-        m.s[cfsVars].add("NIM_THREAD_LOCAL ")
-      else: m.s[cfsVars].add("NIM_THREADVAR ")
+    if optThreads in m.config.globalOptions: m.s[cfsVars].add("NIM_THREADVAR ")
     m.s[cfsVars].add(getTypeDesc(m, s.loc.t))
     m.s[cfsVars].addf(" $1;$n", [s.loc.r])
 

compiler/ccgtypes.nim

@@ -12,6 +12,7 @@
 # ------------------------- Name Mangling --------------------------------
 
 import sighashes, modulegraphs
+from lowerings import createObj
 
 proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope
 
@@ -35,97 +36,19 @@ proc mangleField(m: BModule; name: PIdent): string =
   if isKeyword(name):
     result.add "_0"
 
-
-proc encodeName*(name: string): string = 
-  result = mangle(name)
-  result = $result.len & result
-
-proc makeUnique(m: BModule; s: PSym, name: string = ""): Rope = 
-  let str = if name == "": s.name.s else: name
-  result.add str
-  result.add "__"
-  result.add m.g.graph.ifaces[s.itemId.module].uniqueName
-  result.add "_u"
-  result.add $s.itemId.item
-
-proc encodeSym*(m: BModule; s: PSym; makeUnique: bool = false): string = 
-  #Module::Type
-  var name = s.name.s 
-  if makeUnique:
-    name = $makeUnique(m, s, name)
-  "N" & encodeName(s.skipGenericOwner.name.s) & encodeName(name) & "E"
-
-proc elementType*(n: PType): PType {.inline.} = n.sons[^1]
-
-proc encodeType*(m: BModule; t: PType): string = 
-  result = ""
-  var kindName = ($t.kind)[2..^1]
-  kindName[0] = toLower($kindName[0])[0]
-  case t.kind
-  of tyObject, tyEnum, tyDistinct, tyUserTypeClass, tyGenericParam: 
-    result = encodeSym(m, t.sym)
-  of tyGenericInst, tyUserTypeClassInst, tyGenericBody:
-    result = encodeName(t[0].sym.name.s)
-    result.add "I"
-    for i in 1..<t.len - 1: 
-      result.add encodeType(m, t[i])
-    result.add "E"
-  of tySequence, tyOpenArray, tyArray, tyVarargs, tyTuple, tyProc, tySet, tyTypeDesc,
-    tyPtr, tyRef, tyVar, tyLent, tySink, tyStatic, tyUncheckedArray, tyOr, tyAnd, tyBuiltInTypeClass:
-    result = 
-      case t.kind:
-      of tySequence: encodeName("seq") 
-      else: encodeName(kindName)
-    result.add "I"
-    for i in 0..<t.len:
-      let s = t[i]  
-      if s.isNil: continue
-      result.add encodeType(m, s)
-    result.add "E"
-  of tyRange:
-    var val = "range_"
-    if t.n[0].typ.kind in {tyFloat..tyFloat128}: 
-      val.addFloat t.n[0].floatVal 
-      val.add "_" 
-      val.addFloat t.n[1].floatVal
-    else: 
-      val.add $t.n[0].intVal & "_" & $t.n[1].intVal
-    result = encodeName(val)
-  of tyString..tyUInt64, tyPointer, tyBool, tyChar, tyVoid, tyAnything, tyNil, tyEmpty: 
-    result = encodeName(kindName)
-  of tyAlias, tyInferred, tyOwned: 
-    result = encodeType(m, t.elementType)
-  else:
-    assert false, "encodeType " & $t.kind
-
-proc mangleProc(m: BModule; s: PSym; makeUnique: bool): string = 
-  result = "_Z"  # Common prefix in Itanium ABI
-  result.add encodeSym(m, s, makeUnique)
-  if s.typ.len > 1: #we dont care about the return param
-    for i in 1..<s.typ.len: 
-      if s.typ[i].isNil: continue
-      result.add encodeType(m, s.typ[i])
-  
-  if result in m.g.mangledPrcs:
-    result = mangleProc(m, s, true)
-  else:
-    m.g.mangledPrcs.incl(result)
+when false:
+  proc hashOwner(s: PSym): SigHash =
+    var m = s
+    while m.kind != skModule: m = m.owner
+    let p = m.owner
+    assert p.kind == skPackage
+    result = gDebugInfo.register(p.name.s, m.name.s)
 
 proc mangleName(m: BModule; s: PSym): Rope =
-  if $s.loc.r == "":
-    var result: Rope
-    if s.kind in routineKinds and optCDebug in m.g.config.globalOptions and
-      m.g.config.symbolFiles == disabledSf: 
-      result = mangleProc(m, s, false).rope
-    else:
-      result = s.name.s.mangle.rope
-      result.add "__"
-      result.add m.g.graph.ifaces[s.itemId.module].uniqueName
-      result.add "_u"
-      result.add $s.itemId.item # s.disamb #
-    if m.hcrOn:
-      result.add "_"
-      result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts, m.config))
+  result = s.loc.r
+  if result == nil:
+    result = s.name.s.mangle.rope
+    result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts))
     s.loc.r = result
     writeMangledName(m.ndi, s, m.config)
 
@@ -268,13 +191,13 @@ proc mapType(conf: ConfigRef; typ: PType; kind: TSymKind): TCTypeKind =
   of tySequence: result = ctNimSeq
   of tyProc: result = if typ.callConv != ccClosure: ctProc else: ctStruct
   of tyString: result = ctNimStr
-  of tyCstring: result = ctCString
+  of tyCString: result = ctCString
   of tyInt..tyUInt64:
     result = TCTypeKind(ord(typ.kind) - ord(tyInt) + ord(ctInt))
   of tyStatic:
     if typ.n != nil: result = mapType(conf, lastSon typ, kind)
-    else: doAssert(false, "mapType: " & $typ.kind)
-  else: doAssert(false, "mapType: " & $typ.kind)
+    else: doAssert(false, "mapType")
+  else: doAssert(false, "mapType")
 
 proc mapReturnType(conf: ConfigRef; typ: PType): TCTypeKind =
   #if skipTypes(typ, typedescInst).kind == tyArray: result = ctPtr
@@ -295,18 +218,12 @@ proc isObjLackingTypeField(typ: PType): bool {.inline.} =
   result = (typ.kind == tyObject) and ((tfFinal in typ.flags) and
       (typ[0] == nil) or isPureObject(typ))
 
-proc isInvalidReturnType(conf: ConfigRef; typ: PType, isProc = true): bool =
+proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool =
   # Arrays and sets cannot be returned by a C procedure, because C is
   # such a poor programming language.
   # We exclude records with refs too. This enhances efficiency and
   # is necessary for proper code generation of assignments.
-  var rettype = typ
-  var isAllowedCall = true
-  if isProc:
-    rettype = rettype[0]
-    isAllowedCall = typ.callConv in {ccClosure, ccInline, ccNimCall}
-  if rettype == nil or (isAllowedCall and
-                    getSize(conf, rettype) > conf.target.floatSize*3):
+  if rettype == nil or (tfByCopy notin rettype.flags and getSize(conf, rettype) > conf.target.floatSize*3):
     result = true
   else:
     case mapType(conf, rettype, skResult)
@@ -342,12 +259,38 @@ proc addAbiCheck(m: BModule, t: PType, name: Rope) =
     m.s[cfsTypeInfo].addf("NIM_STATIC_ASSERT(sizeof($1) == $2, $3);$n", [name, rope(size), msg2.rope])
     # see `testCodegenABICheck` for example error message it generates
 
+proc ccgIntroducedPtr(conf: ConfigRef; s: PSym, retType: PType): bool =
+  var pt = skipTypes(s.typ, typedescInst)
+  assert skResult != s.kind
 
-proc fillResult(conf: ConfigRef; param: PNode, proctype: PType) =
+  if tfByRef in pt.flags: return true
+  elif tfByCopy in pt.flags: return false
+  case pt.kind
+  of tyObject:
+    if s.typ.sym != nil and sfForward in s.typ.sym.flags:
+      # forwarded objects are *always* passed by pointers for consistency!
+      result = true
+    elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
+      result = true           # requested anyway
+    elif (tfFinal in pt.flags) and (pt[0] == nil):
+      result = false          # no need, because no subtyping possible
+    else:
+      result = true           # ordinary objects are always passed by reference,
+                              # otherwise casting doesn't work
+  of tyTuple:
+    result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options)
+  else:
+    result = false
+  # first parameter and return type is 'lent T'? --> use pass by pointer
+  if s.position == 0 and retType != nil and retType.kind == tyLent:
+    result = not (pt.kind in {tyVar, tyArray, tyOpenArray, tyVarargs, tyRef, tyPtr, tyPointer} or
+      pt.kind == tySet and mapSetType(conf, pt) == ctArray)
+
+proc fillResult(conf: ConfigRef; param: PNode) =
   fillLoc(param.sym.loc, locParam, param, ~"Result",
           OnStack)
   let t = param.sym.typ
-  if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, proctype):
+  if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, t):
     incl(param.sym.loc.flags, lfIndirect)
     param.sym.loc.storage = OnUnknown
 
@@ -376,7 +319,7 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
     else:
       discard cgsym(m, "NimStringDesc")
       result = typeNameOrLiteral(m, typ, "NimStringDesc*")
-  of tyCstring: result = typeNameOrLiteral(m, typ, "NCSTRING")
+  of tyCString: result = typeNameOrLiteral(m, typ, "NCSTRING")
   of tyBool: result = typeNameOrLiteral(m, typ, "NIM_BOOL")
   of tyChar: result = typeNameOrLiteral(m, typ, "NIM_CHAR")
   of tyNil: result = typeNameOrLiteral(m, typ, "void*")
@@ -391,7 +334,7 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
   else: result = nil
 
   if result != nil and typ.isImportedType():
-    let sig = hashType(typ, m.config)
+    let sig = hashType typ
     if cacheGetType(m.typeCache, sig) == nil:
       m.typeCache[sig] = result
 
@@ -451,10 +394,10 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet; kind: TSymKind): R
     if isImportedCppType(etB) and t.kind == tyGenericInst:
       result = getTypeDescAux(m, t, check, kind)
     else:
-      result = getTypeForward(m, t, hashType(t, m.config))
+      result = getTypeForward(m, t, hashType(t))
       pushType(m, t)
   of tySequence:
-    let sig = hashType(t, m.config)
+    let sig = hashType(t)
     if optSeqDestructors in m.config.globalOptions:
       if skipTypes(etB[0], typedescInst).kind == tyEmpty:
         internalError(m.config, "cannot map the empty seq type to a C type")
@@ -487,7 +430,7 @@ proc getSeqPayloadType(m: BModule; t: PType): Rope =
   #result = getTypeForward(m, t, hashType(t)) & "_Content"
 
 proc seqV2ContentType(m: BModule; t: PType; check: var IntSet) =
-  let sig = hashType(t, m.config)
+  let sig = hashType(t)
   let result = cacheGetType(m.typeCache, sig)
   if result == nil:
     discard getTypeDescAux(m, t, check, skVar)
@@ -512,7 +455,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
                    check: var IntSet, declareEnvironment=true;
                    weakDep=false) =
   params = nil
-  if t[0] == nil or isInvalidReturnType(m.config, t):
+  if t[0] == nil or isInvalidReturnType(m.config, t[0]):
     rettype = ~"void"
   else:
     rettype = getTypeDescAux(m, t[0], check, skResult)
@@ -537,7 +480,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
       params.add(~"NIM_NOALIAS ")
     params.add(param.loc.r)
     # declare the len field for open arrays:
-    var arr = param.typ.skipTypes({tyGenericInst})
+    var arr = param.typ
     if arr.kind in {tyVar, tyLent, tySink}: arr = arr.lastSon
     var j = 0
     while arr.kind in {tyOpenArray, tyVarargs}:
@@ -547,17 +490,12 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
       params.addf(", NI $1Len_$2", [param.loc.r, j.rope])
       inc(j)
       arr = arr[0].skipTypes({tySink})
-  if t[0] != nil and isInvalidReturnType(m.config, t):
+  if t[0] != nil and isInvalidReturnType(m.config, t[0]):
     var arr = t[0]
     if params != nil: params.add(", ")
     if mapReturnType(m.config, t[0]) != ctArray:
-      if isHeaderFile in m.flags:
-        # still generates types for `--header`
-        params.add(getTypeDescAux(m, arr, check, skResult))
-        params.add("*")
-      else:
-        params.add(getTypeDescWeak(m, arr, check, skResult))
-        params.add("*")
+      params.add(getTypeDescWeak(m, arr, check, skResult))
+      params.add("*")
     else:
       params.add(getTypeDescAux(m, arr, check, skResult))
     params.addf(" Result", [])
@@ -674,7 +612,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
 
   if typ.kind == tyObject:
     if typ[0] == nil:
-      if lacksMTypeField(typ):
+      if (typ.sym != nil and sfPure in typ.sym.flags) or tfFinal in typ.flags:
         appcg(m, result, " {$n", [])
       else:
         if optTinyRtti in m.config.globalOptions:
@@ -752,7 +690,7 @@ proc resolveStarsInCppType(typ: PType, idx, stars: int): PType =
                else: result.elemType
 
 proc getOpenArrayDesc(m: BModule, t: PType, check: var IntSet; kind: TSymKind): Rope =
-  let sig = hashType(t, m.config)
+  let sig = hashType(t)
   if kind == skParam:
     result = getTypeDescWeak(m, t[0], check, kind) & "*"
   else:
@@ -776,7 +714,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
     # C type generation into an analysis and a code generation phase somehow.
   if t.sym != nil: useHeader(m, t.sym)
   if t != origTyp and origTyp.sym != nil: useHeader(m, origTyp.sym)
-  let sig = hashType(origTyp, m.config)
+  let sig = hashType(origTyp)
 
   defer: # defer is the simplest in this case
     if isImportedType(t) and not m.typeABICache.containsOrIncl(sig):
@@ -805,7 +743,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
         result = getTypeDescAux(m, et, check, kind) & star
       else:
         # no restriction! We have a forward declaration for structs
-        let name = getTypeForward(m, et, hashType(et, m.config))
+        let name = getTypeForward(m, et, hashType et)
         result = name & star
         m.typeCache[sig] = result
     of tySequence:
@@ -814,7 +752,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
         m.typeCache[sig] = result
       else:
         # no restriction! We have a forward declaration for structs
-        let name = getTypeForward(m, et, hashType(et, m.config))
+        let name = getTypeForward(m, et, hashType et)
         result = name & seqStar(m) & star
         m.typeCache[sig] = result
         pushType(m, et)
@@ -977,7 +915,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin
           discard # addAbiCheck(m, t, result) # already handled elsewhere
   of tySet:
     # Don't use the imported name as it may be scoped: 'Foo::SomeKind'
-    result = $t.kind & '_' & t.lastSon.typeName & $t.lastSon.hashType(m.config)
+    result = $t.kind & '_' & t.lastSon.typeName & $t.lastSon.hashType
     m.typeCache[sig] = result
     if not isImportedType(t):
       let s = int(getSize(m.config, t))
@@ -1106,8 +1044,8 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
   # compute type flags for GC optimization
   var flags = 0
   if not containsGarbageCollectedRef(typ): flags = flags or 1
-  if not canFormAcycle(m.g.graph, typ): flags = flags or 2
-  #else echo("can contain a cycle: " & typeToString(typ))
+  if not canFormAcycle(typ): flags = flags or 2
+  #else MessageOut("can contain a cycle: " & typeToString(typ))
   if flags != 0:
     m.s[cfsTypeInit3].addf("$1.flags = $2;$n", [nameHcr, rope(flags)])
   discard cgsym(m, "TNimType")
@@ -1150,7 +1088,7 @@ proc discriminatorTableName(m: BModule, objtype: PType, d: PSym): Rope =
     objtype = objtype[0].skipTypes(abstractPtrs)
   if objtype.sym == nil:
     internalError(m.config, d.info, "anonymous obj with discriminator")
-  result = "NimDT_$1_$2" % [rope($hashType(objtype, m.config)), rope(d.name.s.mangle)]
+  result = "NimDT_$1_$2" % [rope($hashType(objtype)), rope(d.name.s.mangle)]
 
 proc rope(arg: Int128): Rope = rope($arg)
 
@@ -1330,10 +1268,10 @@ proc genArrayInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
 
 proc fakeClosureType(m: BModule; owner: PSym): PType =
   # we generate the same RTTI as for a tuple[pointer, ref tuple[]]
-  result = newType(tyTuple, nextTypeId m.idgen, owner)
-  result.rawAddSon(newType(tyPointer, nextTypeId m.idgen, owner))
-  var r = newType(tyRef, nextTypeId m.idgen, owner)
-  let obj = createObj(m.g.graph, m.idgen, owner, owner.info, final=false)
+  result = newType(tyTuple, owner)
+  result.rawAddSon(newType(tyPointer, owner))
+  var r = newType(tyRef, owner)
+  let obj = createObj(m.g.graph, owner, owner.info, final=false)
   r.rawAddSon(obj)
   result.rawAddSon(r)
 
@@ -1344,12 +1282,12 @@ proc genDeepCopyProc(m: BModule; s: PSym; result: Rope) =
   m.s[cfsTypeInit3].addf("$1.deepcopy =(void* (N_RAW_NIMCALL*)(void*))$2;$n",
      [result, s.loc.r])
 
-proc declareNimType(m: BModule, name: string; str: Rope, module: int) =
+proc declareNimType(m: BModule, name: string; str: Rope, ownerModule: PSym) =
   let nr = rope(name)
   if m.hcrOn:
     m.s[cfsData].addf("static $2* $1;$n", [str, nr])
     m.s[cfsTypeInit1].addf("\t$1 = ($3*)hcrGetGlobal($2, \"$1\");$n",
-          [str, getModuleDllPath(m, module), nr])
+          [str, getModuleDllPath(m, ownerModule), nr])
   else:
     m.s[cfsData].addf("extern $2 $1;$n", [str, nr])
 
@@ -1358,7 +1296,7 @@ proc genTypeInfo2Name(m: BModule; t: PType): Rope =
   var it = t
   while it != nil:
     it = it.skipTypes(skipPtrs)
-    if it.sym != nil and tfFromGeneric notin it.flags:
+    if it.sym != nil:
       var m = it.sym.owner
       while m != nil and m.kind != skModule: m = m.owner
       if m == nil or sfSystemModule in m.flags:
@@ -1371,16 +1309,16 @@ proc genTypeInfo2Name(m: BModule; t: PType): Rope =
         res.add m.name.s & "."
         res.add it.sym.name.s
     else:
-      res.add $hashType(it, m.config)
+      res.add $hashType(it)
     res.add "|"
     it = it[0]
   result = makeCString(res)
 
-proc isTrivialProc(g: ModuleGraph; s: PSym): bool {.inline.} = getBody(g, s).len == 0
+proc isTrivialProc(s: PSym): bool {.inline.} = s.ast[bodyPos].len == 0
 
 proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp): Rope =
-  let theProc = getAttachedOp(m.g.graph, t, op)
-  if theProc != nil and not isTrivialProc(m.g.graph, theProc):
+  let theProc = t.attachedOps[op]
+  if theProc != nil and not isTrivialProc(theProc):
     # the prototype of a destructor is ``=destroy(x: var T)`` and that of a
     # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling
     # convention at least:
@@ -1392,7 +1330,7 @@ proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp): Rope =
     result = theProc.loc.r
 
     when false:
-      if not canFormAcycle(m.g.graph, t) and op == attachedTrace:
+      if not canFormAcycle(t) and op == attachedTrace:
         echo "ayclic but has this =trace ", t, " ", theProc.ast
   else:
     when false:
@@ -1405,7 +1343,7 @@ proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp): Rope =
 
 proc genTypeInfoV2Impl(m: BModule, t, origType: PType, name: Rope; info: TLineInfo) =
   var typeName: Rope
-  if t.kind in {tyObject, tyDistinct}:
+  if t.kind == tyObject:
     if incompleteType(t):
       localError(m.config, info, "request for RTTI generation for incomplete object: " &
                  typeToString(t))
@@ -1417,25 +1355,25 @@ proc genTypeInfoV2Impl(m: BModule, t, origType: PType, name: Rope; info: TLineIn
   m.s[cfsData].addf("N_LIB_PRIVATE TNimTypeV2 $1;$n", [name])
   let destroyImpl = genHook(m, t, info, attachedDestructor)
   let traceImpl = genHook(m, t, info, attachedTrace)
+  let disposeImpl = genHook(m, t, info, attachedDispose)
 
   var flags = 0
-  if not canFormAcycle(m.g.graph, t): flags = flags or 1
+  if not canFormAcycle(t): flags = flags or 1
 
-  addf(m.s[cfsTypeInit3], "$1.destructor = (void*)$2; $1.size = sizeof($3); $1.align = NIM_ALIGNOF($3); $1.name = $4;$n; $1.traceImpl = (void*)$5; $1.flags = $6;", [
+  addf(m.s[cfsTypeInit3], "$1.destructor = (void*)$2; $1.size = sizeof($3); $1.align = NIM_ALIGNOF($3); $1.name = $4;$n; $1.traceImpl = (void*)$5; $1.disposeImpl = (void*)$6; $1.flags = $7;", [
     name, destroyImpl, getTypeDesc(m, t), typeName,
-    traceImpl, rope(flags)])
+    traceImpl, disposeImpl, rope(flags)])
 
   if t.kind == tyObject and t.len > 0 and t[0] != nil and optEnableDeepCopy in m.config.globalOptions:
     discard genTypeInfoV1(m, t, info)
 
 proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope =
   let origType = t
-  # distinct types can have their own destructors
-  var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses - {tyDistinct})
+  var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses)
 
   let prefixTI = if m.hcrOn: "(" else: "(&"
 
-  let sig = hashType(origType, m.config)
+  let sig = hashType(origType)
   result = m.typeInfoMarkerV2.getOrDefault(sig)
   if result != nil:
     return prefixTI.rope & result & ")".rope
@@ -1451,10 +1389,11 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope =
   result = "NTIv2$1_" % [rope($sig)]
   m.typeInfoMarkerV2[sig] = result
 
-  let owner = t.skipTypes(typedescPtrs).itemId.module
-  if owner != m.module.position and moduleOpenForCodegen(m.g.graph, FileIndex owner):
+  let owner = t.skipTypes(typedescPtrs).owner.getModule
+  if owner != m.module:
     # make sure the type info is created in the owner module
-    discard genTypeInfoV2(m.g.modules[owner], origType, info)
+    assert m.g.modules[owner.position] != nil
+    discard genTypeInfoV2(m.g.modules[owner.position], origType, info)
     # reference the type info as extern here
     discard cgsym(m, "TNimTypeV2")
     declareNimType(m, "TNimTypeV2", result, owner)
@@ -1465,48 +1404,21 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope =
   result = prefixTI.rope & result & ")".rope
 
 proc openArrayToTuple(m: BModule; t: PType): PType =
-  result = newType(tyTuple, nextTypeId m.idgen, t.owner)
-  let p = newType(tyPtr, nextTypeId m.idgen, t.owner)
-  let a = newType(tyUncheckedArray, nextTypeId m.idgen, t.owner)
+  result = newType(tyTuple, t.owner)
+  let p = newType(tyPtr, t.owner)
+  let a = newType(tyUncheckedArray, t.owner)
   a.add t.lastSon
   p.add a
   result.add p
   result.add getSysType(m.g.graph, t.owner.info, tyInt)
 
-proc typeToC(t: PType): string =
-  ## Just for more readable names, the result doesn't have
-  ## to be unique.
-  let s = typeToString(t)
-  result = newStringOfCap(s.len)
-  for i in 0..<s.len:
-    let c = s[i]
-    case c
-    of 'a'..'z':
-      result.add c
-    of 'A'..'Z':
-      result.add toLowerAscii(c)
-    of ' ':
-      discard
-    of ',':
-      result.add '_'
-    of '.':
-      result.add 'O'
-    of '[', '(', '{':
-      result.add 'L'
-    of ']', ')', '}':
-      result.add 'T'
-    else:
-      # We mangle upper letters and digits too so that there cannot
-      # be clashes with our special meanings
-      result.addInt ord(c)
-
 proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
   let origType = t
   var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses)
 
   let prefixTI = if m.hcrOn: "(" else: "(&"
 
-  let sig = hashType(origType, m.config)
+  let sig = hashType(origType)
   result = m.typeInfoMarker.getOrDefault(sig)
   if result != nil:
     return prefixTI.rope & result & ")".rope
@@ -1520,34 +1432,25 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
     m.typeInfoMarker[sig] = marker.str
     return prefixTI.rope & marker.str & ")".rope
 
-  result = "NTI$1$2_" % [rope(typeToC(t)), rope($sig)]
+  result = "NTI$1_" % [rope($sig)]
   m.typeInfoMarker[sig] = result
 
-  let old = m.g.graph.emittedTypeInfo.getOrDefault($result)
-  if old != FileIndex(0):
-    discard cgsym(m, "TNimType")
-    discard cgsym(m, "TNimNode")
-    declareNimType(m, "TNimType", result, old.int)
-    return prefixTI.rope & result & ")".rope
-
-  var owner = t.skipTypes(typedescPtrs).itemId.module
-  if owner != m.module.position and moduleOpenForCodegen(m.g.graph, FileIndex owner):
+  let owner = t.skipTypes(typedescPtrs).owner.getModule
+  if owner != m.module:
     # make sure the type info is created in the owner module
-    discard genTypeInfoV1(m.g.modules[owner], origType, info)
+    assert m.g.modules[owner.position] != nil
+    discard genTypeInfoV1(m.g.modules[owner.position], origType, info)
     # reference the type info as extern here
     discard cgsym(m, "TNimType")
     discard cgsym(m, "TNimNode")
     declareNimType(m, "TNimType", result, owner)
     return prefixTI.rope & result & ")".rope
-  else:
-    owner = m.module.position.int32
 
   m.g.typeInfoMarker[sig] = (str: result, owner: owner)
-  rememberEmittedTypeInfo(m.g.graph, FileIndex(owner), $result)
 
   case t.kind
   of tyEmpty, tyVoid: result = rope"0"
-  of tyPointer, tyBool, tyChar, tyCstring, tyString, tyInt..tyUInt64, tyVar, tyLent:
+  of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar, tyLent:
     genTypeInfoAuxBase(m, t, t, result, rope"0", info)
   of tyStatic:
     if t.n != nil: result = genTypeInfoV1(m, lastSon t, info)
@@ -1588,11 +1491,10 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
     genTupleInfo(m, x, origType, result, info)
   else: internalError(m.config, "genTypeInfoV1(" & $t.kind & ')')
 
-  var op = getAttachedOp(m.g.graph, t, attachedDeepCopy)
-  if op == nil:
-    op = getAttachedOp(m.g.graph, origType, attachedDeepCopy)
-  if op != nil:
-    genDeepCopyProc(m, op, result)
+  if t.attachedOps[attachedDeepCopy] != nil:
+    genDeepCopyProc(m, t.attachedOps[attachedDeepCopy], result)
+  elif origType.attachedOps[attachedDeepCopy] != nil:
+    genDeepCopyProc(m, origType.attachedOps[attachedDeepCopy], result)
 
   if optTinyRtti in m.config.globalOptions and t.kind == tyObject and sfImportc notin t.sym.flags:
     let v2info = genTypeInfoV2(m, origType, info)
@@ -1603,9 +1505,3 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
 
 proc genTypeSection(m: BModule, n: PNode) =
   discard
-
-proc genTypeInfo*(config: ConfigRef, m: BModule, t: PType; info: TLineInfo): Rope =
-  if optTinyRtti in config.globalOptions:
-    result = genTypeInfoV2(m, t, info)
-  else:
-    result = genTypeInfoV1(m, t, info)

compiler/ccgutils.nim

@@ -10,13 +10,8 @@
 # This module declares some helpers for the C code generator.
 
 import
-  ast, types, hashes, strutils, msgs, wordrecg,
-  platform, trees, options, cgendata
-
-import std/[hashes, strutils]
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
+  ast, hashes, strutils, msgs, wordrecg,
+  platform, trees, options
 
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
   case n.kind
@@ -111,38 +106,3 @@ proc mangle*(name: string): string =
       requiresUnderscore = true
   if requiresUnderscore:
     result.add "_"
-
-proc mapSetType(conf: ConfigRef; typ: PType): TCTypeKind =
-  case int(getSize(conf, typ))
-  of 1: result = ctInt8
-  of 2: result = ctInt16
-  of 4: result = ctInt32
-  of 8: result = ctInt64
-  else: result = ctArray
-
-proc ccgIntroducedPtr*(conf: ConfigRef; s: PSym, retType: PType): bool =
-  var pt = skipTypes(s.typ, typedescInst)
-  assert skResult != s.kind
-
-  if tfByRef in pt.flags: return true
-  elif tfByCopy in pt.flags: return false
-  case pt.kind
-  of tyObject:
-    if s.typ.sym != nil and sfForward in s.typ.sym.flags:
-      # forwarded objects are *always* passed by pointers for consistency!
-      result = true
-    elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
-      result = true           # requested anyway
-    elif (tfFinal in pt.flags) and (pt[0] == nil):
-      result = false          # no need, because no subtyping possible
-    else:
-      result = true           # ordinary objects are always passed by reference,
-                              # otherwise casting doesn't work
-  of tyTuple:
-    result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options)
-  else:
-    result = false
-  # first parameter and return type is 'lent T'? --> use pass by pointer
-  if s.position == 0 and retType != nil and retType.kind == tyLent:
-    result = not (pt.kind in {tyVar, tyArray, tyOpenArray, tyVarargs, tyRef, tyPtr, tyPointer} or
-      pt.kind == tySet and mapSetType(conf, pt) == ctArray)

compiler/cgen.nim

@@ -13,16 +13,18 @@ import
   ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets,
   nversion, nimsets, msgs, bitsets, idents, types,
   ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth,
-  rodutils, renderer, cgendata, aliases,
-  lowerings, tables, sets, ndi, lineinfos, pathutils, transf,
-  injectdestructors, astmsgs, modulepaths
+  rodutils, renderer, cgendata, ccgmerge, aliases,
+  lowerings, tables, sets, ndi, lineinfos, pathutils, transf, enumtostr,
+  injectdestructors
 
 when not defined(leanCompiler):
   import spawn, semparallel
 
 import strutils except `%` # collides with ropes.`%`
 
-from ic / ic import ModuleBackendFlag
+from modulegraphs import ModuleGraph, PPassContext
+from lineinfos import
+  warnGcMem, errXMustBeCompileTime, hintDependency, errGenerated, errCannotOpenFile
 import dynlib
 
 when not declared(dynlib.libCandidates):
@@ -48,13 +50,8 @@ proc addForwardedProc(m: BModule, prc: PSym) =
   m.g.forwardedProcs.add(prc)
 
 proc findPendingModule(m: BModule, s: PSym): BModule =
-  # TODO fixme
-  if m.config.symbolFiles == v2Sf:
-    let ms = s.itemId.module  #getModule(s)
-    result = m.g.modules[ms]
-  else:
-    var ms = getModule(s)
-    result = m.g.modules[ms.position]
+  var ms = getModule(s)
+  result = m.g.modules[ms.position]
 
 proc initLoc(result: var TLoc, k: TLocKind, lode: PNode, s: TStorageLoc) =
   result.k = k
@@ -100,13 +97,10 @@ proc getCFile(m: BModule): AbsoluteFile
 proc getModuleDllPath(m: BModule): Rope =
   let (dir, name, ext) = splitFile(getCFile(m))
   let filename = strutils.`%`(platform.OS[m.g.config.target.targetOS].dllFrmt, [name & ext])
-  result = makeCString(dir.string & "/" & filename)
-
-proc getModuleDllPath(m: BModule, module: int): Rope =
-  result = getModuleDllPath(m.g.modules[module])
+  return makeCString(dir.string & "/" & filename)
 
 proc getModuleDllPath(m: BModule, s: PSym): Rope =
-  result = getModuleDllPath(m.g.modules[s.itemId.module])
+  return getModuleDllPath(findPendingModule(m, s))
 
 import macros
 
@@ -159,11 +153,6 @@ macro ropecg(m: BModule, frmt: static[FormatStr], args: untyped): Rope =
         inc(i)
         result.add newCall(formatValue, resVar, args[num])
         inc(num)
-      of '^':
-        flushStrLit()
-        inc(i)
-        result.add newCall(formatValue, resVar, args[^1])
-        inc(num)
       of '0'..'9':
         var j = 0
         while true:
@@ -260,8 +249,7 @@ proc genCLineDir(r: var Rope, filename: string, line: int; conf: ConfigRef) =
         [rope(makeSingleLineCString(filename)), rope(line)])
 
 proc genCLineDir(r: var Rope, info: TLineInfo; conf: ConfigRef) =
-  if optLineDir in conf.options:
-    genCLineDir(r, toFullPath(conf, info), info.safeLineNm, conf)
+  genCLineDir(r, toFullPath(conf, info), info.safeLineNm, conf)
 
 proc freshLineInfo(p: BProc; info: TLineInfo): bool =
   if p.lastLineInfo.line != info.line or
@@ -275,13 +263,16 @@ proc genLineDir(p: BProc, t: PNode) =
 
   if optEmbedOrigSrc in p.config.globalOptions:
     p.s(cpsStmts).add(~"//" & sourceLine(p.config, t.info) & "\L")
-  genCLineDir(p.s(cpsStmts), t.info, p.config)
+  genCLineDir(p.s(cpsStmts), toFullPath(p.config, t.info), line, p.config)
   if ({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and
       (p.prc == nil or sfPure notin p.prc.flags) and t.info.fileIndex != InvalidFileIdx:
     if freshLineInfo(p, t.info):
       linefmt(p, cpsStmts, "nimln_($1, $2);$n",
               [line, quotedFilename(p.config, t.info)])
 
+proc postStmtActions(p: BProc) {.inline.} =
+  p.s(cpsStmts).add(p.module.injectStmt)
+
 proc accessThreadLocalVar(p: BProc, s: PSym)
 proc emulatedThreadVars(conf: ConfigRef): bool {.inline.}
 proc genProc(m: BModule, prc: PSym)
@@ -308,12 +299,6 @@ proc lenExpr(p: BProc; a: TLoc): Rope =
   else:
     result = "($1 ? $1->$2 : 0)" % [rdLoc(a), lenField(p)]
 
-proc dataFieldAccessor(p: BProc, sym: Rope): Rope =
-  if optSeqDestructors in p.config.globalOptions:
-    result = "(" & sym & ").p"
-  else:
-    result = sym
-
 proc dataField(p: BProc): Rope =
   if optSeqDestructors in p.config.globalOptions:
     result = rope".p->data"
@@ -380,8 +365,7 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: var TLoc,
     else:
       linefmt(p, section, "$1.m_type = $2;$n", [r, genTypeInfoV1(p.module, t, a.lode.info)])
   of frEmbedded:
-    # inheritance in C++ does not allow struct initialization: bug #18410
-    if not p.module.compileToCpp and optTinyRtti in p.config.globalOptions:
+    if optTinyRtti in p.config.globalOptions:
       var tmp: TLoc
       if mode == constructRefObj:
         let objType = t.skipTypes(abstractInst+{tyRef})
@@ -422,12 +406,7 @@ proc resetLoc(p: BProc, loc: var TLoc) =
   if isImportedCppType(typ): return
   if optSeqDestructors in p.config.globalOptions and typ.kind in {tyString, tySequence}:
     assert rdLoc(loc) != nil
-
-    let atyp = skipTypes(loc.t, abstractInst)
-    if atyp.kind in {tyVar, tyLent}:
-      linefmt(p, cpsStmts, "$1->len = 0; $1->p = NIM_NIL;$n", [rdLoc(loc)])
-    else:
-      linefmt(p, cpsStmts, "$1.len = 0; $1.p = NIM_NIL;$n", [rdLoc(loc)])
+    linefmt(p, cpsStmts, "$1.len = 0; $1.p = NIM_NIL;$n", [rdLoc(loc)])
   elif not isComplexValueType(typ):
     if containsGcRef:
       var nilLoc: TLoc
@@ -532,9 +511,6 @@ proc localVarDecl(p: BProc; n: PNode): Rope =
     if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy)
   if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0:
     result.addf("NIM_ALIGN($1) ", [rope(s.alignment)])
-
-  genCLineDir(result, n.info, p.config)
-
   result.add getTypeDesc(p.module, s.typ, skVar)
   if s.constraint.isNil:
     if sfRegister in s.flags: result.add(" register")
@@ -879,19 +855,14 @@ proc closureSetup(p: BProc, prc: PSym) =
             [rdLoc(env.loc), getTypeDesc(p.module, env.typ)])
 
 proc containsResult(n: PNode): bool =
-  result = false
-  case n.kind
-  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkFormalParams:
-    discard
-  of nkSym:
-    if n.sym.kind == skResult:
-      result = true
+  if n.kind == nkSym and n.sym.kind == skResult:
+    result = true
   else:
-    for i in 0..<n.len:
+    for i in 0..<n.safeLen:
       if containsResult(n[i]): return true
 
 const harmless = {nkConstSection, nkTypeSection, nkEmpty, nkCommentStmt, nkTemplateDef,
-                  nkMacroDef, nkMixinStmt, nkBindStmt, nkFormalParams} +
+                  nkMacroDef, nkMixinStmt, nkBindStmt} +
                   declarativeDefs
 
 proc easyResultAsgn(n: PNode): PNode =
@@ -1040,16 +1011,16 @@ proc genProcAux(m: BModule, prc: PSym) =
   var returnStmt: Rope = nil
   assert(prc.ast != nil)
 
-  var procBody = transformBody(m.g.graph, m.idgen, prc, cache = false)
+  var procBody = transformBody(m.g.graph, prc, cache = false)
   if sfInjectDestructors in prc.flags:
-    procBody = injectDestructorCalls(m.g.graph, m.idgen, prc, procBody)
+    procBody = injectDestructorCalls(m.g.graph, prc, procBody)
 
   if sfPure notin prc.flags and prc.typ[0] != nil:
     if resultPos >= prc.ast.len:
       internalError(m.config, prc.info, "proc has no result symbol")
     let resNode = prc.ast[resultPos]
     let res = resNode.sym # get result symbol
-    if not isInvalidReturnType(m.config, prc.typ):
+    if not isInvalidReturnType(m.config, prc.typ[0]):
       if sfNoInit in prc.flags: incl(res.flags, sfNoInit)
       if sfNoInit in prc.flags and p.module.compileToCpp and (let val = easyResultAsgn(procBody); val != nil):
         var decl = localVarDecl(p, resNode)
@@ -1063,7 +1034,7 @@ proc genProcAux(m: BModule, prc: PSym) =
         initLocalVar(p, res, immediateAsgn=false)
       returnStmt = ropecg(p.module, "\treturn $1;$n", [rdLoc(res.loc)])
     else:
-      fillResult(p.config, resNode, prc.typ)
+      fillResult(p.config, resNode)
       assignParam(p, res, prc.typ[0])
       # We simplify 'unsureAsgn(result, nil); unsureAsgn(result, x)'
       # to 'unsureAsgn(result, x)'
@@ -1139,7 +1110,7 @@ proc genProcPrototype(m: BModule, sym: PSym) =
   useHeader(m, sym)
   if lfNoDecl in sym.loc.flags: return
   if lfDynamicLib in sym.loc.flags:
-    if sym.itemId.module != m.module.position and
+    if getModule(sym).id != m.module.id and
         not containsOrIncl(m.declaredThings, sym.id):
       m.s[cfsVars].add(ropecg(m, "$1 $2 $3;$n",
                         [(if isReloadable(m, sym): "static" else: "extern"),
@@ -1172,6 +1143,22 @@ proc genProcNoForward(m: BModule, prc: PSym) =
   if lfNoDecl in prc.loc.flags:
     fillProcLoc(m, prc.ast[namePos])
     genProcPrototype(m, prc)
+  elif prc.typ.callConv == ccInline:
+    # We add inline procs to the calling module to enable C based inlining.
+    # This also means that a check with ``q.declaredThings`` is wrong, we need
+    # a check for ``m.declaredThings``.
+    if not containsOrIncl(m.declaredThings, prc.id):
+      #if prc.loc.k == locNone:
+      # mangle the inline proc based on the module where it is defined -
+      # not on the first module that uses it
+      fillProcLoc(findPendingModule(m, prc), prc.ast[namePos])
+      #elif {sfExportc, sfImportc} * prc.flags == {}:
+      #  # reset name to restore consistency in case of hashing collisions:
+      #  echo "resetting ", prc.id, " by ", m.module.name.s
+      #  prc.loc.r = nil
+      #  prc.loc.r = mangleName(m, prc)
+      genProcPrototype(m, prc)
+      genProcAux(m, prc)
   elif lfDynamicLib in prc.loc.flags:
     var q = findPendingModule(m, prc)
     fillProcLoc(q, prc.ast[namePos])
@@ -1186,24 +1173,6 @@ proc genProcNoForward(m: BModule, prc: PSym) =
             [prc.loc.r, getTypeDesc(q, prc.loc.t), getModuleDllPath(m, q.module)])
     else:
       symInDynamicLibPartial(m, prc)
-  elif prc.typ.callConv == ccInline:
-    # We add inline procs to the calling module to enable C based inlining.
-    # This also means that a check with ``q.declaredThings`` is wrong, we need
-    # a check for ``m.declaredThings``.
-    if not containsOrIncl(m.declaredThings, prc.id):
-      #if prc.loc.k == locNone:
-      # mangle the inline proc based on the module where it is defined -
-      # not on the first module that uses it
-      let m2 = if m.config.symbolFiles != disabledSf: m
-               else: findPendingModule(m, prc)
-      fillProcLoc(m2, prc.ast[namePos])
-      #elif {sfExportc, sfImportc} * prc.flags == {}:
-      #  # reset name to restore consistency in case of hashing collisions:
-      #  echo "resetting ", prc.id, " by ", m.module.name.s
-      #  prc.loc.r = nil
-      #  prc.loc.r = mangleName(m, prc)
-      genProcPrototype(m, prc)
-      genProcAux(m, prc)
   elif sfImportc notin prc.flags:
     var q = findPendingModule(m, prc)
     fillProcLoc(q, prc.ast[namePos])
@@ -1227,16 +1196,48 @@ proc genProcNoForward(m: BModule, prc: PSym) =
     if sfInfixCall notin prc.flags: genProcPrototype(m, prc)
 
 proc requestConstImpl(p: BProc, sym: PSym) =
-  if genConstSetup(p, sym):
-    let m = p.module
-    # declare implementation:
-    var q = findPendingModule(m, sym)
-    if q != nil and not containsOrIncl(q.declaredThings, sym.id):
-      assert q.initProc.module == q
-      genConstDefinition(q, p, sym)
-    # declare header:
-    if q != m and not containsOrIncl(m.declaredThings, sym.id):
-      genConstHeader(m, q, p, sym)
+  var m = p.module
+  useHeader(m, sym)
+  if sym.loc.k == locNone:
+    fillLoc(sym.loc, locData, sym.ast, mangleName(p.module, sym), OnStatic)
+  if m.hcrOn: incl(sym.loc.flags, lfIndirect)
+
+  if lfNoDecl in sym.loc.flags: return
+  # declare implementation:
+  var q = findPendingModule(m, sym)
+  if q != nil and not containsOrIncl(q.declaredThings, sym.id):
+    assert q.initProc.module == q
+    # add a suffix for hcr - will later init the global pointer with this data
+    let actualConstName = if m.hcrOn: sym.loc.r & "_const" else: sym.loc.r
+    q.s[cfsData].addf("N_LIB_PRIVATE NIM_CONST $1 $2 = $3;$n",
+        [getTypeDesc(q, sym.typ), actualConstName,
+        genBracedInit(q.initProc, sym.ast, isConst = true, sym.typ)])
+    if m.hcrOn:
+      # generate the global pointer with the real name
+      q.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r])
+      # register it (but ignore the boolean result of hcrRegisterGlobal)
+      q.initProc.procSec(cpsLocals).addf(
+        "\thcrRegisterGlobal($1, \"$2\", sizeof($3), NULL, (void**)&$2);$n",
+        [getModuleDllPath(q, sym), sym.loc.r, rdLoc(sym.loc)])
+      # always copy over the contents of the actual constant with the _const
+      # suffix ==> this means that the constant is reloadable & updatable!
+      q.initProc.procSec(cpsLocals).add(ropecg(q,
+        "\t#nimCopyMem((void*)$1, (NIM_CONST void*)&$2, sizeof($3));$n",
+        [sym.loc.r, actualConstName, rdLoc(sym.loc)]))
+  # declare header:
+  if q != m and not containsOrIncl(m.declaredThings, sym.id):
+    assert(sym.loc.r != nil)
+    if m.hcrOn:
+      m.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]);
+      m.initProc.procSec(cpsLocals).addf(
+        "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r,
+        getTypeDesc(m, sym.loc.t, skVar), getModuleDllPath(q, sym)])
+    else:
+      let headerDecl = "extern NIM_CONST $1 $2;$n" %
+          [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]
+      m.s[cfsData].add(headerDecl)
+      if sfExportc in sym.flags and p.module.g.generatedHeader != nil:
+        p.module.g.generatedHeader.s[cfsData].add(headerDecl)
 
 proc isActivated(prc: PSym): bool = prc.typ != nil
 
@@ -1266,10 +1267,10 @@ proc genVarPrototype(m: BModule, n: PNode) =
   if sym.owner.id != m.module.id:
     # else we already have the symbol generated!
     assert(sym.loc.r != nil)
-    incl(m.declaredThings, sym.id)
     if sfThread in sym.flags:
       declareThreadVar(m, sym, true)
     else:
+      incl(m.declaredThings, sym.id)
       if sym.kind in {skLet, skVar, skField, skForVar} and sym.alignment > 0:
         m.s[cfsVars].addf "NIM_ALIGN($1) ", [rope(sym.alignment)]
       m.s[cfsVars].add(if m.hcrOn: "static " else: "extern ")
@@ -1294,15 +1295,20 @@ proc addNimDefines(result: var Rope; conf: ConfigRef) {.inline.} =
   if conf.isDefined("nimEmulateOverflowChecks"):
     result.add("#define NIM_EmulateOverflowChecks\L")
 
-proc headerTop(): Rope =
-  result = "/* Generated by Nim Compiler v$1 */$N" % [rope(VersionAsString)]
-
 proc getCopyright(conf: ConfigRef; cfile: Cfile): Rope =
-  result = headerTop()
-  if optCompileOnly notin conf.globalOptions:
-    result.add ("/* Compiled for: $1, $2, $3 */$N" &
-        "/* Command for C compiler:$n   $4 */$N") %
-        [rope(platform.OS[conf.target.targetOS].name),
+  if optCompileOnly in conf.globalOptions:
+    result = ("/* Generated by Nim Compiler v$1 */$N" &
+        "/*   (c) " & copyrightYear & " Andreas Rumpf */$N" &
+        "/* The generated code is subject to the original license. */$N") %
+        [rope(VersionAsString)]
+  else:
+    result = ("/* Generated by Nim Compiler v$1 */$N" &
+        "/*   (c) " & copyrightYear & " Andreas Rumpf */$N" &
+        "/* The generated code is subject to the original license. */$N" &
+        "/* Compiled for: $2, $3, $4 */$N" &
+        "/* Command for C compiler:$n   $5 */$N") %
+        [rope(VersionAsString),
+        rope(platform.OS[conf.target.targetOS].name),
         rope(platform.CPU[conf.target.targetCPU].name),
         rope(extccomp.CC[conf.cCompiler].name),
         rope(getCompileCFileCmd(conf, cfile))]
@@ -1312,25 +1318,23 @@ proc getFileHeader(conf: ConfigRef; cfile: Cfile): Rope =
   if conf.hcrOn: result.add("#define NIM_HOT_CODE_RELOADING\L")
   addNimDefines(result, conf)
 
-proc getSomeNameForModule(conf: ConfigRef, filename: AbsoluteFile): Rope =
-  ## Returns a mangled module name.
-  result.add mangleModuleName(conf, filename).mangle
-
-proc getSomeNameForModule(m: BModule): Rope =
-  ## Returns a mangled module name.
-  assert m.module.kind == skModule
-  assert m.module.owner.kind == skPackage
-  result.add mangleModuleName(m.g.config, m.filename).mangle
+proc getSomeNameForModule(m: PSym): Rope =
+  assert m.kind == skModule
+  assert m.owner.kind == skPackage
+  if {sfSystemModule, sfMainModule} * m.flags == {}:
+    result = m.owner.name.s.mangle.rope
+    result.add "_"
+  result.add m.name.s.mangle
 
 proc getSomeInitName(m: BModule, suffix: string): Rope =
   if not m.hcrOn:
-    result = getSomeNameForModule(m)
+    result = getSomeNameForModule(m.module)
   result.add suffix
 
 proc getInitName(m: BModule): Rope =
   if sfMainModule in m.module.flags:
     # generate constant name for main module, for "easy" debugging.
-    result = rope(m.config.nimMainPrefix) & rope"NimMainModule"
+    result = rope"NimMainModule"
   else:
     result = getSomeInitName(m, "Init000")
 
@@ -1363,75 +1367,51 @@ proc genMainProc(m: BModule) =
     preMainCode.add("\tinitStackBottomWith_actual((void *)&inner);\L")
     preMainCode.add("\t(*inner)();\L")
   else:
-    preMainCode.add("\t$1PreMain();\L" % [rope m.config.nimMainPrefix])
-
-  var posixCmdLine: Rope
-  if optNoMain notin m.config.globalOptions:
-    posixCmdLine.add "\tN_LIB_PRIVATE int cmdCount;\L"
-    posixCmdLine.add "\tN_LIB_PRIVATE char** cmdLine;\L"
-    posixCmdLine.add "\tN_LIB_PRIVATE char** gEnv;\L"
+    preMainCode.add("\tPreMain();\L")
 
   const
+    # not a big deal if we always compile these 3 global vars... makes the HCR code easier
+    PosixCmdLine =
+      "N_LIB_PRIVATE int cmdCount;$N" &
+      "N_LIB_PRIVATE char** cmdLine;$N" &
+      "N_LIB_PRIVATE char** gEnv;$N"
+
     # The use of a volatile function pointer to call Pre/NimMainInner
     # prevents inlining of the NimMainInner function and dependent
     # functions, which might otherwise merge their stack frames.
-
-    PreMainVolatileBody =
-      "\tvoid (*volatile inner)(void);$N" &
-      "\tinner = $3PreMainInner;$N" &
-      "$1" &
-      "\t(*inner)();$N"
-
-    PreMainNonVolatileBody =
-      "$1" &
-      "\t$3PreMainInner();$N"
-
-    PreMainBodyStart = "$N" &
-      "N_LIB_PRIVATE void $3PreMainInner(void) {$N" &
+    PreMainBody = "$N" &
+      "N_LIB_PRIVATE void PreMainInner(void) {$N" &
       "$2" &
       "}$N$N" &
-      "$4" &
-      "N_LIB_PRIVATE void $3PreMain(void) {$N"
-
-    PreMainBodyEnd =
+      PosixCmdLine &
+      "N_LIB_PRIVATE void PreMain(void) {$N" &
+      "\tvoid (*volatile inner)(void);$N" &
+      "\tinner = PreMainInner;$N" &
+      "$1" &
+      "\t(*inner)();$N" &
       "}$N$N"
 
     MainProcs =
-      "\t$^NimMain();$N"
+      "\tNimMain();$N"
 
     MainProcsWithResult =
       MainProcs & ("\treturn $1nim_program_result;$N")
 
-    NimMainInner = "N_LIB_PRIVATE N_CDECL(void, $5NimMainInner)(void) {$N" &
+    NimMainInner = "N_LIB_PRIVATE N_CDECL(void, NimMainInner)(void) {$N" &
         "$1" &
       "}$N$N"
 
-    NimMainVolatileBody =
-      "\tvoid (*volatile inner)(void);$N" &
-      "$4" &
-      "\tinner = $5NimMainInner;$N" &
-      "$2" &
-      "\t(*inner)();$N"
-
-    NimMainNonVolatileBody =
-      "$4" &
-      "$2" &
-      "\t$5NimMainInner();$N"
-
-    NimMainProcStart =
-      "N_CDECL(void, $5NimMain)(void) {$N"
-
-    NimMainProcEnd =
+    NimMainProc =
+      "N_CDECL(void, NimMain)(void) {$N" &
+        "\tvoid (*volatile inner)(void);$N" &
+        "$4" &
+        "\tinner = NimMainInner;$N" &
+        "$2" &
+        "\t(*inner)();$N" &
       "}$N$N"
 
-    NimMainProc = NimMainProcStart & NimMainVolatileBody & NimMainProcEnd
-
-    NimSlimMainProc = NimMainProcStart & NimMainNonVolatileBody & NimMainProcEnd
-
     NimMainBody = NimMainInner & NimMainProc
 
-    NimSlimMainBody = NimMainInner & NimSlimMainProc
-
     PosixCMain =
       "int main(int argc, char** args, char** env) {$N" &
         "\tcmdLine = args;$N" &
@@ -1470,7 +1450,7 @@ proc genMainProc(m: BModule) =
 
     GenodeNimMain =
       "extern Genode::Env *nim_runtime_env;$N" &
-      "extern \"C\" void nim_component_construct(Genode::Env*);$N$N" &
+      "extern void nim_component_construct(Genode::Env*);$N$N" &
       NimMainBody
 
     ComponentConstruct =
@@ -1492,45 +1472,38 @@ proc genMainProc(m: BModule) =
     m.includeHeader("<libc/component.h>")
 
   let initStackBottomCall =
-    if m.config.target.targetOS == osStandalone or m.config.selectedGC in {gcNone, gcArc, gcOrc}: "".rope
+    if m.config.target.targetOS == osStandalone or m.config.selectedGC == gcNone: "".rope
     else: ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", [])
   inc(m.labels)
-  if m.config.selectedGC notin {gcNone, gcArc, gcOrc}:
-    appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit, m.config.nimMainPrefix, posixCmdLine])
-  else:
-    appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainNonVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit, m.config.nimMainPrefix, posixCmdLine])
+  appcg(m, m.s[cfsProcs], PreMainBody, [m.g.mainDatInit, m.g.otherModsInit])
 
   if m.config.target.targetOS == osWindows and
       m.config.globalOptions * {optGenGuiApp, optGenDynLib} != {}:
     if optGenGuiApp in m.config.globalOptions:
       const nimMain = WinNimMain
       appcg(m, m.s[cfsProcs], nimMain,
-        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix])
+        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode])
     else:
       const nimMain = WinNimDllMain
       appcg(m, m.s[cfsProcs], nimMain,
-        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix])
+        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode])
   elif m.config.target.targetOS == osGenode:
     const nimMain = GenodeNimMain
     appcg(m, m.s[cfsProcs], nimMain,
-        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix])
+        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode])
   elif optGenDynLib in m.config.globalOptions:
     const nimMain = PosixNimDllMain
     appcg(m, m.s[cfsProcs], nimMain,
-        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix])
+        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode])
   elif m.config.target.targetOS == osStandalone:
     const nimMain = NimMainBody
     appcg(m, m.s[cfsProcs], nimMain,
-        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix])
+        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode])
   else:
-    if m.config.selectedGC notin {gcNone, gcArc, gcOrc}:
-      const nimMain = NimMainBody
-      appcg(m, m.s[cfsProcs], nimMain,
-          [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix])
-    else:
-      const nimMain = NimSlimMainBody
-      appcg(m, m.s[cfsProcs], nimMain,
-          [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix])
+    const nimMain = NimMainBody
+    appcg(m, m.s[cfsProcs], nimMain,
+        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode])
+
 
   if optNoMain notin m.config.globalOptions:
     if m.config.cppCustomNamespace.len > 0:
@@ -1540,53 +1513,27 @@ proc genMainProc(m: BModule) =
         m.config.globalOptions * {optGenGuiApp, optGenDynLib} != {}:
       if optGenGuiApp in m.config.globalOptions:
         const otherMain = WinCMain
-        appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: "", m.config.nimMainPrefix])
+        appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: ""])
       else:
         const otherMain = WinCDllMain
-        appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix])
+        appcg(m, m.s[cfsProcs], otherMain, [])
     elif m.config.target.targetOS == osGenode:
       const otherMain = ComponentConstruct
-      appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix])
+      appcg(m, m.s[cfsProcs], otherMain, [])
     elif optGenDynLib in m.config.globalOptions:
       const otherMain = PosixCDllMain
-      appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix])
+      appcg(m, m.s[cfsProcs], otherMain, [])
     elif m.config.target.targetOS == osStandalone:
       const otherMain = StandaloneCMain
-      appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix])
+      appcg(m, m.s[cfsProcs], otherMain, [])
     else:
       const otherMain = PosixCMain
-      appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: "", m.config.nimMainPrefix])
+      appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: ""])
+
 
     if m.config.cppCustomNamespace.len > 0:
       m.s[cfsProcs].add openNamespaceNim(m.config.cppCustomNamespace)
 
-proc registerInitProcs*(g: BModuleList; m: PSym; flags: set[ModuleBackendFlag]) =
-  ## Called from the IC backend.
-  if HasDatInitProc in flags:
-    let datInit = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "DatInit000"
-    g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit])
-    g.mainDatInit.addf("\t$1();$N", [datInit])
-  if HasModuleInitProc in flags:
-    let init = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "Init000"
-    g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init])
-    let initCall = "\t$1();$N" % [init]
-    if sfMainModule in m.flags:
-      g.mainModInit.add(initCall)
-    elif sfSystemModule in m.flags:
-      g.mainDatInit.add(initCall) # systemInit must called right after systemDatInit if any
-    else:
-      g.otherModsInit.add(initCall)
-
-proc whichInitProcs*(m: BModule): set[ModuleBackendFlag] =
-  # called from IC.
-  result = {}
-  if m.hcrOn or m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0:
-    result.incl HasModuleInitProc
-  for i in cfsTypeInit1..cfsDynLibInit:
-    if m.s[i].len != 0:
-      result.incl HasDatInitProc
-      break
-
 proc registerModuleToMain(g: BModuleList; m: BModule) =
   let
     init = m.getInitName
@@ -1604,7 +1551,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) =
     hcrModuleMeta.addf("\t\"\"};$n", [])
     hcrModuleMeta.addf("$nN_LIB_EXPORT N_NIMCALL(void**, HcrGetImportedModules)() { return (void**)hcr_module_list; }$n", [])
     hcrModuleMeta.addf("$nN_LIB_EXPORT N_NIMCALL(char*, HcrGetSigHash)() { return \"$1\"; }$n$n",
-                          [($sigHash(m.module, m.config)).rope])
+                          [($sigHash(m.module)).rope])
     if sfMainModule in m.module.flags:
       g.mainModProcs.add(hcrModuleMeta)
       g.mainModProcs.addf("static void* hcr_handle;$N", [])
@@ -1621,9 +1568,6 @@ proc registerModuleToMain(g: BModuleList; m: BModule) =
       # nasty nasty hack to get the command line functionality working with HCR
       # register the 2 variables on behalf of the os module which might not even
       # be loaded (in which case it will get collected but that is not a problem)
-      # EDIT: indeed, this hack, in combination with another un-necessary one
-      # (`makeCString` was doing line wrap of string litterals) was root cause for
-      # bug #16265.
       let osModulePath = ($systemModulePath).replace("stdlib_system", "stdlib_os").rope
       g.mainDatInit.addf("\thcrAddModule($1);\n", [osModulePath])
       g.mainDatInit.add("\tint* cmd_count;\n")
@@ -1675,13 +1619,14 @@ proc genDatInitCode(m: BModule) =
   for i in cfsTypeInit1..cfsDynLibInit:
     if m.s[i].len != 0:
       moduleDatInitRequired = true
+      prc.add(genSectionStart(i, m.config))
       prc.add(m.s[i])
+      prc.add(genSectionEnd(i, m.config))
 
   prc.addf("}$N$N", [])
 
   if moduleDatInitRequired:
     m.s[cfsDatInitProc].add(prc)
-    #rememberFlag(m.g.graph, m.module, HasDatInitProc)
 
 # Very similar to the contents of symInDynamicLib - basically only the
 # things needed for the hot code reloading runtime procs to be loaded
@@ -1734,7 +1679,9 @@ proc genInitCode(m: BModule) =
     if m.thing.s(section).len > 0:
       moduleInitRequired = true
       if addHcrGuards: prc.add("\tif (nim_hcr_do_init_) {\n\n")
+      prc.add(genSectionStart(section, m.config))
       prc.add(m.thing.s(section))
+      prc.add(genSectionEnd(section, m.config))
       if addHcrGuards: prc.add("\n\t} // nim_hcr_do_init_\n")
 
   if m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0:
@@ -1812,7 +1759,6 @@ proc genInitCode(m: BModule) =
 
   if moduleInitRequired or sfMainModule in m.module.flags:
     m.s[cfsInitProc].add(prc)
-    #rememberFlag(m.g.graph, m.module, HasModuleInitProc)
 
   genDatInitCode(m)
 
@@ -1827,21 +1773,28 @@ proc genModule(m: BModule, cfile: Cfile): Rope =
   var moduleIsEmpty = true
 
   result = getFileHeader(m.config, cfile)
+  result.add(genMergeInfo(m))
 
   generateThreadLocalStorage(m)
   generateHeaders(m)
+  result.add(genSectionStart(cfsHeaders, m.config))
   result.add(m.s[cfsHeaders])
   if m.config.cppCustomNamespace.len > 0:
     result.add openNamespaceNim(m.config.cppCustomNamespace)
+  result.add(genSectionEnd(cfsHeaders, m.config))
+  result.add(genSectionStart(cfsFrameDefines, m.config))
   if m.s[cfsFrameDefines].len > 0:
     result.add(m.s[cfsFrameDefines])
   else:
     result.add("#define nimfr_(x, y)\n#define nimln_(x, y)\n")
+  result.add(genSectionEnd(cfsFrameDefines, m.config))
 
   for i in cfsForwardTypes..cfsProcs:
     if m.s[i].len > 0:
       moduleIsEmpty = false
+      result.add(genSectionStart(i, m.config))
       result.add(m.s[i])
+      result.add(genSectionEnd(i, m.config))
 
   if m.s[cfsInitProc].len > 0:
     moduleIsEmpty = false
@@ -1895,7 +1848,7 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: AbsoluteFile): BModule
 proc rawNewModule(g: BModuleList; module: PSym; conf: ConfigRef): BModule =
   result = rawNewModule(g, module, AbsoluteFile toFullPath(conf, module.position.FileIndex))
 
-proc newModule*(g: BModuleList; module: PSym; conf: ConfigRef): BModule =
+proc newModule(g: BModuleList; module: PSym; conf: ConfigRef): BModule =
   # we should create only one cgen module for each module sym
   result = rawNewModule(g, module, conf)
   if module.position >= g.modules.len:
@@ -1911,10 +1864,9 @@ template injectG() {.dirty.} =
 when not defined(nimHasSinkInference):
   {.pragma: nosinks.}
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext {.nosinks.} =
   injectG()
   result = newModule(g, module, graph.config)
-  result.idgen = idgen
   if optGenIndex in graph.config.globalOptions and g.generatedHeader == nil:
     let f = if graph.config.headerFile.len > 0: AbsoluteFile graph.config.headerFile
             else: graph.config.projectFull
@@ -1923,7 +1875,11 @@ proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext
     incl g.generatedHeader.flags, isHeaderFile
 
 proc writeHeader(m: BModule) =
-  var result = headerTop()
+  var result = ("/* Generated by Nim Compiler v$1 */$N" &
+        "/*   (c) 2017 Andreas Rumpf */$N" &
+        "/* The generated code is subject to the original license. */$N") %
+        [rope(VersionAsString)]
+
   var guard = "__$1__" % [m.filename.splitFile.name.rope]
   result.addf("#ifndef $1$n#define $1$n", [guard])
   addNimDefines(result, m.config)
@@ -1931,13 +1887,15 @@ proc writeHeader(m: BModule) =
 
   generateThreadLocalStorage(m)
   for i in cfsHeaders..cfsProcs:
+    result.add(genSectionStart(i, m.config))
     result.add(m.s[i])
+    result.add(genSectionEnd(i, m.config))
     if m.config.cppCustomNamespace.len > 0 and i == cfsHeaders: result.add openNamespaceNim(m.config.cppCustomNamespace)
   result.add(m.s[cfsInitProc])
 
   if optGenDynLib in m.config.globalOptions:
     result.add("N_LIB_IMPORT ")
-  result.addf("N_CDECL(void, $1NimMain)(void);$n", [rope m.config.nimMainPrefix])
+  result.addf("N_CDECL(void, NimMain)(void);$n", [])
   if m.config.cppCustomNamespace.len > 0: result.add closeNamespaceNim()
   result.addf("#endif /* $1 */$n", [guard])
   if not writeRope(result, m.filename):
@@ -1948,7 +1906,7 @@ proc getCFile(m: BModule): AbsoluteFile =
       if m.compileToCpp: ".nim.cpp"
       elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m"
       else: ".nim.c"
-  result = changeFileExt(completeCfilePath(m.config, mangleModuleName(m.config, m.cfilename).AbsoluteFile), ext)
+  result = changeFileExt(completeCfilePath(m.config, withPackageName(m.config, m.cfilename)), ext)
 
 when false:
   proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext =
@@ -1976,27 +1934,25 @@ proc addHcrInitGuards(p: BProc, n: PNode, inInitGuard: var bool) =
 
     genStmts(p, n)
 
-proc genTopLevelStmt*(m: BModule; n: PNode) =
-  ## Also called from `ic/cbackend.nim`.
-  if passes.skipCodegen(m.config, n): return
+proc myProcess(b: PPassContext, n: PNode): PNode =
+  result = n
+  if b == nil: return
+  var m = BModule(b)
+  if passes.skipCodegen(m.config, n) or
+      not moduleHasChanged(m.g.graph, m.module):
+    return
   m.initProc.options = initProcOptions(m)
   #softRnl = if optLineDir in m.config.options: noRnl else: rnl
   # XXX replicate this logic!
-  var transformedN = transformStmt(m.g.graph, m.idgen, m.module, n)
+  var transformedN = transformStmt(m.g.graph, m.module, n)
   if sfInjectDestructors in m.module.flags:
-    transformedN = injectDestructorCalls(m.g.graph, m.idgen, m.module, transformedN)
+    transformedN = injectDestructorCalls(m.g.graph, m.module, transformedN)
 
   if m.hcrOn:
     addHcrInitGuards(m.initProc, transformedN, m.inHcrInitGuard)
   else:
     genProcBody(m.initProc, transformedN)
 
-proc myProcess(b: PPassContext, n: PNode): PNode =
-  result = n
-  if b != nil:
-    var m = BModule(b)
-    genTopLevelStmt(m, n)
-
 proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
   if optForceFullMake notin m.config.globalOptions:
     if not moduleHasChanged(m.g.graph, m.module):
@@ -2030,26 +1986,46 @@ proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
 proc writeModule(m: BModule, pending: bool) =
   template onExit() = close(m.ndi, m.config)
   let cfile = getCFile(m)
-  if moduleHasChanged(m.g.graph, m.module):
+  if true or optForceFullMake in m.config.globalOptions:
+    if moduleHasChanged(m.g.graph, m.module):
+      genInitCode(m)
+      finishTypeDescriptions(m)
+      if sfMainModule in m.module.flags:
+        # generate main file:
+        genMainProc(m)
+        m.s[cfsProcHeaders].add(m.g.mainModProcs)
+        generateThreadVarsSize(m)
+
+    var cf = Cfile(nimname: m.module.name.s, cname: cfile,
+                   obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
+    var code = genModule(m, cf)
+    if code != nil or m.config.symbolFiles != disabledSf:
+      when hasTinyCBackend:
+        if m.config.cmd == cmdRun:
+          tccgen.compileCCode($code, m.config)
+          onExit()
+          return
+
+      if not shouldRecompile(m, code, cf): cf.flags = {CfileFlag.Cached}
+      addFileToCompile(m.config, cf)
+  elif pending and mergeRequired(m) and sfMainModule notin m.module.flags:
+    let cf = Cfile(nimname: m.module.name.s, cname: cfile,
+                   obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
+    mergeFiles(cfile, m)
     genInitCode(m)
     finishTypeDescriptions(m)
-    if sfMainModule in m.module.flags:
-      # generate main file:
-      genMainProc(m)
-      m.s[cfsProcHeaders].add(m.g.mainModProcs)
-      generateThreadVarsSize(m)
-
-  var cf = Cfile(nimname: m.module.name.s, cname: cfile,
-                  obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
-  var code = genModule(m, cf)
-  if code != nil or m.config.symbolFiles != disabledSf:
-    when hasTinyCBackend:
-      if m.config.cmd == cmdTcc:
-        tccgen.compileCCode($code, m.config)
-        onExit()
-        return
-
-    if not shouldRecompile(m, code, cf): cf.flags = {CfileFlag.Cached}
+    var code = genModule(m, cf)
+    if code != nil:
+      if not writeRope(code, cfile):
+        rawMessage(m.config, errCannotOpenFile, cfile.string)
+      addFileToCompile(m.config, cf)
+  else:
+    # Consider: first compilation compiles ``system.nim`` and produces
+    # ``system.c`` but then compilation fails due to an error. This means
+    # that ``system.o`` is missing, so we need to call the C compiler for it:
+    var cf = Cfile(nimname: m.module.name.s, cname: cfile,
+                   obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
+    if not fileExists(cf.obj): cf.flags = {CfileFlag.Cached}
     addFileToCompile(m.config, cf)
   onExit()
 
@@ -2057,13 +2033,26 @@ proc updateCachedModule(m: BModule) =
   let cfile = getCFile(m)
   var cf = Cfile(nimname: m.module.name.s, cname: cfile,
                  obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
-  if sfMainModule notin m.module.flags:
-    genMainProc(m)
-  cf.flags = {CfileFlag.Cached}
-  addFileToCompile(m.config, cf)
 
-proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
-  ## Also called from IC.
+  if mergeRequired(m) and sfMainModule notin m.module.flags:
+    mergeFiles(cfile, m)
+    genInitCode(m)
+    finishTypeDescriptions(m)
+    var code = genModule(m, cf)
+    if code != nil:
+      if not writeRope(code, cfile):
+        rawMessage(m.config, errCannotOpenFile, cfile.string)
+      addFileToCompile(m.config, cf)
+  else:
+    if sfMainModule notin m.module.flags:
+      genMainProc(m)
+    cf.flags = {CfileFlag.Cached}
+    addFileToCompile(m.config, cf)
+
+proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
+  result = n
+  if b == nil: return
+  var m = BModule(b)
   if sfMainModule in m.module.flags:
     # phase ordering problem here: We need to announce this
     # dependency to 'nimTestErrorFlag' before system.c has been written to disk.
@@ -2101,7 +2090,7 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
         discard cgsym(m, "rawWrite")
 
       # raise dependencies on behalf of genMainProc
-      if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcOrc}:
+      if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone:
         discard cgsym(m, "initStackBottomWith")
       if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
         discard cgsym(m, "initThreadVarsEmulation")
@@ -2114,12 +2103,6 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
   let mm = m
   m.g.modulesClosed.add mm
 
-
-proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
-  result = n
-  if b == nil: return
-  finalCodegenActions(graph, BModule(b), n)
-
 proc genForwardedProcs(g: BModuleList) =
   # Forward declared proc:s lack bodies when first encountered, so they're given
   # a second pass here
@@ -2127,7 +2110,8 @@ proc genForwardedProcs(g: BModuleList) =
   while g.forwardedProcs.len > 0:
     let
       prc = g.forwardedProcs.pop()
-      m = g.modules[prc.itemId.module]
+      ms = getModule(prc)
+      m = g.modules[ms.position]
     if sfForward in prc.flags:
       internalError(m.config, prc.info, "still forwarded: " & prc.name.s)
 

compiler/cgendata.nim

@@ -102,7 +102,7 @@ type
 
   TTypeSeq* = seq[PType]
   TypeCache* = Table[SigHash, Rope]
-  TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: int32]]
+  TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: PSym]]
 
   CodegenFlag* = enum
     preventStackTrace,  # true if stack traces need to be prevented
@@ -112,8 +112,7 @@ type
     isHeaderFile,       # C source file is the header file
     includesStringh,    # C source file already includes ``<string.h>``
     objHasKidsValid     # whether we can rely on tfObjHasKids
-    useAliveDataFromDce # use the `alive: IntSet` field instead of
-                        # computing alive data on our own.
+
 
   BModuleList* = ref object of RootObj
     mainModProcs*, mainModInit*, otherModsInit*, mainDatInit*: Rope
@@ -138,7 +137,6 @@ type
                             # unconditionally...
                             # nimtvDeps is VERY hard to cache because it's
                             # not a list of IDs nor can it be made to be one.
-    mangledPrcs*: HashSet[string]
 
   TCGen = object of PPassContext # represents a C source file
     s*: TCFileSections        # sections of the C file
@@ -156,7 +154,6 @@ type
     forwTypeCache*: TypeCache # cache for forward declarations of types
     declaredThings*: IntSet   # things we have declared in this .c file
     declaredProtos*: IntSet   # prototypes we have declared in this .c file
-    alive*: IntSet            # symbol IDs of alive data as computed by `dce.nim`
     headerFiles*: seq[string] # needed headers to include
     typeInfoMarker*: TypeCache # needed for generating type information
     typeInfoMarkerV2*: TypeCache
@@ -171,6 +168,7 @@ type
     labels*: Natural          # for generating unique module-scope names
     extensionLoaders*: array['0'..'9', Rope] # special procs for the
                                              # OpenGL wrapper
+    injectStmt*: Rope
     sigConflicts*: CountTable[SigHash]
     g*: BModuleList
     ndi*: NdiFile
@@ -202,7 +200,7 @@ proc newProc*(prc: PSym, module: BModule): BProc =
   result.sigConflicts = initCountTable[string]()
 
 proc newModuleList*(g: ModuleGraph): BModuleList =
-  BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: int32]](),
+  BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: PSym]](),
     config: g.config, graph: g, nimtvDeclared: initIntSet())
 
 iterator cgenModules*(g: BModuleList): BModule =

compiler/cgmeth.nim

@@ -46,7 +46,6 @@ proc methodCall*(n: PNode; conf: ConfigRef): PNode =
   # replace ordinary method by dispatcher method:
   let disp = getDispatcher(result[0].sym)
   if disp != nil:
-    result[0].typ = disp.typ
     result[0].sym = disp
     # change the arguments to up/downcasts to fit the dispatcher's parameters:
     for i in 1..<result.len:
@@ -68,7 +67,7 @@ proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
     while true:
       aa = skipTypes(aa, {tyGenericInst, tyAlias})
       bb = skipTypes(bb, {tyGenericInst, tyAlias})
-      if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef, tyLent, tySink}:
+      if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef, tyLent}:
         aa = aa.lastSon
         bb = bb.lastSon
       else:
@@ -108,14 +107,11 @@ proc attachDispatcher(s: PSym, dispatcher: PNode) =
       s.ast[resultPos] = newNodeI(nkEmpty, s.info)
     s.ast[dispatcherPos] = dispatcher
 
-proc createDispatcher(s: PSym; g: ModuleGraph; idgen: IdGenerator): PSym =
-  var disp = copySym(s, nextSymId(idgen))
+proc createDispatcher(s: PSym): PSym =
+  var disp = copySym(s)
   incl(disp.flags, sfDispatcher)
   excl(disp.flags, sfExported)
-  let old = disp.typ
-  disp.typ = copyType(disp.typ, nextTypeId(idgen), disp.typ.owner)
-  copyTypeProps(g, idgen.module, disp.typ, old)
-
+  disp.typ = copyType(disp.typ, disp.typ.owner, false)
   # we can't inline the dispatcher itself (for now):
   if disp.typ.callConv == ccInline: disp.typ.callConv = ccNimCall
   disp.ast = copyTree(s.ast)
@@ -123,7 +119,7 @@ proc createDispatcher(s: PSym; g: ModuleGraph; idgen: IdGenerator): PSym =
   disp.loc.r = nil
   if s.typ[0] != nil:
     if disp.ast.len > resultPos:
-      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, nextSymId(idgen))
+      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym)
     else:
       # We've encountered a method prototype without a filled-in
       # resultPos slot. We put a placeholder in there that will
@@ -161,7 +157,7 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
       if disp.typ.lockLevel < meth.typ.lockLevel:
         disp.typ.lockLevel = meth.typ.lockLevel
 
-proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
+proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
   var witness: PSym
   for i in 0..<g.methods.len:
     let disp = g.methods[i].dispatcher
@@ -181,8 +177,10 @@ proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
     of Invalid:
       if witness.isNil: witness = g.methods[i].methods[0]
   # create a new dispatcher:
-  g.methods.add((methods: @[s], dispatcher: createDispatcher(s, g, idgen)))
+  g.methods.add((methods: @[s], dispatcher: createDispatcher(s)))
   #echo "adding ", s.info
+  #if fromCache:
+  #  internalError(s.info, "no method dispatcher found")
   if witness != nil:
     localError(g.config, s.info, "invalid declaration order; cannot attach '" & s.name.s &
                        "' to method defined here: " & g.config$witness.info)

compiler/closureiters.nim

@@ -156,7 +156,6 @@ type
     curExcHandlingState: int # Negative for except, positive for finally
     nearestFinally: int # Index of the nearest finally block. For try/except it
                     # is their finally. For finally it is parent finally. Otherwise -1
-    idgen: IdGenerator
 
 const
   nkSkip = {nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
@@ -180,7 +179,7 @@ proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
   ctx.newStateAssgn(newIntTypeNode(stateNo, ctx.g.getSysType(TLineInfo(), tyInt)))
 
 proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
-  result = newSym(skVar, getIdent(ctx.g.cache, name), nextSymId(ctx.idgen), ctx.fn, ctx.fn.info)
+  result = newSym(skVar, getIdent(ctx.g.cache, name), ctx.fn, ctx.fn.info)
   result.typ = typ
   assert(not typ.isNil)
 
@@ -193,7 +192,7 @@ proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
   else:
     let envParam = getEnvParam(ctx.fn)
     # let obj = envParam.typ.lastSon
-    result = addUniqueField(envParam.typ.lastSon, result, ctx.g.cache, ctx.idgen)
+    result = addUniqueField(envParam.typ.lastSon, result, ctx.g.cache)
 
 proc newEnvVarAccess(ctx: Ctx, s: PSym): PNode =
   if ctx.stateVarSym.isNil:
@@ -417,11 +416,8 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
 
 
 proc newEnvVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode =
-  if isEmptyType(v.typ):
-    result = v
-  else:
-    result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
-    result.info = v.info
+  result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
+  result.info = v.info
 
 proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) =
   if input.kind == nkStmtListExpr:
@@ -822,13 +818,10 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   let retStmt =
     if ctx.nearestFinally == 0:
       # last finally, we can return
-      let retValue = if ctx.fn.typ[0].isNil:
-                   ctx.g.emptyNode
-                 else:
-                   newTree(nkFastAsgn,
-                           newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen), info),
-                           ctx.newTmpResultAccess())
-      newTree(nkReturnStmt, retValue)
+      let asgn = newTree(nkFastAsgn,
+        newSymNode(getClosureIterResult(ctx.g, ctx.fn), info),
+        ctx.newTmpResultAccess())
+      newTree(nkReturnStmt, asgn)
     else:
       # bubble up to next finally
       newTree(nkGotoState, ctx.g.newIntLit(info, ctx.nearestFinally))
@@ -852,9 +845,7 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
   case n.kind
   of nkReturnStmt:
     # We're somewhere in try, transform to finally unrolling
-    if ctx.nearestFinally == 0:
-      # return is within the finally
-      return
+    assert(ctx.nearestFinally != 0)
 
     result = newNodeI(nkStmtList, n.info)
 
@@ -1078,7 +1069,7 @@ proc transformStateAssignments(ctx: var Ctx, n: PNode): PNode =
       if n[0][0].kind != nkEmpty:
         var a = newNodeI(nkAsgn, n[0][0].info)
         var retVal = n[0][0] #liftCapturedVars(n[0], owner, d, c)
-        a.add newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen))
+        a.add newSymNode(getClosureIterResult(ctx.g, ctx.fn))
         a.add retVal
         retStmt.add(a)
       else:
@@ -1150,11 +1141,11 @@ proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
     for i in 0..<n.len:
       n[i] = ctx.skipThroughEmptyStates(n[i])
 
-proc newArrayType(g: ModuleGraph; n: int, t: PType; idgen: IdGenerator; owner: PSym): PType =
-  result = newType(tyArray, nextTypeId(idgen), owner)
+proc newArrayType(g: ModuleGraph; n: int, t: PType, owner: PSym): PType =
+  result = newType(tyArray, owner)
 
-  let rng = newType(tyRange, nextTypeId(idgen), owner)
-  rng.n = newTree(nkRange, g.newIntLit(owner.info, 0), g.newIntLit(owner.info, n - 1))
+  let rng = newType(tyRange, owner)
+  rng.n = newTree(nkRange, g.newIntLit(owner.info, 0), g.newIntLit(owner.info, n))
   rng.rawAddSon(t)
 
   result.rawAddSon(rng)
@@ -1162,7 +1153,7 @@ proc newArrayType(g: ModuleGraph; n: int, t: PType; idgen: IdGenerator; owner: P
 
 proc createExceptionTable(ctx: var Ctx): PNode {.inline.} =
   result = newNodeI(nkBracket, ctx.fn.info)
-  result.typ = ctx.g.newArrayType(ctx.exceptionTable.len, ctx.g.getSysType(ctx.fn.info, tyInt16), ctx.idgen, ctx.fn)
+  result.typ = ctx.g.newArrayType(ctx.exceptionTable.len, ctx.g.getSysType(ctx.fn.info, tyInt16), ctx.fn)
 
   for i in ctx.exceptionTable:
     let elem = newIntNode(nkIntLit, i)
@@ -1323,12 +1314,10 @@ type
     finallys: seq[PNode]
     config: ConfigRef
     blocks: seq[(PNode, int)]
-    idgen: IdGenerator
   FreshVarsContext = object
     tab: Table[int, PSym]
     config: ConfigRef
     info: TLineInfo
-    idgen: IdGenerator
 
 proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
   case n.kind
@@ -1347,7 +1336,7 @@ proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
         let idefs = copyNode(it)
         for v in 0..it.len-3:
           if it[v].kind == nkSym:
-            let x = copySym(it[v].sym, nextSymId(c.idgen))
+            let x = copySym(it[v].sym)
             c.tab[it[v].sym.id] = x
             idefs.add newSymNode(x)
           else:
@@ -1371,24 +1360,20 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode =
   # detect: 'finally: raises X' which is currently not supported. We produce
   # an error for this case for now. All this will be done properly with Yuriy's
   # patch.
-
   result = n
   case n.kind
   of nkTryStmt:
     let f = n.lastSon
-    var didAddSomething = false
     if f.kind == nkFinally:
       c.finallys.add f.lastSon
-      didAddSomething = true
 
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
 
-    if didAddSomething:
+    if f.kind == nkFinally:
       discard c.finallys.pop()
 
   of nkWhileStmt, nkBlockStmt:
-    if n.hasYields == false: return n
     c.blocks.add((n, c.finallys.len))
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
@@ -1411,29 +1396,27 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode =
       if fin >= 0:
         result = newNodeI(nkStmtList, n.info)
         for i in countdown(c.finallys.high, fin):
-          var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info, idgen: c.idgen)
-          result.add freshVars(copyTree(c.finallys[i]), vars)
-          c.idgen = vars.idgen
+          var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info)
+          result.add freshVars(preprocess(c, c.finallys[i]), vars)
         result.add n
   of nkSkip: discard
   else:
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
 
-proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n: PNode): PNode =
+proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   var ctx: Ctx
   ctx.g = g
   ctx.fn = fn
-  ctx.idgen = idgen
 
   if getEnvParam(fn).isNil:
     # Lambda lifting was not done yet. Use temporary :state sym, which will
     # be handled specially by lambda lifting. Local temp vars (if needed)
     # should follow the same logic.
-    ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), nextSymId(idgen), fn, fn.info)
-    ctx.stateVarSym.typ = g.createClosureIterStateType(fn, idgen)
-  ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), nextSymId(idgen), fn, fn.info)
-  var pc = PreprocessContext(finallys: @[], config: g.config, idgen: idgen)
+    ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), fn, fn.info)
+    ctx.stateVarSym.typ = g.createClosureIterStateType(fn)
+  ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), fn, fn.info)
+  var pc = PreprocessContext(finallys: @[], config: g.config)
   var n = preprocess(pc, n.toStmtList)
   #echo "transformed into ", n
   #var n = n.toStmtList

compiler/cmdlinehelper.nim

@@ -13,6 +13,8 @@ import
   options, idents, nimconf, extccomp, commands, msgs,
   lineinfos, modulegraphs, condsyms, os, pathutils, parseopt
 
+from strutils import normalize
+
 proc prependCurDir*(f: AbsoluteFile): AbsoluteFile =
   when defined(unix):
     if os.isAbsolute(f.string): result = f
@@ -39,40 +41,42 @@ proc initDefinesProg*(self: NimProg, conf: ConfigRef, name: string) =
 
 proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   self.processCmdLine(passCmd1, "", conf)
-  if conf.projectIsCmd and conf.projectName in ["-", ""]:
-    handleCmdInput(conf)
-  elif self.supportsStdinFile and conf.projectName == "-":
+  if self.supportsStdinFile and conf.projectName == "-":
     handleStdinInput(conf)
   elif conf.projectName != "":
-    setFromProjectName(conf, conf.projectName)
+    try:
+      conf.projectFull = canonicalizePath(conf, AbsoluteFile conf.projectName)
+    except OSError:
+      conf.projectFull = AbsoluteFile conf.projectName
+    let p = splitFile(conf.projectFull)
+    let dir = if p.dir.isEmpty: AbsoluteDir getCurrentDir() else: p.dir
+    conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile dir)
+    conf.projectName = p.name
   else:
     conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir())
 
-proc loadConfigsAndProcessCmdLine*(self: NimProg, cache: IdentCache; conf: ConfigRef;
+proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef;
                                    graph: ModuleGraph): bool =
   if self.suggestMode:
-    conf.setCmd cmdIdeTools
-  if conf.cmd == cmdNimscript:
-    incl(conf.globalOptions, optWasNimscript)
-  loadConfigs(DefaultConfig, cache, conf, graph.idgen) # load all config files
+    conf.command = "nimsuggest"
+  loadConfigs(DefaultConfig, cache, conf) # load all config files
 
   if not self.suggestMode:
     let scriptFile = conf.projectFull.changeFileExt("nims")
     # 'nim foo.nims' means to just run the NimScript file and do nothing more:
     if fileExists(scriptFile) and scriptFile == conf.projectFull:
-      if conf.cmd == cmdNone: conf.setCmd cmdNimscript
-      if conf.cmd == cmdNimscript: return false
+      if conf.command == "":
+        conf.command = "e"
+        return false
+      elif conf.command.normalize == "e":
+        return false
+
   # now process command line arguments again, because some options in the
   # command line can overwrite the config file's settings
-  if conf.backend != backendJs: # bug #19059
-    extccomp.initVars(conf)
+  extccomp.initVars(conf)
   self.processCmdLine(passCmd2, "", conf)
-  if conf.cmd == cmdNone:
+  if conf.command == "":
     rawMessage(conf, errGenerated, "command missing")
 
   graph.suggestMode = self.suggestMode
   return true
-
-proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef; graph: ModuleGraph): bool =
-  ## Alias for loadConfigsAndProcessCmdLine, here for backwards compatibility
-  loadConfigsAndProcessCmdLine(self, cache, conf, graph)

compiler/commands.nim

@@ -9,8 +9,9 @@
 
 # This module handles the parsing of command line arguments.
 
+
 # We do this here before the 'import' statement so 'defined' does not get
-# confused with 'TGCMode.gcMarkAndSweep' etc.
+# confused with 'TGCMode.gcGenerational' etc.
 template bootSwitch(name, expr, userString) =
   # Helper to build boot constants, for debugging you can 'echo' the else part.
   const name = if expr: " " & userString else: ""
@@ -21,18 +22,23 @@ bootSwitch(usedDanger, defined(danger), "-d:danger")
 bootSwitch(useLinenoise, defined(nimUseLinenoise) or defined(useLinenoise), "-d:nimUseLinenoise")
 bootSwitch(usedBoehm, defined(boehmgc), "--gc:boehm")
 bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
+bootSwitch(usedGenerational, defined(gcgenerational), "--gc:generational")
 bootSwitch(usedGoGC, defined(gogc), "--gc:go")
 bootSwitch(usedNoGC, defined(nogc), "--gc:none")
 
-import std/[setutils, os, strutils, parseutils, parseopt, sequtils, strtabs]
 import
-  msgs, options, nversion, condsyms, extccomp, platform,
-  wordrecg, nimblecmd, lineinfos, pathutils, pathnorm
+  os, msgs, options, nversion, condsyms, strutils, extccomp, platform,
+  wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos,
+  pathutils, strtabs
 
-from ast import setUseIc, eqTypeFlags, tfGcSafe, tfNoSideEffect
+from incremental import nimIncremental
+from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect
 
 # but some have deps to imported modules. Yay.
 bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
+bootSwitch(usedNativeStacktrace,
+  defined(nativeStackTrace) and nativeStackTraceSupported,
+  "-d:nativeStackTrace")
 bootSwitch(usedFFI, hasFFI, "-d:nimHasLibFFI")
 
 type
@@ -92,13 +98,12 @@ proc writeVersionInfo(conf: ConfigRef; pass: TCmdLinePass) =
                {msgStdout})
 
     const gitHash {.strdefine.} = gorge("git log -n 1 --format=%H").strip
-      # xxx move this logic to std/private/gitutils
     when gitHash.len == 40:
       msgWriteln(conf, "git hash: " & gitHash, {msgStdout})
 
     msgWriteln(conf, "active boot switches:" & usedRelease & usedDanger &
-      usedTinyC & useLinenoise &
-      usedFFI & usedBoehm & usedMarkAndSweep & usedGoGC & usedNoGC,
+      usedTinyC & useLinenoise & usedNativeStacktrace &
+      usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC,
                {msgStdout})
     msgQuit(0)
 
@@ -113,7 +118,7 @@ const
   errInvalidCmdLineOption = "invalid command line option: '$1'"
   errOnOrOffExpectedButXFound = "'on' or 'off' expected, but '$1' found"
   errOnOffOrListExpectedButXFound = "'on', 'off' or 'list' expected, but '$1' found"
-  errOffHintsError = "'off', 'hint', 'error' or 'usages' expected, but '$1' found"
+  errOffHintsError = "'off', 'hint' or 'error' expected, but '$1' found"
 
 proc invalidCmdLineOption(conf: ConfigRef; pass: TCmdLinePass, switch: string, info: TLineInfo) =
   if switch == " ": localError(conf, info, errInvalidCmdLineOption % "-")
@@ -137,15 +142,6 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
   elif switch[i] == '[': arg = substr(switch, i)
   else: invalidCmdLineOption(conf, pass, switch, info)
 
-template switchOn(arg: string): bool =
-  # xxx use `switchOn` wherever appropriate
-  case arg.normalize
-  of "", "on": true
-  of "off": false
-  else:
-    localError(conf, info, errOnOrOffExpectedButXFound % arg)
-    false
-
 proc processOnOffSwitch(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass,
                         info: TLineInfo) =
   case arg.normalize
@@ -181,7 +177,7 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
                          info: TLineInfo; orig: string; conf: ConfigRef) =
   var id = ""  # arg = key or [key] or key:val or [key]:val;  with val=on|off
   var i = 0
-  var notes: set[TMsgKind]
+  var n = hintMin
   var isBracket = false
   if i < arg.len and arg[i] == '[':
     isBracket = true
@@ -196,38 +192,36 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
   if i == arg.len: discard
   elif i < arg.len and (arg[i] in {':', '='}): inc(i)
   else: invalidCmdLineOption(conf, pass, orig, info)
+  if state == wHint:
+    let x = findStr(hintMin..hintMax, id, errUnknown)
+    if x != errUnknown: n = TNoteKind(x)
+    else: localError(conf, info, "unknown hint: " & id)
+  else:
+    let x = findStr(warnMin..warnMax, id, errUnknown)
+    if x != errUnknown: n = TNoteKind(x)
+    else: localError(conf, info, "unknown warning: " & id)
 
-  let isSomeHint = state in {wHint, wHintAsError}
-  template findNote(noteMin, noteMax, name) =
-    # unfortunately, hintUser and warningUser clash, otherwise implementation would simplify a bit
-    let x = findStr(noteMin, noteMax, id, errUnknown)
-    if x != errUnknown: notes = {TNoteKind(x)}
-    else: localError(conf, info, "unknown $#: $#" % [name, id])
-  case id.normalize
-  of "all": # other note groups would be easy to support via additional cases
-    notes = if isSomeHint: {hintMin..hintMax} else: {warnMin..warnMax}
-  elif isSomeHint: findNote(hintMin, hintMax, "hint")
-  else: findNote(warnMin, warnMax, "warning")
   var val = substr(arg, i).normalize
   if val == "": val = "on"
   if val notin ["on", "off"]:
-    # xxx in future work we should also allow users to have control over `foreignPackageNotes`
-    # so that they can enable `hints|warnings|warningAsErrors` for all the code they depend on.
     localError(conf, info, errOnOrOffExpectedButXFound % arg)
-  else:
-    let isOn = val == "on"
-    if isOn and id.normalize == "all":
-      localError(conf, info, "only 'all:off' is supported")
-    for n in notes:
-      if n notin conf.cmdlineNotes or pass == passCmd1:
-        if pass == passCmd1: incl(conf.cmdlineNotes, n)
-        incl(conf.modifiedyNotes, n)
-        if state in {wWarningAsError, wHintAsError}:
-          conf.warningAsErrors[n] = isOn # xxx rename warningAsErrors to noteAsErrors
-        else:
-          conf.notes[n] = isOn
-          conf.mainPackageNotes[n] = isOn
-        if not isOn: excl(conf.foreignPackageNotes, n)
+  elif n notin conf.cmdlineNotes or pass == passCmd1:
+    if pass == passCmd1: incl(conf.cmdlineNotes, n)
+    incl(conf.modifiedyNotes, n)
+    case val
+    of "on":
+      if state == wWarningAsError:
+        incl(conf.warningAsErrors, n)
+      else:
+        incl(conf.notes, n)
+        incl(conf.mainPackageNotes, n)
+    of "off":
+      if state == wWarningAsError:
+        excl(conf.warningAsErrors, n)
+      else:
+        excl(conf.notes, n)
+        excl(conf.mainPackageNotes, n)
+        excl(conf.foreignPackageNotes, n)
 
 proc processCompile(conf: ConfigRef; filename: string) =
   var found = findFile(conf, filename)
@@ -238,28 +232,23 @@ const
   errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found"
-  errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found"
-
-template warningOptionNoop(switch: string) =
-  warningDeprecated(conf, info, "'$#' is deprecated, now a noop" % switch)
-
-template deprecatedAlias(oldName, newName: string) =
-  warningDeprecated(conf, info, "'$#' is a deprecated alias for '$#'" % [oldName, newName])
+  errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found"
 
 proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo): bool =
   case switch.normalize
-  of "gc", "mm":
+  of "gc":
     case arg.normalize
     of "boehm": result = conf.selectedGC == gcBoehm
     of "refc": result = conf.selectedGC == gcRefc
+    of "v2": result = false
     of "markandsweep": result = conf.selectedGC == gcMarkAndSweep
+    of "generational": result = false
     of "destructors", "arc": result = conf.selectedGC == gcArc
     of "orc": result = conf.selectedGC == gcOrc
     of "hooks": result = conf.selectedGC == gcHooks
     of "go": result = conf.selectedGC == gcGo
     of "none": result = conf.selectedGC == gcNone
     of "stack", "regions": result = conf.selectedGC == gcRegions
-    of "v2", "generational": warningOptionNoop(arg)
     else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "opt":
     case arg.normalize
@@ -309,13 +298,12 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
     result = conf.options * {optNaNCheck, optInfCheck} == {optNaNCheck, optInfCheck}
   of "infchecks": result = contains(conf.options, optInfCheck)
   of "nanchecks": result = contains(conf.options, optNaNCheck)
+  of "nilchecks": result = false # not a thing
   of "objchecks": result = contains(conf.options, optObjCheck)
   of "fieldchecks": result = contains(conf.options, optFieldCheck)
   of "rangechecks": result = contains(conf.options, optRangeCheck)
   of "boundchecks": result = contains(conf.options, optBoundsCheck)
-  of "refchecks":
-    warningDeprecated(conf, info, "refchecks is deprecated!")
-    result = contains(conf.options, optRefCheck)
+  of "refchecks": result = contains(conf.options, optRefCheck)
   of "overflowchecks": result = contains(conf.options, optOverflowCheck)
   of "staticboundchecks": result = contains(conf.options, optStaticBoundsCheck)
   of "stylechecks": result = contains(conf.options, optStyleCheck)
@@ -325,13 +313,12 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "symbolfiles": result = conf.symbolFiles != disabledSf
   of "genscript": result = contains(conf.globalOptions, optGenScript)
   of "threads": result = contains(conf.globalOptions, optThreads)
+  of "taintmode": result = contains(conf.globalOptions, optTaintMode)
   of "tlsemulation": result = contains(conf.globalOptions, optTlsEmulation)
   of "implicitstatic": result = contains(conf.options, optImplicitStatic)
-  of "patterns", "trmacros":
-    if switch.normalize == "patterns": deprecatedAlias(switch, "trmacros")
-    result = contains(conf.options, optTrMacros)
+  of "patterns", "trmacros": result = contains(conf.options, optTrMacros)
   of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace)
-  of "nilseqs", "nilchecks", "taintmode": warningOptionNoop(switch)
+  of "nilseqs": result = contains(conf.options, optNilSeqs)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc processPath(conf: ConfigRef; path: string, info: TLineInfo,
@@ -365,158 +352,70 @@ proc processCfgPath(conf: ConfigRef; path: string, info: TLineInfo): AbsoluteDir
 const
   errInvalidNumber = "$1 is not a valid number"
 
-proc makeAbsolute(s: string): AbsoluteFile =
-  if isAbsolute(s):
-    AbsoluteFile pathnorm.normalizePath(s)
-  else:
-    AbsoluteFile pathnorm.normalizePath(os.getCurrentDir() / s)
-
-proc setTrackingInfo(conf: ConfigRef; dirty, file, line, column: string,
-                     info: TLineInfo) =
-  ## set tracking info, common code for track, trackDirty, & ideTrack
-  var ln, col: int
-  if parseUtils.parseInt(line, ln) <= 0:
-    localError(conf, info, errInvalidNumber % line)
-  if parseUtils.parseInt(column, col) <= 0:
-    localError(conf, info, errInvalidNumber % column)
-
-  let a = makeAbsolute(file)
-  if dirty == "":
-    conf.m.trackPos = newLineInfo(conf, a, ln, col)
-  else:
-    let dirtyOriginalIdx = fileInfoIdx(conf, a)
-    if dirtyOriginalIdx.int32 >= 0:
-      msgs.setDirtyFile(conf, dirtyOriginalIdx, makeAbsolute(dirty))
-    conf.m.trackPos = newLineInfo(dirtyOriginalIdx, ln, col)
-
 proc trackDirty(conf: ConfigRef; arg: string, info: TLineInfo) =
   var a = arg.split(',')
   if a.len != 4: localError(conf, info,
                             "DIRTY_BUFFER,ORIGINAL_FILE,LINE,COLUMN expected")
-  setTrackingInfo(conf, a[0], a[1], a[2], a[3], info)
+  var line, column: int
+  if parseUtils.parseInt(a[2], line) <= 0:
+    localError(conf, info, errInvalidNumber % a[1])
+  if parseUtils.parseInt(a[3], column) <= 0:
+    localError(conf, info, errInvalidNumber % a[2])
+
+  let dirtyOriginalIdx = fileInfoIdx(conf, AbsoluteFile a[1])
+  if dirtyOriginalIdx.int32 >= 0:
+    msgs.setDirtyFile(conf, dirtyOriginalIdx, AbsoluteFile a[0])
+
+  conf.m.trackPos = newLineInfo(dirtyOriginalIdx, line, column)
 
 proc track(conf: ConfigRef; arg: string, info: TLineInfo) =
   var a = arg.split(',')
   if a.len != 3: localError(conf, info, "FILE,LINE,COLUMN expected")
-  setTrackingInfo(conf, "", a[0], a[1], a[2], info)
-
-proc trackIde(conf: ConfigRef; cmd: IdeCmd, arg: string, info: TLineInfo) =
-  ## set the tracking info related to an ide cmd, supports optional dirty file
-  var a = arg.split(',')
-  case a.len
-  of 4:
-    setTrackingInfo(conf, a[0], a[1], a[2], a[3], info)
-  of 3:
-    setTrackingInfo(conf, "", a[0], a[1], a[2], info)
-  else:
-    localError(conf, info, "[DIRTY_BUFFER,]ORIGINAL_FILE,LINE,COLUMN expected")
-  conf.ideCmd = cmd
+  var line, column: int
+  if parseUtils.parseInt(a[1], line) <= 0:
+    localError(conf, info, errInvalidNumber % a[1])
+  if parseUtils.parseInt(a[2], column) <= 0:
+    localError(conf, info, errInvalidNumber % a[2])
+  conf.m.trackPos = newLineInfo(conf, AbsoluteFile a[0], line, column)
 
 proc dynlibOverride(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
   if pass in {passCmd2, passPP}:
     expectArg(conf, switch, arg, pass, info)
     options.inclDynlibOverride(conf, arg)
 
-template handleStdinOrCmdInput =
+proc handleStdinInput*(conf: ConfigRef) =
+  conf.projectName = "stdinfile"
   conf.projectFull = conf.projectName.AbsoluteFile
   conf.projectPath = AbsoluteDir getCurrentDir()
+  conf.projectIsStdin = true
   if conf.outDir.isEmpty:
     conf.outDir = getNimcacheDir(conf)
 
-proc handleStdinInput*(conf: ConfigRef) =
-  conf.projectName = "stdinfile"
-  conf.projectIsStdin = true
-  handleStdinOrCmdInput()
-
-proc handleCmdInput*(conf: ConfigRef) =
-  conf.projectName = "cmdfile"
-  handleStdinOrCmdInput()
-
-proc parseCommand*(command: string): Command =
-  case command.normalize
-  of "c", "cc", "compile", "compiletoc": cmdCompileToC
-  of "cpp", "compiletocpp": cmdCompileToCpp
-  of "objc", "compiletooc": cmdCompileToOC
-  of "js", "compiletojs": cmdCompileToJS
-  of "r": cmdCrun
-  of "run": cmdTcc
-  of "check": cmdCheck
-  of "e": cmdNimscript
-  of "doc0": cmdDoc0
-  of "doc2", "doc": cmdDoc
-  of "doc2tex": cmdDoc2tex
-  of "rst2html": cmdRst2html
-  of "rst2tex": cmdRst2tex
-  of "jsondoc0": cmdJsondoc0
-  of "jsondoc2", "jsondoc": cmdJsondoc
-  of "ctags": cmdCtags
-  of "buildindex": cmdBuildindex
-  of "gendepend": cmdGendepend
-  of "dump": cmdDump
-  of "parse": cmdParse
-  of "rod": cmdRod
-  of "secret": cmdInteractive
-  of "nop", "help": cmdNop
-  of "jsonscript": cmdJsonscript
-  else: cmdUnknown
-
-proc setCmd*(conf: ConfigRef, cmd: Command) =
-  ## sets cmd, backend so subsequent flags can query it (e.g. so --gc:arc can be ignored for backendJs)
-  # Note that `--backend` can override the backend, so the logic here must remain reversible.
-  conf.cmd = cmd
-  case cmd
-  of cmdCompileToC, cmdCrun, cmdTcc: conf.backend = backendC
-  of cmdCompileToCpp: conf.backend = backendCpp
-  of cmdCompileToOC: conf.backend = backendObjc
-  of cmdCompileToJS: conf.backend = backendJs
-  else: discard
-
-proc setCommandEarly*(conf: ConfigRef, command: string) =
-  conf.command = command
-  setCmd(conf, command.parseCommand)
-  # command early customizations
-  # must be handled here to honor subsequent `--hint:x:on|off`
-  case conf.cmd
-  of cmdRst2html, cmdRst2tex: # xxx see whether to add others: cmdGendepend, etc.
-    conf.foreignPackageNotes = {hintSuccessX}
-  else:
-    conf.foreignPackageNotes = foreignPackageNotesDefault
-
-proc specialDefine(conf: ConfigRef, key: string; pass: TCmdLinePass) =
+proc specialDefine(conf: ConfigRef, key: string) =
   # Keep this syncronized with the default config/nim.cfg!
   if cmpIgnoreStyle(key, "nimQuirky") == 0:
     conf.exc = excQuirky
   elif cmpIgnoreStyle(key, "release") == 0 or cmpIgnoreStyle(key, "danger") == 0:
-    if pass in {passCmd1, passPP}:
-      conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize}
-      conf.globalOptions.excl {optExcessiveStackTrace, optCDebug}
-      conf.options.incl optOptimizeSpeed
+    conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize}
+    conf.globalOptions.excl {optExcessiveStackTrace, optCDebug}
+    conf.options.incl optOptimizeSpeed
   if cmpIgnoreStyle(key, "danger") == 0 or cmpIgnoreStyle(key, "quick") == 0:
-    if pass in {passCmd1, passPP}:
-      conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck,
-        optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir}
-      conf.globalOptions.excl {optCDebug}
+    conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck,
+      optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir}
+    conf.globalOptions.excl {optCDebug}
 
 proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
                     conf: ConfigRef) =
   var
     key, val: string
   case switch.normalize
-  of "eval":
-    expectArg(conf, switch, arg, pass, info)
-    conf.projectIsCmd = true
-    conf.cmdInput = arg # can be empty (a nim file with empty content is valid too)
-    if conf.cmd == cmdNone:
-      conf.command = "e"
-      conf.setCmd cmdNimscript # better than `cmdCrun` as a default
-      conf.implicitCmd = true
   of "path", "p":
     expectArg(conf, switch, arg, pass, info)
     for path in nimbleSubs(conf, arg):
       addPath(conf, if pass == passPP: processCfgPath(conf, path, info)
                     else: processPath(conf, path, info), info)
   of "nimblepath", "babelpath":
-    if switch.normalize == "babelpath": deprecatedAlias(switch, "nimblepath")
+    # keep the old name for compat
     if pass in {passCmd2, passPP} and optNoNimblePath notin conf.globalOptions:
       expectArg(conf, switch, arg, pass, info)
       var path = processPath(conf, arg, info, notRelativeToProj=true)
@@ -525,7 +424,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         path = nimbleDir / RelativeDir"pkgs"
       nimblePath(conf, path, info)
   of "nonimblepath", "nobabelpath":
-    if switch.normalize == "nobabelpath": deprecatedAlias(switch, "nonimblepath")
     expectNoArg(conf, switch, arg, pass, info)
     disableNimblePath(conf)
   of "clearnimblepath":
@@ -538,10 +436,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     conf.lazyPaths.keepItIf(it != path)
   of "nimcache":
     expectArg(conf, switch, arg, pass, info)
-    var arg = arg
-    # refs bug #18674, otherwise `--os:windows` messes up with `--nimcache` set
-    # in config nims files, e.g. via: `import os; switch("nimcache", "/tmp/somedir")`
-    if conf.target.targetOS == osWindows and DirSep == '/': arg = arg.replace('\\', '/')
     conf.nimcacheDir = processPath(conf, arg, info, notRelativeToProj=true)
   of "out", "o":
     expectArg(conf, switch, arg, pass, info)
@@ -563,18 +457,23 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
     conf.backend = backend
   of "doccmd": conf.docCmd = arg
+  of "mainmodule", "m":
+    discard "allow for backwards compatibility, but don't do anything"
   of "define", "d":
     expectArg(conf, switch, arg, pass, info)
     if {':', '='} in arg:
       splitSwitch(conf, arg, key, val, pass, info)
-      specialDefine(conf, key, pass)
+      specialDefine(conf, key)
       defineSymbol(conf.symbols, key, val)
     else:
-      specialDefine(conf, arg, pass)
+      specialDefine(conf, arg)
       defineSymbol(conf.symbols, arg)
   of "undef", "u":
     expectArg(conf, switch, arg, pass, info)
     undefSymbol(conf.symbols, arg)
+  of "symbol":
+    expectArg(conf, switch, arg, pass, info)
+    # deprecated, do nothing
   of "compile":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}: processCompile(conf, arg)
@@ -596,8 +495,12 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
   of "project":
     processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info)
-  of "gc", "mm":
-    if conf.backend == backendJs: return # for: bug #16033
+  of "gc":
+    if conf.backend == backendJs or conf.command == "js":
+      # for: bug #16033
+      # This might still be imperfect, in rarse corner cases
+      # (where command is reset in nimscript, maybe).
+      return
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
       case arg.normalize
@@ -607,6 +510,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS
       of "refc":
         conf.selectedGC = gcRefc
+      of "v2":
+        message(conf, info, warnDeprecated, "--gc:v2 is deprecated; using default gc")
       of "markandsweep":
         conf.selectedGC = gcMarkAndSweep
         defineSymbol(conf.symbols, "gcmarkandsweep")
@@ -619,8 +524,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         if pass in {passCmd2, passPP}:
           defineSymbol(conf.symbols, "nimSeqsV2")
           defineSymbol(conf.symbols, "nimV2")
-        if conf.exc == excNone and conf.backend != backendCpp:
-          conf.exc = excGoto
       of "orc":
         conf.selectedGC = gcOrc
         defineSymbol(conf.symbols, "gcdestructors")
@@ -630,8 +533,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         if pass in {passCmd2, passPP}:
           defineSymbol(conf.symbols, "nimSeqsV2")
           defineSymbol(conf.symbols, "nimV2")
-        if conf.exc == excNone and conf.backend != backendCpp:
-          conf.exc = excGoto
       of "hooks":
         conf.selectedGC = gcHooks
         defineSymbol(conf.symbols, "gchooks")
@@ -648,19 +549,15 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       of "stack", "regions":
         conf.selectedGC = gcRegions
         defineSymbol(conf.symbols, "gcregions")
-      of "v2": warningOptionNoop(arg)
       else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "warnings", "w":
     if processOnOffSwitchOrList(conf, {optWarns}, arg, pass, info): listWarnings(conf)
   of "warning": processSpecificNote(arg, wWarning, pass, info, switch, conf)
   of "hint": processSpecificNote(arg, wHint, pass, info, switch, conf)
   of "warningaserror": processSpecificNote(arg, wWarningAsError, pass, info, switch, conf)
-  of "hintaserror": processSpecificNote(arg, wHintAsError, pass, info, switch, conf)
   of "hints":
     if processOnOffSwitchOrList(conf, {optHints}, arg, pass, info): listHints(conf)
-  of "threadanalysis":
-    if conf.backend == backendJs: discard
-    else: processOnOffSwitchG(conf, {optThreadAnalysis}, arg, pass, info)
+  of "threadanalysis": processOnOffSwitchG(conf, {optThreadAnalysis}, arg, pass, info)
   of "stacktrace": processOnOffSwitch(conf, {optStackTrace}, arg, pass, info)
   of "stacktracemsgs": processOnOffSwitch(conf, {optStackTraceMsgs}, arg, pass, info)
   of "excessivestacktrace": processOnOffSwitchG(conf, {optExcessiveStackTrace}, arg, pass, info)
@@ -699,32 +596,32 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     else:
       undefSymbol(conf.symbols, "hotcodereloading")
       undefSymbol(conf.symbols, "useNimRtl")
+  of "nilseqs": processOnOffSwitch(conf, {optNilSeqs}, arg, pass, info)
   of "checks", "x": processOnOffSwitch(conf, ChecksOptions, arg, pass, info)
   of "floatchecks":
     processOnOffSwitch(conf, {optNaNCheck, optInfCheck}, arg, pass, info)
   of "infchecks": processOnOffSwitch(conf, {optInfCheck}, arg, pass, info)
   of "nanchecks": processOnOffSwitch(conf, {optNaNCheck}, arg, pass, info)
+  of "nilchecks": discard "no such thing anymore"
   of "objchecks": processOnOffSwitch(conf, {optObjCheck}, arg, pass, info)
   of "fieldchecks": processOnOffSwitch(conf, {optFieldCheck}, arg, pass, info)
   of "rangechecks": processOnOffSwitch(conf, {optRangeCheck}, arg, pass, info)
   of "boundchecks": processOnOffSwitch(conf, {optBoundsCheck}, arg, pass, info)
-  of "refchecks":
-    warningDeprecated(conf, info, "refchecks is deprecated!")
-    processOnOffSwitch(conf, {optRefCheck}, arg, pass, info)
+  of "refchecks": processOnOffSwitch(conf, {optRefCheck}, arg, pass, info)
   of "overflowchecks": processOnOffSwitch(conf, {optOverflowCheck}, arg, pass, info)
   of "staticboundchecks": processOnOffSwitch(conf, {optStaticBoundsCheck}, arg, pass, info)
   of "stylechecks": processOnOffSwitch(conf, {optStyleCheck}, arg, pass, info)
   of "linedir": processOnOffSwitch(conf, {optLineDir}, arg, pass, info)
   of "assertions", "a": processOnOffSwitch(conf, {optAssert}, arg, pass, info)
+  of "deadcodeelim": discard # deprecated, dead code elim always on
   of "threads":
-    if conf.backend == backendJs: discard
-    else: processOnOffSwitchG(conf, {optThreads}, arg, pass, info)
+    processOnOffSwitchG(conf, {optThreads}, arg, pass, info)
     #if optThreads in conf.globalOptions: conf.setNote(warnGcUnsafe)
   of "tlsemulation": processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info)
+  of "taintmode": processOnOffSwitchG(conf, {optTaintMode}, arg, pass, info)
   of "implicitstatic":
     processOnOffSwitch(conf, {optImplicitStatic}, arg, pass, info)
   of "patterns", "trmacros":
-    if switch.normalize == "patterns": deprecatedAlias(switch, "trmacros")
     processOnOffSwitch(conf, {optTrMacros}, arg, pass, info)
   of "opt":
     expectArg(conf, switch, arg, pass, info)
@@ -776,7 +673,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "clib":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
-      conf.cLinkedLibs.add arg
+      conf.cLinkedLibs.add processPath(conf, arg, info).string
   of "header":
     if conf != nil: conf.headerFile = arg
     incl(conf.globalOptions, optGenIndex)
@@ -851,19 +748,19 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "help", "h":
     expectNoArg(conf, switch, arg, pass, info)
     helpOnError(conf, pass)
-  of "symbolfiles", "incremental", "ic":
-    if switch.normalize == "symbolfiles": deprecatedAlias(switch, "incremental")
-      # xxx maybe also ic, since not in help?
-    if pass in {passCmd2, passPP}:
-      case arg.normalize
-      of "on": conf.symbolFiles = v2Sf
-      of "off": conf.symbolFiles = disabledSf
-      of "writeonly": conf.symbolFiles = writeOnlySf
-      of "readonly": conf.symbolFiles = readOnlySf
-      of "v2": conf.symbolFiles = v2Sf
-      of "stress": conf.symbolFiles = stressTest
-      else: localError(conf, info, "invalid option for --incremental: " & arg)
-    setUseIc(conf.symbolFiles != disabledSf)
+  of "symbolfiles": discard "ignore for backwards compat"
+  of "incremental":
+    when not nimIncremental:
+      localError(conf, info, "the compiler was not built with " &
+        "incremental compilation features; bootstrap with " &
+        "-d:nimIncremental to enable")
+    case arg.normalize
+    of "on": conf.symbolFiles = v2Sf
+    of "off": conf.symbolFiles = disabledSf
+    of "writeonly": conf.symbolFiles = writeOnlySf
+    of "readonly": conf.symbolFiles = readOnlySf
+    of "v2": conf.symbolFiles = v2Sf
+    else: localError(conf, info, "invalid option for --incremental: " & arg)
   of "skipcfg":
     processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info)
   of "skipprojcfg":
@@ -873,7 +770,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "skipparentcfg":
     processOnOffSwitchG(conf, {optSkipParentConfigFiles}, arg, pass, info)
   of "genscript", "gendeps":
-    if switch.normalize == "gendeps": deprecatedAlias(switch, "genscript")
     processOnOffSwitchG(conf, {optGenScript}, arg, pass, info)
     processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info)
   of "colors": processOnOffSwitchG(conf, {optUseColors}, arg, pass, info)
@@ -885,9 +781,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     splitSwitch(conf, arg, key, val, pass, info)
     os.putEnv(key, val)
   of "cc":
-    if conf.backend != backendJs: # bug #19330
-      expectArg(conf, switch, arg, pass, info)
-      setCC(conf, arg, info)
+    expectArg(conf, switch, arg, pass, info)
+    setCC(conf, arg, info)
   of "track":
     expectArg(conf, switch, arg, pass, info)
     track(conf, arg, info)
@@ -898,50 +793,28 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectNoArg(conf, switch, arg, pass, info)
     conf.ideCmd = ideSug
   of "def":
+    expectNoArg(conf, switch, arg, pass, info)
+    conf.ideCmd = ideDef
+  of "eval":
     expectArg(conf, switch, arg, pass, info)
-    trackIde(conf, ideDef, arg, info)
+    conf.evalExpr = arg
   of "context":
     expectNoArg(conf, switch, arg, pass, info)
     conf.ideCmd = ideCon
   of "usages":
-    expectArg(conf, switch, arg, pass, info)
-    trackIde(conf, ideUse, arg, info)
-  of "defusages":
-    expectArg(conf, switch, arg, pass, info)
-    trackIde(conf, ideDus, arg, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    conf.ideCmd = ideUse
   of "stdout":
     processOnOffSwitchG(conf, {optStdout}, arg, pass, info)
-  of "filenames":
-    case arg.normalize
-    of "abs": conf.filenameOption = foAbs
-    of "canonical": conf.filenameOption = foCanonical
-    of "legacyrelproj": conf.filenameOption = foLegacyRelProj
-    else: localError(conf, info, "expected: abs|canonical|legacyRelProj, got: $1" % arg)
-  of "processing":
-    incl(conf.notes, hintProcessing)
-    incl(conf.mainPackageNotes, hintProcessing)
-    case arg.normalize
-    of "dots": conf.hintProcessingDots = true
-    of "filenames": conf.hintProcessingDots = false
-    of "off":
-      excl(conf.notes, hintProcessing)
-      excl(conf.mainPackageNotes, hintProcessing)
-    else: localError(conf, info, "expected: dots|filenames|off, got: $1" % arg)
-  of "unitsep":
-    conf.unitSep = if switchOn(arg): "\31" else: ""
   of "listfullpaths":
-    # xxx in future work, use `warningDeprecated`
-    conf.filenameOption = if switchOn(arg): foAbs else: foCanonical
-  of "spellsuggest":
-    if arg.len == 0: conf.spellSuggestMax = spellSuggestSecretSauce
-    elif arg == "auto": conf.spellSuggestMax = spellSuggestSecretSauce
-    else: conf.spellSuggestMax = parseInt(arg)
-  of "declaredlocs":
-    processOnOffSwitchG(conf, {optDeclaredLocs}, arg, pass, info)
+    processOnOffSwitchG(conf, {optListFullPaths}, arg, pass, info)
   of "dynliboverride":
     dynlibOverride(conf, switch, arg, pass, info)
   of "dynliboverrideall":
     processOnOffSwitchG(conf, {optDynlibOverrideAll}, arg, pass, info)
+  of "cs":
+    # only supported for compatibility. Does nothing.
+    expectArg(conf, switch, arg, pass, info)
   of "experimental":
     if arg.len == 0:
       conf.features.incl oldExperimentalFeatures
@@ -971,7 +844,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if conf != nil:
       conf.cppDefine(arg)
   of "newruntime":
-    warningDeprecated(conf, info, "newruntime is deprecated, use arc/orc instead!")
     expectNoArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
       doAssert(conf != nil)
@@ -1019,6 +891,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "1.0":
       defineSymbol(conf.symbols, "NimMajor", "1")
       defineSymbol(conf.symbols, "NimMinor", "0")
+      # always be compatible with 1.0.100:
+      defineSymbol(conf.symbols, "NimPatch", "100")
       # old behaviors go here:
       defineSymbol(conf.symbols, "nimOldRelativePathBehavior")
       undefSymbol(conf.symbols, "nimDoesntTrackDefects")
@@ -1027,11 +901,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "1.2":
       defineSymbol(conf.symbols, "NimMajor", "1")
       defineSymbol(conf.symbols, "NimMinor", "2")
+      # always be compatible with 1.2.100:
+      defineSymbol(conf.symbols, "NimPatch", "100")
       conf.globalOptions.incl optNimV12Emulation
     else:
-      localError(conf, info, "unknown Nim version; currently supported values are: `1.0`, `1.2`")
-    # always be compatible with 1.x.100:
-    defineSymbol(conf.symbols, "NimPatch", "100")
+      localError(conf, info, "unknown Nim version; currently supported values are: {1.0}")
   of "benchmarkvm":
     processOnOffSwitchG(conf, {optBenchmarkVM}, arg, pass, info)
   of "profilevm":
@@ -1045,15 +919,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitchG(conf, {optPanics}, arg, pass, info)
     if optPanics in conf.globalOptions:
       defineSymbol(conf.symbols, "nimPanics")
-  of "sourcemap": # xxx document in --fullhelp
+  of "sourcemap":
     conf.globalOptions.incl optSourcemap
     conf.options.incl optLineDir
   of "deepcopy":
     processOnOffSwitchG(conf, {optEnableDeepCopy}, arg, pass, info)
   of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -)
     handleStdinInput(conf)
-  of "nilseqs", "nilchecks", "symbol", "taintmode", "cs", "deadcodeelim": warningOptionNoop(switch)
-  of "nimmainprefix": conf.nimMainPrefix = arg
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)
@@ -1063,6 +935,7 @@ proc processCommand*(switch: string, pass: TCmdLinePass; config: ConfigRef) =
   splitSwitch(config, switch, cmd, arg, pass, gCmdLineInfo)
   processSwitch(cmd, arg, pass, gCmdLineInfo, config)
 
+
 proc processSwitch*(pass: TCmdLinePass; p: OptParser; config: ConfigRef) =
   # hint[X]:off is parsed as (p.key = "hint[X]", p.val = "off")
   # we transform it to (key = hint, val = [X]:off)
@@ -1076,22 +949,19 @@ proc processSwitch*(pass: TCmdLinePass; p: OptParser; config: ConfigRef) =
 
 proc processArgument*(pass: TCmdLinePass; p: OptParser;
                       argsCount: var int; config: ConfigRef): bool =
-  if argsCount == 0 and config.implicitCmd:
-    argsCount.inc
   if argsCount == 0:
     # nim filename.nims  is the same as "nim e filename.nims":
     if p.key.endsWith(".nims"):
-      config.setCmd cmdNimscript
+      config.command = "e"
       incl(config.globalOptions, optWasNimscript)
       config.projectName = unixToNativePath(p.key)
       config.arguments = cmdLineRest(p)
       result = true
-    elif pass != passCmd2: setCommandEarly(config, p.key)
+    elif pass != passCmd2:
+      config.command = p.key
   else:
     if pass == passCmd1: config.commandArgs.add p.key
     if argsCount == 1:
-      if p.key.endsWith(".nims"):
-        incl(config.globalOptions, optWasNimscript)
       # support UNIX style filenames everywhere for portable build scripts:
       if config.projectName.len == 0:
         config.projectName = unixToNativePath(p.key)

compiler/concepts.nim

@@ -1,339 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2020 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## New styled concepts for Nim. See https://github.com/nim-lang/RFCs/issues/168
-## for details. Note this is a first implementation and only the "Concept matching"
-## section has been implemented.
-
-import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types, intsets
-
-from magicsys import addSonSkipIntLit
-
-const
-  logBindings = false
-
-## Code dealing with Concept declarations
-## --------------------------------------
-
-proc declareSelf(c: PContext; info: TLineInfo) =
-  ## Adds the magical 'Self' symbols to the current scope.
-  let ow = getCurrOwner(c)
-  let s = newSym(skType, getIdent(c.cache, "Self"), nextSymId(c.idgen), ow, info)
-  s.typ = newType(tyTypeDesc, nextTypeId(c.idgen), ow)
-  s.typ.flags.incl {tfUnresolved, tfPacked}
-  s.typ.add newType(tyEmpty, nextTypeId(c.idgen), ow)
-  addDecl(c, s, info)
-
-proc isSelf*(t: PType): bool {.inline.} =
-  ## Is this the magical 'Self' type?
-  t.kind == tyTypeDesc and tfPacked in t.flags
-
-proc makeTypeDesc*(c: PContext, typ: PType): PType =
-  if typ.kind == tyTypeDesc and not isSelf(typ):
-    result = typ
-  else:
-    result = newTypeS(tyTypeDesc, c)
-    incl result.flags, tfCheckedForDestructor
-    result.addSonSkipIntLit(typ, c.idgen)
-
-proc semConceptDecl(c: PContext; n: PNode): PNode =
-  ## Recursive helper for semantic checking for the concept declaration.
-  ## Currently we only support (possibly empty) lists of statements
-  ## containing 'proc' declarations and the like.
-  case n.kind
-  of nkStmtList, nkStmtListExpr:
-    result = shallowCopy(n)
-    for i in 0..<n.len:
-      result[i] = semConceptDecl(c, n[i])
-  of nkProcDef..nkIteratorDef, nkFuncDef:
-    result = c.semExpr(c, n, {efWantStmt})
-  of nkTypeClassTy:
-    result = shallowCopy(n)
-    for i in 0..<n.len-1:
-      result[i] = n[i]
-    result[^1] = semConceptDecl(c, n[^1])
-  else:
-    localError(c.config, n.info, "unexpected construct in the new-styled concept: " & renderTree(n))
-    result = n
-
-proc semConceptDeclaration*(c: PContext; n: PNode): PNode =
-  ## Semantic checking for the concept declaration. Runs
-  ## when we process the concept itself, not its matching process.
-  assert n.kind == nkTypeClassTy
-  inc c.inConceptDecl
-  openScope(c)
-  declareSelf(c, n.info)
-  result = semConceptDecl(c, n)
-  rawCloseScope(c)
-  dec c.inConceptDecl
-
-## Concept matching
-## ----------------
-
-type
-  MatchCon = object ## Context we pass around during concept matching.
-    inferred: seq[(PType, PType)] ## we need a seq here so that we can easily undo inferences \
-      ## that turned out to be wrong.
-    marker: IntSet ## Some protection against wild runaway recursions.
-    potentialImplementation: PType ## the concrete type that might match the concept we try to match.
-    magic: TMagic  ## mArrGet and mArrPut is wrong in system.nim and
-                   ## cannot be fixed that easily.
-                   ## Thus we special case it here.
-
-proc existingBinding(m: MatchCon; key: PType): PType =
-  ## checks if we bound the type variable 'key' already to some
-  ## concrete type.
-  for i in 0..<m.inferred.len:
-    if m.inferred[i][0] == key: return m.inferred[i][1]
-  return nil
-
-proc conceptMatchNode(c: PContext; n: PNode; m: var MatchCon): bool
-
-proc matchType(c: PContext; f, a: PType; m: var MatchCon): bool =
-  ## The heart of the concept matching process. 'f' is the formal parameter of some
-  ## routine inside the concept that we're looking for. 'a' is the formal parameter
-  ## of a routine that might match.
-  const
-    ignorableForArgType = {tyVar, tySink, tyLent, tyOwned, tyGenericInst, tyAlias, tyInferred}
-  case f.kind
-  of tyAlias:
-    result = matchType(c, f.lastSon, a, m)
-  of tyTypeDesc:
-    if isSelf(f):
-      #let oldLen = m.inferred.len
-      result = matchType(c, a, m.potentialImplementation, m)
-      #echo "self is? ", result, " ", a.kind, " ", a, " ", m.potentialImplementation, " ", m.potentialImplementation.kind
-      #m.inferred.setLen oldLen
-      #echo "A for ", result, " to ", typeToString(a), " to ", typeToString(m.potentialImplementation)
-    else:
-      if a.kind == tyTypeDesc and f.len == a.len:
-        for i in 0..<a.len:
-          if not matchType(c, f[i], a[i], m): return false
-        return true
-
-  of tyGenericInvocation:
-    if a.kind == tyGenericInst and a[0].kind == tyGenericBody:
-      if sameType(f[0], a[0]) and f.len == a.len-1:
-        for i in 1 ..< f.len:
-          if not matchType(c, f[i], a[i], m): return false
-        return true
-  of tyGenericParam:
-    let ak = a.skipTypes({tyVar, tySink, tyLent, tyOwned})
-    if ak.kind in {tyTypeDesc, tyStatic} and not isSelf(ak):
-      result = false
-    else:
-      let old = existingBinding(m, f)
-      if old == nil:
-        if f.len > 0 and f[0].kind != tyNone:
-          # also check the generic's constraints:
-          let oldLen = m.inferred.len
-          result = matchType(c, f[0], a, m)
-          m.inferred.setLen oldLen
-          if result:
-            when logBindings: echo "A adding ", f, " ", ak
-            m.inferred.add((f, ak))
-        elif m.magic == mArrGet and ak.kind in {tyArray, tyOpenArray, tySequence, tyVarargs, tyCstring, tyString}:
-          when logBindings: echo "B adding ", f, " ", lastSon ak
-          m.inferred.add((f, lastSon ak))
-          result = true
-        else:
-          when logBindings: echo "C adding ", f, " ", ak
-          m.inferred.add((f, ak))
-          #echo "binding ", typeToString(ak), " to ", typeToString(f)
-          result = true
-      elif not m.marker.containsOrIncl(old.id):
-        result = matchType(c, old, ak, m)
-        if m.magic == mArrPut and ak.kind == tyGenericParam:
-          result = true
-    #echo "B for ", result, " to ", typeToString(a), " to ", typeToString(m.potentialImplementation)
-
-  of tyVar, tySink, tyLent, tyOwned:
-    # modifiers in the concept must be there in the actual implementation
-    # too but not vice versa.
-    if a.kind == f.kind:
-      result = matchType(c, f.sons[0], a.sons[0], m)
-    elif m.magic == mArrPut:
-      result = matchType(c, f.sons[0], a, m)
-    else:
-      result = false
-  of tyEnum, tyObject, tyDistinct:
-    result = sameType(f, a)
-  of tyEmpty, tyString, tyCstring, tyPointer, tyNil, tyUntyped, tyTyped, tyVoid:
-    result = a.skipTypes(ignorableForArgType).kind == f.kind
-  of tyBool, tyChar, tyInt..tyUInt64:
-    let ak = a.skipTypes(ignorableForArgType)
-    result = ak.kind == f.kind or ak.kind == tyOrdinal or
-       (ak.kind == tyGenericParam and ak.len > 0 and ak[0].kind == tyOrdinal)
-  of tyConcept:
-    let oldLen = m.inferred.len
-    let oldPotentialImplementation = m.potentialImplementation
-    m.potentialImplementation = a
-    result = conceptMatchNode(c, f.n.lastSon, m)
-    m.potentialImplementation = oldPotentialImplementation
-    if not result:
-      m.inferred.setLen oldLen
-  of tyArray, tyTuple, tyVarargs, tyOpenArray, tyRange, tySequence, tyRef, tyPtr,
-     tyGenericInst:
-    let ak = a.skipTypes(ignorableForArgType - {f.kind})
-    if ak.kind == f.kind and f.len == ak.len:
-      for i in 0..<ak.len:
-        if not matchType(c, f[i], ak[i], m): return false
-      return true
-  of tyOr:
-    let oldLen = m.inferred.len
-    if a.kind == tyOr:
-      # say the concept requires 'int|float|string' if the potentialImplementation
-      # says 'int|string' that is good enough.
-      var covered = 0
-      for i in 0..<f.len:
-        for j in 0..<a.len:
-          let oldLenB = m.inferred.len
-          let r = matchType(c, f[i], a[j], m)
-          if r:
-            inc covered
-            break
-          m.inferred.setLen oldLenB
-
-      result = covered >= a.len
-      if not result:
-        m.inferred.setLen oldLen
-    else:
-      for i in 0..<f.len:
-        result = matchType(c, f[i], a, m)
-        if result: break # and remember the binding!
-        m.inferred.setLen oldLen
-  of tyNot:
-    if a.kind == tyNot:
-      result = matchType(c, f[0], a[0], m)
-    else:
-      let oldLen = m.inferred.len
-      result = not matchType(c, f[0], a, m)
-      m.inferred.setLen oldLen
-  of tyAnything:
-    result = true
-  of tyOrdinal:
-    result = isOrdinalType(a, allowEnumWithHoles = false) or a.kind == tyGenericParam
-  else:
-    result = false
-
-proc matchReturnType(c: PContext; f, a: PType; m: var MatchCon): bool =
-  ## Like 'matchType' but with extra logic dealing with proc return types
-  ## which can be nil or the 'void' type.
-  if f.isEmptyType:
-    result = a.isEmptyType
-  elif a == nil:
-    result = false
-  else:
-    result = matchType(c, f, a, m)
-
-proc matchSym(c: PContext; candidate: PSym, n: PNode; m: var MatchCon): bool =
-  ## Checks if 'candidate' matches 'n' from the concept body. 'n' is a nkProcDef
-  ## or similar.
-
-  # watch out: only add bindings after a completely successful match.
-  let oldLen = m.inferred.len
-
-  let can = candidate.typ.n
-  let con = n[0].sym.typ.n
-
-  if can.len < con.len:
-    # too few arguments, cannot be a match:
-    return false
-
-  let common = min(can.len, con.len)
-  for i in 1 ..< common:
-    if not matchType(c, con[i].typ, can[i].typ, m):
-      m.inferred.setLen oldLen
-      return false
-
-  if not matchReturnType(c, n[0].sym.typ.sons[0], candidate.typ.sons[0], m):
-    m.inferred.setLen oldLen
-    return false
-
-  # all other parameters have to be optional parameters:
-  for i in common ..< can.len:
-    assert can[i].kind == nkSym
-    if can[i].sym.ast == nil:
-      # has too many arguments one of which is not optional:
-      m.inferred.setLen oldLen
-      return false
-
-  return true
-
-proc matchSyms(c: PContext, n: PNode; kinds: set[TSymKind]; m: var MatchCon): bool =
-  ## Walk the current scope, extract candidates which the same name as 'n[namePos]',
-  ## 'n' is the nkProcDef or similar from the concept that we try to match.
-  let candidates = searchInScopesFilterBy(c, n[namePos].sym.name, kinds)
-  for candidate in candidates:
-    #echo "considering ", typeToString(candidate.typ), " ", candidate.magic
-    m.magic = candidate.magic
-    if matchSym(c, candidate, n, m): return true
-  result = false
-
-proc conceptMatchNode(c: PContext; n: PNode; m: var MatchCon): bool =
-  ## Traverse the concept's AST ('n') and see if every declaration inside 'n'
-  ## can be matched with the current scope.
-  case n.kind
-  of nkStmtList, nkStmtListExpr:
-    for i in 0..<n.len:
-      if not conceptMatchNode(c, n[i], m):
-        return false
-    return true
-  of nkProcDef, nkFuncDef:
-    # procs match any of: proc, template, macro, func, method, converter.
-    # The others are more specific.
-    # XXX: Enforce .noSideEffect for 'nkFuncDef'? But then what are the use cases...
-    const filter = {skProc, skTemplate, skMacro, skFunc, skMethod, skConverter}
-    result = matchSyms(c, n, filter, m)
-  of nkTemplateDef:
-    result = matchSyms(c, n, {skTemplate}, m)
-  of nkMacroDef:
-    result = matchSyms(c, n, {skMacro}, m)
-  of nkConverterDef:
-    result = matchSyms(c, n, {skConverter}, m)
-  of nkMethodDef:
-    result = matchSyms(c, n, {skMethod}, m)
-  of nkIteratorDef:
-    result = matchSyms(c, n, {skIterator}, m)
-  else:
-    # error was reported earlier.
-    result = false
-
-proc conceptMatch*(c: PContext; concpt, arg: PType; bindings: var TIdTable; invocation: PType): bool =
-  ## Entry point from sigmatch. 'concpt' is the concept we try to match (here still a PType but
-  ## we extract its AST via 'concpt.n.lastSon'). 'arg' is the type that might fullfill the
-  ## concept's requirements. If so, we return true and fill the 'bindings' with pairs of
-  ## (typeVar, instance) pairs. ('typeVar' is usually simply written as a generic 'T'.)
-  ## 'invocation' can be nil for atomic concepts. For non-atomic concepts, it contains the
-  ## 'C[S, T]' parent type that we look for. We need this because we need to store bindings
-  ## for 'S' and 'T' inside 'bindings' on a successful match. It is very important that
-  ## we do not add any bindings at all on an unsuccessful match!
-  var m = MatchCon(inferred: @[], potentialImplementation: arg)
-  result = conceptMatchNode(c, concpt.n.lastSon, m)
-  if result:
-    for (a, b) in m.inferred:
-      if b.kind == tyGenericParam:
-        var dest = b
-        while true:
-          dest = existingBinding(m, dest)
-          if dest == nil or dest.kind != tyGenericParam: break
-        if dest != nil:
-          bindings.idTablePut(a, dest)
-          when logBindings: echo "A bind ", a, " ", dest
-      else:
-        bindings.idTablePut(a, b)
-        when logBindings: echo "B bind ", a, " ", b
-    # we have a match, so bind 'arg' itself to 'concpt':
-    bindings.idTablePut(concpt, arg)
-    # invocation != nil means we have a non-atomic concept:
-    if invocation != nil and arg.kind == tyGenericInst and invocation.len == arg.len-1:
-      # bind even more generic parameters
-      assert invocation.kind == tyGenericInvocation
-      for i in 1 ..< invocation.len:
-        bindings.idTablePut(invocation[i], arg[i])

compiler/condsyms.nim

@@ -34,58 +34,56 @@ proc countDefinedSymbols*(symbols: StringTableRef): int =
 proc initDefines*(symbols: StringTableRef) =
   # for bootstrapping purposes and old code:
   template defineSymbol(s) = symbols.defineSymbol(s)
-  defineSymbol("nimhygiene") # deadcode
-  defineSymbol("niminheritable") # deadcode
-  defineSymbol("nimmixin") # deadcode
-  defineSymbol("nimeffects") # deadcode
-  defineSymbol("nimbabel") # deadcode
-  defineSymbol("nimcomputedgoto") # deadcode
-  defineSymbol("nimunion") # deadcode
-  defineSymbol("nimnewshared") # deadcode
-  defineSymbol("nimNewTypedesc") # deadcode
-  defineSymbol("nimrequiresnimframe") # deadcode
-  defineSymbol("nimparsebiggestfloatmagic") # deadcode
-  defineSymbol("nimalias") # deadcode
-  defineSymbol("nimlocks") # deadcode
-  defineSymbol("nimnode") # deadcode pending `nimnode` reference in opengl package
-    # refs https://github.com/nim-lang/opengl/pull/79
-  defineSymbol("nimvarargstyped") # deadcode
-  defineSymbol("nimtypedescfixed") # deadcode
-  defineSymbol("nimKnowsNimvm") # deadcode
-  defineSymbol("nimArrIdx") # deadcode
-  defineSymbol("nimHasalignOf") # deadcode
-  defineSymbol("nimDistros") # deadcode
-  defineSymbol("nimHasCppDefine") # deadcode
-  defineSymbol("nimGenericInOutFlags") # deadcode
-  when false: defineSymbol("nimHasOpt") # deadcode
-  defineSymbol("nimNoArrayToCstringConversion") # deadcode
-  defineSymbol("nimHasRunnableExamples") # deadcode
-  defineSymbol("nimNewDot") # deadcode
-  defineSymbol("nimHasNilChecks") # deadcode
-  defineSymbol("nimSymKind") # deadcode
-  defineSymbol("nimVmEqIdent") # deadcode
-  defineSymbol("nimNoNil") # deadcode
-  defineSymbol("nimNoZeroTerminator") # deadcode
-  defineSymbol("nimNotNil") # deadcode
-  defineSymbol("nimVmExportFixed") # deadcode
-  defineSymbol("nimHasSymOwnerInMacro") # deadcode
-  defineSymbol("nimNewRuntime") # deadcode
-  defineSymbol("nimIncrSeqV3") # xxx: turn this into deadcode
-  defineSymbol("nimAshr") # deadcode
-  defineSymbol("nimNoNilSeqs") # deadcode
-  defineSymbol("nimNoNilSeqs2") # deadcode
-  defineSymbol("nimHasUserErrors") # deadcode
-  defineSymbol("nimUncheckedArrayTyp") # deadcode
-  defineSymbol("nimHasTypeof") # deadcode
-  defineSymbol("nimErrorProcCanHaveBody") # deadcode
-  defineSymbol("nimHasInstantiationOfInMacro") # deadcode
-  defineSymbol("nimHasHotCodeReloading") # deadcode
-  defineSymbol("nimHasNilSeqs") # deadcode
-  defineSymbol("nimHasSignatureHashInMacro") # deadcode
-  defineSymbol("nimHasDefault") # deadcode
-  defineSymbol("nimMacrosSizealignof") # deadcode
-
-  # > 0.20.0
+  defineSymbol("nimhygiene")
+  defineSymbol("niminheritable")
+  defineSymbol("nimmixin")
+  defineSymbol("nimeffects")
+  defineSymbol("nimbabel")
+  defineSymbol("nimcomputedgoto")
+  defineSymbol("nimunion")
+  defineSymbol("nimnewshared")
+  defineSymbol("nimNewTypedesc")
+  defineSymbol("nimrequiresnimframe")
+  defineSymbol("nimparsebiggestfloatmagic")
+  defineSymbol("nimalias")
+  defineSymbol("nimlocks")
+  defineSymbol("nimnode")
+  defineSymbol("nimvarargstyped")
+  defineSymbol("nimtypedescfixed")
+  defineSymbol("nimKnowsNimvm")
+  defineSymbol("nimArrIdx")
+  defineSymbol("nimHasalignOf")
+  defineSymbol("nimDistros")
+  defineSymbol("nimHasCppDefine")
+  defineSymbol("nimGenericInOutFlags")
+  when false: defineSymbol("nimHasOpt")
+  defineSymbol("nimNoArrayToCstringConversion")
+  defineSymbol("nimNewRoof")
+  defineSymbol("nimHasRunnableExamples")
+  defineSymbol("nimNewDot")
+  defineSymbol("nimHasNilChecks")
+  defineSymbol("nimSymKind")
+  defineSymbol("nimVmEqIdent")
+  defineSymbol("nimNoNil")
+  defineSymbol("nimNoZeroTerminator")
+  defineSymbol("nimNotNil")
+  defineSymbol("nimVmExportFixed")
+  defineSymbol("nimHasSymOwnerInMacro")
+  defineSymbol("nimNewRuntime")
+  defineSymbol("nimIncrSeqV3")
+  defineSymbol("nimAshr")
+  defineSymbol("nimNoNilSeqs")
+  defineSymbol("nimNoNilSeqs2")
+  defineSymbol("nimHasUserErrors")
+  defineSymbol("nimUncheckedArrayTyp")
+  defineSymbol("nimHasTypeof")
+  defineSymbol("nimErrorProcCanHaveBody")
+  defineSymbol("nimHasInstantiationOfInMacro")
+  defineSymbol("nimHasHotCodeReloading")
+  defineSymbol("nimHasNilSeqs")
+  defineSymbol("nimHasSignatureHashInMacro")
+  defineSymbol("nimHasDefault")
+  defineSymbol("nimMacrosSizealignof")
   defineSymbol("nimNoZeroExtendMagic")
   defineSymbol("nimMacrosGetNodeId")
   for f in Feature:
@@ -125,20 +123,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasStacktracesModule")
   defineSymbol("nimHasEffectTraitsModule")
   defineSymbol("nimHasCastPragmaBlocks")
-  defineSymbol("nimHasDeclaredLocs")
   defineSymbol("nimHasJsBigIntBackend")
-  defineSymbol("nimHasWarningAsError")
-  defineSymbol("nimHasHintAsError")
-  defineSymbol("nimHasSpellSuggest")
-  defineSymbol("nimHasCustomLiterals")
-  defineSymbol("nimHasUnifiedTuple")
-  defineSymbol("nimHasIterable")
-  defineSymbol("nimHasTypeofVoid")
-  defineSymbol("nimHasDragonBox")
-  defineSymbol("nimHasHintAll")
-  defineSymbol("nimHasTrace")
-  defineSymbol("nimHasEffectsOf")
-
-  defineSymbol("nimHasEnforceNoRaises")
-  defineSymbol("nimHasWarnBareExcept")
-  defineSymbol("nimHasTopDownInference")

compiler/debugutils.nim

@@ -1,56 +0,0 @@
-##[
-Utilities to help with debugging nim compiler.
-
-Experimental API, subject to change.
-]##
-
-#[
-## example
-useful debugging flags:
---stacktrace -d:debug -d:nimDebugUtils
- nim c -o:bin/nim_temp --stacktrace -d:debug -d:nimDebugUtils compiler/nim
-
-## future work
-* expose and improve astalgo.debug, replacing it by std/prettyprints,
-  refs https://github.com/nim-lang/RFCs/issues/385
-]#
-
-import options
-import std/wrapnils
-export wrapnils
-  # allows using things like: `?.n.sym.typ.len`
-
-import std/stackframes
-export stackframes
-  # allows using things like: `setFrameMsg c.config$n.info & " " & $n.kind`
-  # which doesn't log, but augments stacktrace with side channel information
-
-var conf0: ConfigRef
-
-proc onNewConfigRef*(conf: ConfigRef) {.inline.} =
-  ## Caches `conf`, which can be retrieved with `getConfigRef`.
-  ## This avoids having to forward `conf` all the way down the call chain to
-  ## procs that need it during a debugging session.
-  conf0 = conf
-
-proc getConfigRef*(): ConfigRef =
-  ## nil, if -d:nimDebugUtils wasn't specified
-  result = conf0
-
-proc isCompilerDebug*(): bool =
-  ##[
-  Provides a simple way for user code to enable/disable logging in the compiler
-  in a granular way. This can then be used in the compiler as follows:
-  ```nim
-  if isCompilerDebug():
-    echo ?.n.sym.typ.len
-  ```
-  ]##
-  runnableExamples:
-    proc main =
-      echo 2
-      {.define(nimCompilerDebug).}
-      echo 3.5 # code section in which `isCompilerDebug` will be true
-      {.undef(nimCompilerDebug).}
-      echo 'x'
-  conf0.isDefined("nimCompilerDebug")

compiler/depends.nim

@@ -9,12 +9,10 @@
 
 # This module implements a dependency file generator.
 
-import options, ast, ropes, passes, pathutils, msgs, lineinfos
+import
+  options, ast, ropes, idents, passes, modulepaths, pathutils
 
-import modulegraphs
-
-import std/[os, strutils, parseutils]
-import std/private/globs
+from modulegraphs import ModuleGraph, PPassContext
 
 type
   TGen = object of PPassContext
@@ -27,57 +25,9 @@ type
     dotGraph: Rope
 
 proc addDependencyAux(b: Backend; importing, imported: string) =
-  b.dotGraph.addf("\"$1\" -> \"$2\";$n", [rope(importing), rope(imported)])
+  b.dotGraph.addf("$1 -> \"$2\";$n", [rope(importing), rope(imported)])
   # s1 -> s2_4[label="[0-9]"];
 
-proc toNimblePath(s: string, isStdlib: bool): string =
-  const stdPrefix = "std/"
-  const pkgPrefix = "pkg/"
-  if isStdlib:
-    let sub = "lib/"
-    var start = s.find(sub)
-    if start < 0:
-      doAssert false
-    else:
-      start += sub.len
-      let base = s[start..^1]
-
-      if base.startsWith("system") or base.startsWith("std"):
-        result = base
-      else:
-        for dir in stdlibDirs:
-          if base.startsWith(dir):
-            return stdPrefix & base.splitFile.name
-
-        result = stdPrefix & base
-  else:
-    var sub = getEnv("NIMBLE_DIR")
-    if sub.len == 0:
-      sub = ".nimble/pkgs/"
-    else:
-      sub.add "/pkgs/"
-    var start = s.find(sub)
-    if start < 0:
-      sub[^1] = '2'
-      sub.add '/'
-      start = s.find(sub) # /pkgs2
-      if start < 0:
-        return s
-
-    start += sub.len
-    start += skipUntil(s, '/', start)
-    start += 1
-    result = pkgPrefix & s[start..^1]
-
-proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) =
-  doAssert n.kind == nkSym, $n.kind
-
-  let path = splitFile(toProjPath(g.config, n.sym.position.FileIndex))
-  let modulePath = splitFile(toProjPath(g.config, g.module.position.FileIndex))
-  let parent = nativeToUnixPath(modulePath.dir / modulePath.name).toNimblePath(belongsToStdlib(g.graph, g.module))
-  let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym))
-  addDependencyAux(b, parent, child)
-
 proc addDotDependency(c: PPassContext, n: PNode): PNode =
   result = n
   let g = PGen(c)
@@ -85,9 +35,11 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode =
   case n.kind
   of nkImportStmt:
     for i in 0..<n.len:
-      addDependency(c, g, b, n[i])
+      var imported = getModuleName(g.config, n[i])
+      addDependencyAux(b, g.module.name.s, imported)
   of nkFromStmt, nkImportExceptStmt:
-    addDependency(c, g, b, n[0])
+    var imported = getModuleName(g.config, n[0])
+    addDependencyAux(b, g.module.name.s, imported)
   of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr:
     for i in 0..<n.len: discard addDotDependency(c, n[i])
   else:
@@ -102,7 +54,7 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) =
 when not defined(nimHasSinkInference):
   {.pragma: nosinks.}
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext {.nosinks.} =
   var g: PGen
   new(g)
   g.module = module

compiler/dfa.nim

@@ -29,8 +29,9 @@
 ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
 ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
 
-import ast, intsets, lineinfos, renderer
-import std/private/asciitables
+import ast, types, intsets, lineinfos, renderer, asciitables
+
+from patterns import sameTrees
 
 type
   InstrKind* = enum
@@ -308,11 +309,6 @@ when true:
     # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
     # through the loop. We need to prove this is correct for our
     # purposes. But Herb Sutter claims it is. (Proof by authority.)
-    #
-    # EDIT: Actually, we only need to unroll 2 times
-    # because Nim doesn't have a way of breaking/goto-ing into
-    # a loop iteration. Unrolling 2 times is much better for compile
-    # times of nested loops than 3 times, so we do that here.
     #[
     while cond:
       body
@@ -352,12 +348,12 @@ when true:
       # 'while true' is an idiom in Nim and so we produce
       # better code for it:
       withBlock(nil):
-        for i in 0..1:
+        for i in 0..2:
           c.gen(n[1])
     else:
       withBlock(nil):
-        var endings: array[2, TPosition]
-        for i in 0..1:
+        var endings: array[3, TPosition]
+        for i in 0..2:
           c.gen(n[0])
           endings[i] = c.forkI(n)
           c.gen(n[1])
@@ -460,10 +456,6 @@ proc genCase(c: var Con; n: PNode) =
   let isExhaustive = skipTypes(n[0].typ,
     abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
 
-  # we generate endings as a set of chained gotos, this is a bit awkward but it
-  # ensures when recursively traversing the CFG for various analysis, we don't
-  # artificially extended the life of each branch (for the purposes of DFA)
-  # beyond the minimum amount.
   var endings: seq[TPosition] = @[]
   c.gen(n[0])
   for i in 1..<n.len:
@@ -471,14 +463,13 @@ proc genCase(c: var Con; n: PNode) =
     if it.len == 1 or (i == n.len-1 and isExhaustive):
       # treat the last branch as 'else' if this is an exhaustive case statement.
       c.gen(it.lastSon)
-      if endings.len != 0:
-        c.patch(endings[^1])
     else:
       forkT(it.lastSon):
         c.gen(it.lastSon)
-        if endings.len != 0:
-          c.patch(endings[^1])
         endings.add c.gotoI(it.lastSon)
+  for i in countdown(endings.high, 0):
+    let endPos = endings[i]
+    c.patch(endPos)
 
 proc genBlock(c: var Con; n: PNode) =
   withBlock(n[0].sym):
@@ -582,82 +573,49 @@ proc skipConvDfa*(n: PNode): PNode =
       result = result[1]
     else: break
 
-type AliasKind* = enum
-  yes, no, maybe
+proc aliases*(obj, field: PNode): bool =
+  var n = field
+  var obj = obj
+  while true:
+    case obj.kind
+    of {nkObjDownConv, nkObjUpConv, nkAddr, nkHiddenAddr, nkDerefExpr, nkHiddenDeref}:
+      obj = obj[0]
+    of PathKinds1:
+      obj = obj[1]
+    else: break
+  while true:
+    if sameTrees(obj, n): return true
+    case n.kind
+    of PathKinds0:
+      n = n[0]
+    of PathKinds1:
+      n = n[1]
+    else: break
 
-proc aliases*(obj, field: PNode): AliasKind =
-  # obj -> field:
-  # x -> x: true
-  # x -> x.f: true
-  # x.f -> x: false
-  # x.f -> x.f: true
-  # x.f -> x.v: false
-  # x -> x[0]: true
-  # x[0] -> x: false
-  # x[0] -> x[0]: true
-  # x[0] -> x[1]: false
-  # x -> x[i]: true
-  # x[i] -> x: false
-  # x[i] -> x[i]: maybe; Further analysis could make this return true when i is a runtime-constant
-  # x[i] -> x[j]: maybe; also returns maybe if only one of i or j is a compiletime-constant
-  template collectImportantNodes(result, n) =
-    var result: seq[PNode]
-    var n = n
-    while true:
-      case n.kind
-      of PathKinds0 - {nkDotExpr, nkBracketExpr}:
-        n = n[0]
-      of PathKinds1:
-        n = n[1]
-      of nkDotExpr, nkBracketExpr:
-        result.add n
-        n = n[0]
-      of nkSym:
-        result.add n; break
-      else: return no
+type InstrTargetKind* = enum
+  None, Full, Partial
 
-  collectImportantNodes(objImportantNodes, obj)
-  collectImportantNodes(fieldImportantNodes, field)
-
-  # If field is less nested than obj, then it cannot be part of/aliased by obj
-  if fieldImportantNodes.len < objImportantNodes.len: return no
-
-  result = yes
-  for i in 1..objImportantNodes.len:
-    # We compare the nodes leading to the location of obj and field
-    # with each other.
-    # We continue until they diverge, in which case we return no, or
-    # until we reach the location of obj, in which case we do not need
-    # to look further, since field must be part of/aliased by obj now.
-    # If we encounter an element access using an index which is a runtime value,
-    # we simply return maybe instead of yes; should further nodes not diverge.
-    let currFieldPath = fieldImportantNodes[^i]
-    let currObjPath = objImportantNodes[^i]
-
-    if currFieldPath.kind != currObjPath.kind:
-      return no
-
-    case currFieldPath.kind
-    of nkSym:
-      if currFieldPath.sym != currObjPath.sym: return no
-    of nkDotExpr:
-      if currFieldPath[1].sym != currObjPath[1].sym: return no
-    of nkBracketExpr:
-      if currFieldPath[1].kind in nkLiterals and currObjPath[1].kind in nkLiterals:
-        if currFieldPath[1].intVal != currObjPath[1].intVal:
-          return no
-      else:
-        result = maybe
-    else: assert false # unreachable
+proc instrTargets*(insloc, loc: PNode): InstrTargetKind =
+  if sameTrees(insloc, loc) or insloc.aliases(loc):
+    Full    # x -> x; x -> x.f
+  elif loc.aliases(insloc):
+    Partial # x.f -> x
+  else: None
 
 proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
   var n = orig
   while true:
     case n.kind
-    of PathKinds0 - {nkHiddenDeref, nkDerefExpr}:
+    of PathKinds0 - {nkBracketExpr, nkHiddenDeref, nkDerefExpr}:
       n = n[0]
     of PathKinds1:
       n = n[1]
+    of nkBracketExpr:
+      # in a[i] the 'i' must be known
+      if n.len > 1 and n[1].kind in {nkCharLit..nkUInt64Lit}:
+        n = n[0]
+      else:
+        return false
     of nkHiddenDeref, nkDerefExpr:
       # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
       # pointer indirection.
@@ -699,10 +657,9 @@ proc skipTrivials(c: var Con, n: PNode): PNode =
 proc genUse(c: var Con; orig: PNode) =
   let n = c.skipTrivials(orig)
 
-  if n.kind == nkSym:
-    if n.sym.kind in InterestingSyms:
-      c.code.add Instr(n: orig, kind: use)
-  else:
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    c.code.add Instr(n: orig, kind: use)
+  elif n.kind in nkCallKinds:
     gen(c, n)
 
 proc genDef(c: var Con; orig: PNode) =
@@ -772,12 +729,6 @@ proc gen(c: var Con; n: PNode) =
   of nkCharLit..nkNilLit: discard
   of nkAsgn, nkFastAsgn:
     gen(c, n[1])
-
-    if n[0].kind in PathKinds0:
-      let a = c.skipTrivials(n[0])
-      if a.kind in nkCallKinds:
-        gen(c, a)
-
     # watch out: 'obj[i].f2 = value' sets 'f2' but
     # "uses" 'i'. But we are only talking about builtin array indexing so
     # it doesn't matter and 'x = 34' is NOT a usage of 'x'.

compiler/docgen2.nim

@@ -11,7 +11,7 @@
 # semantic checking.
 
 import
-  options, ast, msgs, passes, docgen, lineinfos, pathutils, packages
+  options, ast, msgs, passes, docgen, lineinfos, pathutils
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -23,7 +23,7 @@ type
   PGen = ref TGen
 
 proc shouldProcess(g: PGen): bool =
-  (optWholeProject in g.doc.conf.globalOptions and g.doc.conf.belongsToProjectPackage(g.module)) or
+  (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or
       sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
 
 template closeImpl(body: untyped) {.dirty.} =
@@ -31,7 +31,6 @@ template closeImpl(body: untyped) {.dirty.} =
   let useWarning = sfMainModule notin g.module.flags
   let groupedToc = true
   if shouldProcess(g):
-    finishGenerateDoc(g.doc)
     body
     try:
       generateIndex(g.doc)
@@ -69,18 +68,13 @@ template myOpenImpl(ext: untyped) {.dirty.} =
   g.doc = d
   result = g
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
   myOpenImpl(HtmlExt)
 
-proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
-  myOpenImpl(TexExt)
-
-proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc myOpenJson(graph: ModuleGraph; module: PSym): PPassContext =
   myOpenImpl(JsonExt)
 
 const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
-const docgen2TexPass* = makePass(open = myOpenTex, process = processNode,
-                                 close = close)
 const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
                                   close = closeJson)
 

compiler/enumtostr.nim

@@ -1,16 +1,16 @@
 
 import ast, idents, lineinfos, modulegraphs, magicsys
 
-proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
-  result = newSym(skProc, getIdent(g.cache, "$"), nextSymId idgen, t.owner, info)
+proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
+  result = newSym(skProc, getIdent(g.cache, "$"), t.owner, info)
 
-  let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info)
+  let dest = newSym(skParam, getIdent(g.cache, "e"), result, info)
   dest.typ = t
 
-  let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), result, info)
   res.typ = getSysType(g, info, tyString)
 
-  result.typ = newType(tyProc, nextTypeId idgen, t.owner)
+  result.typ = newType(tyProc, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
   result.typ.n.add newNodeI(nkEffectList, info)
@@ -62,16 +62,16 @@ proc searchObjCase(t: PType; field: PSym): PNode =
     result = searchObjCase(t[0].skipTypes({tyAlias, tyGenericInst, tyRef, tyPtr}), field)
   doAssert result != nil
 
-proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
-  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), nextSymId idgen, t.owner, info)
+proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph): PSym =
+  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), t.owner, info)
 
-  let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info)
+  let dest = newSym(skParam, getIdent(g.cache, "e"), result, info)
   dest.typ = field.typ
 
-  let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), result, info)
   res.typ = getSysType(g, info, tyUInt8)
 
-  result.typ = newType(tyProc, nextTypeId idgen, t.owner)
+  result.typ = newType(tyProc, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
   result.typ.n.add newNodeI(nkEffectList, info)

compiler/errorhandling.nim

@@ -1,79 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2021 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module contains support code for new-styled error
-## handling via an `nkError` node kind.
-
-import ast, renderer, options, strutils, types
-
-type
-  ErrorKind* = enum ## expand as you need.
-    RawTypeMismatchError
-    ExpressionCannotBeCalled
-    CustomError
-    WrongNumberOfArguments
-    AmbiguousCall
-
-proc errorSubNode*(n: PNode): PNode =
-  case n.kind
-  of nkEmpty..nkNilLit:
-    result = nil
-  of nkError:
-    result = n
-  else:
-    result = nil
-    for i in 0..<n.len:
-      result = errorSubNode(n[i])
-      if result != nil: break
-
-proc newError*(wrongNode: PNode; k: ErrorKind; args: varargs[PNode]): PNode =
-  assert wrongNode.kind != nkError
-  let innerError = errorSubNode(wrongNode)
-  if innerError != nil:
-    return innerError
-  result = newNodeIT(nkError, wrongNode.info, newType(tyError, ItemId(module: -1, item: -1), nil))
-  result.add wrongNode
-  result.add newIntNode(nkIntLit, ord(k))
-  for a in args: result.add a
-
-proc newError*(wrongNode: PNode; msg: string): PNode =
-  assert wrongNode.kind != nkError
-  let innerError = errorSubNode(wrongNode)
-  if innerError != nil:
-    return innerError
-  result = newNodeIT(nkError, wrongNode.info, newType(tyError, ItemId(module: -1, item: -1), nil))
-  result.add wrongNode
-  result.add newIntNode(nkIntLit, ord(CustomError))
-  result.add newStrNode(msg, wrongNode.info)
-
-proc errorToString*(config: ConfigRef; n: PNode): string =
-  assert n.kind == nkError
-  assert n.len > 1
-  let wrongNode = n[0]
-  case ErrorKind(n[1].intVal)
-  of RawTypeMismatchError:
-    result = "type mismatch"
-  of ExpressionCannotBeCalled:
-    result = "expression '$1' cannot be called" % wrongNode[0].renderTree
-  of CustomError:
-    result = n[2].strVal
-  of WrongNumberOfArguments:
-    result = "wrong number of arguments"
-  of AmbiguousCall:
-    let a = n[2].sym
-    let b = n[3].sym
-    var args = "("
-    for i in 1..<wrongNode.len:
-      if i > 1: args.add(", ")
-      args.add(typeToString(wrongNode[i].typ))
-    args.add(")")
-    result = "ambiguous call; both $1 and $2 match for: $3" % [
-      getProcHeader(config, a),
-      getProcHeader(config, b),
-      args]

compiler/evalffi.nim

@@ -92,7 +92,7 @@ proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type =
     else: result = nil
   of tyFloat, tyFloat64: result = addr libffi.type_double
   of tyFloat32: result = addr libffi.type_float
-  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCstring, tySequence, tyString, tyUntyped,
+  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyUntyped,
      tyTyped, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
     result = addr libffi.type_pointer
   of tyDistinct, tyAlias, tySink:
@@ -110,8 +110,8 @@ proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI
   else:
     globalError(conf, info, "cannot map calling convention to FFI")
 
-template rd(typ, p: untyped): untyped = (cast[ptr typ](p))[]
-template wr(typ, p, v: untyped): untyped = (cast[ptr typ](p))[] = v
+template rd(T, p: untyped): untyped = (cast[ptr T](p))[]
+template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v
 template `+!`(x, y: untyped): untyped =
   cast[pointer](cast[ByteAddress](x) + y)
 
@@ -177,8 +177,8 @@ const maxPackDepth = 20
 var packRecCheck = 0
 
 proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
-  template awr(typ, v: untyped): untyped =
-    wr(typ, res, v)
+  template awr(T, v: untyped): untyped =
+    wr(T, res, v)
 
   case typ.kind
   of tyBool: awr(bool, v.intVal != 0)
@@ -205,7 +205,7 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
   of tyFloat32: awr(float32, v.floatVal)
   of tyFloat64: awr(float64, v.floatVal)
 
-  of tyPointer, tyProc,  tyCstring, tyString:
+  of tyPointer, tyProc,  tyCString, tyString:
     if v.kind == nkNilLit:
       # nothing to do since the memory is 0 initialized anyway
       discard
@@ -386,7 +386,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
     result = unpackObject(conf, x, typ, n)
   of tyArray:
     result = unpackArray(conf, x, typ, n)
-  of tyCstring, tyString:
+  of tyCString, tyString:
     let p = rd(cstring, x)
     if p.isNil:
       setNil()
@@ -402,7 +402,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
 
 proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode =
   if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyLent, tyPointer,
-                                           tyProc, tyCstring, tyString,
+                                           tyProc, tyCString, tyString,
                                            tySequence}:
     result = newNodeIT(x.kind, x.info, destTyp)
     result.intVal = x.intVal

compiler/evaltempl.nim

@@ -22,7 +22,6 @@ type
     config: ConfigRef
     ic: IdentCache
     instID: int
-    idgen: IdGenerator
 
 proc copyNode(ctx: TemplCtx, a, b: PNode): PNode =
   result = copyNode(a)
@@ -49,7 +48,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
         internalAssert c.config, sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
         if x == nil:
-          x = copySym(s, nextSymId(c.idgen))
+          x = copySym(s)
           # sem'check needs to set the owner properly later, see bug #9476
           x.owner = nil # c.genSymOwner
           #if x.kind == skParam and x.owner.kind == skModule:
@@ -83,14 +82,10 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
         not c.isDeclarative:
       c.isDeclarative = true
       isDeclarative = true
-    if (not c.isDeclarative) and templ.kind in nkCallKinds and isRunnableExamples(templ[0]):
-      # fixes bug #16993, bug #18054
-      discard
-    else:
-      var res = copyNode(c, templ, actual)
-      for i in 0..<templ.len:
-        evalTemplateAux(templ[i], actual, c, res)
-      result.add res
+    var res = copyNode(c, templ, actual)
+    for i in 0..<templ.len:
+      evalTemplateAux(templ[i], actual, c, res)
+    result.add res
     if isDeclarative: c.isDeclarative = false
 
 const
@@ -173,7 +168,6 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
 proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
                    conf: ConfigRef;
                    ic: IdentCache; instID: ref int;
-                   idgen: IdGenerator;
                    fromHlo=false): PNode =
   inc(conf.evalTemplateCounter)
   if conf.evalTemplateCounter > evalTemplateLimit:
@@ -189,9 +183,8 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   ctx.ic = ic
   initIdTable(ctx.mapping)
   ctx.instID = instID[]
-  ctx.idgen = idgen
 
-  let body = tmpl.ast[bodyPos]
+  let body = tmpl.getBody
   #echo "instantion of ", renderTree(body, {renderIds})
   if isAtom(body):
     result = newNodeI(nkPar, body.info)

compiler/extccomp.nim

@@ -12,9 +12,9 @@
 # from a lineinfos file, to provide generalized procedures to compile
 # nim files.
 
-import ropes, platform, condsyms, options, msgs, lineinfos, pathutils, modulepaths
-
-import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar, parseutils]
+import
+  ropes, os, strutils, osproc, platform, condsyms, options, msgs,
+  lineinfos, std / sha1, streams, pathutils, sequtils, times, strtabs
 
 type
   TInfoCCProp* = enum         # properties of the C compiler:
@@ -124,8 +124,8 @@ compiler llvmGcc:
   result.name = "llvm_gcc"
   result.compilerExe = "llvm-gcc"
   result.cppCompiler = "llvm-g++"
-  when defined(macosx) or defined(openbsd):
-    # `llvm-ar` not available
+  when defined(macosx):
+    # OS X has no 'llvm-ar' tool:
     result.buildLib = "ar rcs $libfile $objfiles"
   else:
     result.buildLib = "llvm-ar rcs $libfile $objfiles"
@@ -150,7 +150,7 @@ compiler vcc:
     compileTmpl: "/c$vccplatform $options $include /nologo /Fo$objfile $file",
     buildGui: " /SUBSYSTEM:WINDOWS user32.lib ",
     buildDll: " /LD",
-    buildLib: "vccexe --command:lib$vccplatform /nologo /OUT:$libfile $objfiles",
+    buildLib: "lib /OUT:$libfile $objfiles",
     linkerExe: "cl",
     linkTmpl: "$builddll$vccplatform /Fe$exefile $objfiles $buildgui /nologo $options",
     includeCmd: " /I",
@@ -252,7 +252,7 @@ compiler envcc:
     buildGui: "",
     buildDll: " -shared ",
     buildLib: "", # XXX: not supported yet
-    linkerExe: "",
+    linkerExe: "cc",
     linkTmpl: "-o $exefile $buildgui $builddll $objfiles $options",
     includeCmd: " -I",
     linkDirCmd: "", # XXX: not supported yet
@@ -281,11 +281,6 @@ const
 
   hExt* = ".h"
 
-template writePrettyCmdsStderr(cmd) =
-  if cmd.len > 0:
-    flushDot(conf)
-    stderr.writeLine(cmd)
-
 proc nameToCC*(name: string): TSystemCC =
   ## Returns the kind of compiler referred to by `name`, or ccNone
   ## if the name doesn't refer to any known compiler.
@@ -367,7 +362,6 @@ proc initVars*(conf: ConfigRef) =
 
 proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile,
                         createSubDir: bool = true): AbsoluteFile =
-  ## Generate the absolute file path to the generated modules.
   result = completeGeneratedFilePath(conf, cfile, createSubDir)
 
 proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile =
@@ -378,7 +372,7 @@ proc addFileToCompile*(conf: ConfigRef; cf: Cfile) =
   conf.toCompile.add(cf)
 
 proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) =
-  let key = completeCfilePath(conf, mangleModuleName(conf, nimfile).AbsoluteFile).string
+  let key = completeCfilePath(conf, withPackageName(conf, nimfile)).string
   var value = conf.cfileSpecificOptions.getOrDefault(key)
   if strutils.find(value, option, 0) < 0:
     addOpt(value, option)
@@ -484,10 +478,7 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} =
            (conf.target.hostOS == osWindows)
 
 proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool =
-  # List of possible file extensions taken from gcc
-  for ext in [".C", ".cc", ".cpp", ".CPP", ".c++", ".cp", ".cxx"]:
-    if cfile.string.endsWith(ext): return true
-  false
+  conf.backend == backendCpp and not cfile.string.endsWith(".c")
 
 proc envFlags(conf: ConfigRef): string =
   result = if conf.backend == backendCpp:
@@ -495,14 +486,14 @@ proc envFlags(conf: ConfigRef): string =
           else:
             getEnv("CFLAGS")
 
-proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; isCpp: bool): string =
+proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string =
   if compiler == ccEnv:
-    result = if isCpp:
+    result = if useCpp(conf, cfile):
               getEnv("CXX")
             else:
               getEnv("CC")
   else:
-    result = if isCpp:
+    result = if useCpp(conf, cfile):
               CC[compiler].cppCompiler
             else:
               CC[compiler].compilerExe
@@ -516,36 +507,47 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool =
     result = false # assume an old or crappy GCC
     var exe = getConfigVar(conf, conf.cCompiler, ".exe")
     if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
-    # NOTE: should we need the full version, use -dumpfullversion
-    let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except IOError, OSError, ValueError: ("", 1)
+    let (s, exitCode) = try: execCmdEx(exe & " --version") except: ("", 1)
     if exitCode == 0:
-      var major: int
-      discard parseInt(s, major)
-      result = major >= 5
+      var i = 0
+      var j = 0
+      # the version is the last part of the first line:
+      while i < s.len and s[i] != '\n':
+        if s[i] in {' ', '\t'}: j = i+1
+        inc i
+      if j > 0:
+        var major = 0
+        while j < s.len and s[j] in {'0'..'9'}:
+          major = major * 10 + (ord(s[j]) - ord('0'))
+          inc j
+        if i < s.len and s[j] == '.': inc j
+        while j < s.len and s[j] in {'0'..'9'}:
+          inc j
+        if j+1 < s.len and s[j] == '.' and s[j+1] in {'0'..'9'}:
+          # we found a third version number, chances are high
+          # we really parsed the version:
+          result = major >= 5
   else:
     result = conf.cCompiler == ccCLang
 
 proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
-           else: getCompilerExe(conf, compiler, optMixedMode in conf.globalOptions or conf.backend == backendCpp)
+           elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler
+           else: getCompilerExe(conf, compiler, AbsoluteFile"")
 
 proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
                          isMainFile = false; produceOutput = false): string =
-  let
-    c = conf.cCompiler
-    isCpp = useCpp(conf, cfile.cname)
+  let c = conf.cCompiler
   # We produce files like module.nim.cpp, so the absolute Nim filename is not
   # cfile.name but `cfile.cname.changeFileExt("")`:
   var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string)
-  if isCpp:
+  if useCpp(conf, cfile.cname):
     # needs to be prepended so that --passc:-std=c++17 can override default.
     # we could avoid allocation by making cFileSpecificOptions inplace
     options = CC[c].cppXsupport & ' ' & options
-    # If any C++ file was compiled, we need to use C++ driver for linking as well
-    incl conf.globalOptions, optMixedMode
 
   var exe = getConfigVar(conf, c, ".exe")
-  if exe.len == 0: exe = getCompilerExe(conf, c, isCpp)
+  if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname)
 
   if needsExeExt(conf): exe = addFileExt(exe, "exe")
   if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and
@@ -565,14 +567,14 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
 
     compilePattern = joinPath(conf.cCompilerPath, exe)
   else:
-    compilePattern = exe
+    compilePattern = getCompilerExe(conf, c, cfile.cname)
 
   includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)]))
 
-  let cf = if noAbsolutePaths(conf): AbsoluteFile extractFilename(cfile.cname.string)
+  var cf = if noAbsolutePaths(conf): AbsoluteFile extractFilename(cfile.cname.string)
            else: cfile.cname
 
-  let objfile =
+  var objfile =
     if cfile.obj.isEmpty:
       if CfileFlag.External notin cfile.flags or noAbsolutePaths(conf):
         toObjFile(conf, cf).string
@@ -627,8 +629,8 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
 proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
   if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good
 
-  let hashFile = toGeneratedFile(conf, conf.mangleModuleName(cfile.cname).AbsoluteFile, "sha1")
-  let currentHash = footprint(conf, cfile)
+  var hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1")
+  var currentHash = footprint(conf, cfile)
   var f: File
   if open(f, hashFile.string, fmRead):
     let oldHash = parseSecureHash(f.readLine())
@@ -642,10 +644,8 @@ proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
       close(f)
 
 proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) =
-  # we want to generate the hash file unconditionally
-  let extFileChanged = externalFileChanged(conf, c)
   if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and
-      not extFileChanged:
+      not externalFileChanged(conf, c):
     c.flags.incl CfileFlag.Cached
   else:
     # make sure Nim keeps recompiling the external file on reruns
@@ -660,13 +660,10 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) =
   addExternalFileToCompile(conf, c)
 
 proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
-                objfiles: string, isDllBuild: bool, removeStaticFile: bool): string =
+                objfiles: string, isDllBuild: bool): string =
   if optGenStaticLib in conf.globalOptions:
-    if removeStaticFile:
-      removeFile output # fixes: bug #16947
     result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(output),
-                                            "objfiles", objfiles,
-                                            "vccplatform", vccplatform(conf)]
+                                            "objfiles", objfiles]
   else:
     var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe")
     if linkerExe.len == 0: linkerExe = getLinkerExe(conf, conf.cCompiler)
@@ -747,9 +744,8 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
   if optCDebug in conf.globalOptions and conf.cCompiler == ccVcc:
     result.add " /Zi /FS /Od"
 
-template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string,
-                    removeStaticFile = false): string =
-  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions, removeStaticFile)
+template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string): string =
+  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions)
 
 template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped) =
   try:
@@ -819,22 +815,10 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) =
   else:
     writeFile(linkerArgs, args)
   try:
-    when defined(macosx):
-      execLinkCmd(conf, "xargs " & cmd.substr(0, last) & " < " & linkerArgs)
-    else:
-      execLinkCmd(conf, cmd.substr(0, last) & " @" & linkerArgs)
+    execLinkCmd(conf, cmd.substr(0, last) & " @" & linkerArgs)
   finally:
     removeFile(linkerArgs)
 
-proc linkViaShellScript(conf: ConfigRef; cmd: string) =
-  let linkerScript = conf.projectName & "_" & "linkerScript.sh"
-  writeFile(linkerScript, cmd)
-  let shell = getEnv("SHELL")
-  try:
-    execLinkCmd(conf, shell & " " & linkerScript)
-  finally:
-    removeFile(linkerScript)
-
 proc getObjFilePath(conf: ConfigRef, f: Cfile): string =
   if noAbsolutePaths(conf): f.obj.extractFilename
   else: f.obj.string
@@ -848,9 +832,9 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu
 proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string =
   if conf.hasHint(hintCC):
     if optListCmd in conf.globalOptions or conf.verbosity > 1:
-      result = MsgKindToStr[hintCC] % (demangleModuleName(path.splitFile.name) & ": " & compileCmd)
+      result = MsgKindToStr[hintCC] % (demanglePackageName(path.splitFile.name) & ": " & compileCmd)
     else:
-      result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name)
+      result = MsgKindToStr[hintCC] % demanglePackageName(path.splitFile.name)
 
 proc callCCompiler*(conf: ConfigRef) =
   var
@@ -863,7 +847,8 @@ proc callCCompiler*(conf: ConfigRef) =
   var script: Rope = nil
   var cmds: TStringSeq
   var prettyCmds: TStringSeq
-  let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx])
+  let prettyCb = proc (idx: int) =
+    if prettyCmds[idx].len > 0: echo prettyCmds[idx]
 
   for idx, it in conf.toCompile:
     # call the C compiler for the .c file:
@@ -901,7 +886,7 @@ proc callCCompiler*(conf: ConfigRef) =
         let objFile = conf.getObjFilePath(x)
         let buildDll = idx != mainFileIdx
         let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll)
-        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll, removeStaticFile = true))
+        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll))
         # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache
         # for more info check the comment inside of getLinkCmd() where the /PDB:<filename> MSVC flag is used
         if isVSCompatible(conf):
@@ -914,8 +899,8 @@ proc callCCompiler*(conf: ConfigRef) =
       # only if not cached - copy the resulting main file from the nimcache folder to its originally intended destination
       if CfileFlag.Cached notin conf.toCompile[mainFileIdx].flags:
         let mainObjFile = getObjFilePath(conf, conf.toCompile[mainFileIdx])
-        let src = conf.hcrLinkTargetName(mainObjFile, true)
-        let dst = conf.prepareToWriteOutput
+        var src = conf.hcrLinkTargetName(mainObjFile, true)
+        var dst = conf.prepareToWriteOutput
         copyFileWithPermissions(src.string, dst.string)
     else:
       for x in conf.toCompile:
@@ -924,20 +909,15 @@ proc callCCompiler*(conf: ConfigRef) =
         objfiles.add(quoteShell(objFile))
       let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput
                        else: AbsoluteFile(conf.projectName)
-
-      linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true)
+      linkCmd = getLinkCmd(conf, mainOutput, objfiles)
       extraCmds = getExtraCmds(conf, mainOutput)
       if optCompileOnly notin conf.globalOptions:
-        const MaxCmdLen = when defined(windows): 8_000 elif defined(macosx): 260_000 else: 32_000
+        const MaxCmdLen = when defined(windows): 8_000 else: 32_000
         if linkCmd.len > MaxCmdLen:
-          when defined(macosx):
-            # macOS's `ar` does not support response files
-            linkViaShellScript(conf, linkCmd)
-          else:
-            # Windows's command line limit is about 8K (don't laugh...) so C compilers on
-            # Windows support a feature where the command line can be passed via ``@linkcmd``
-            # to them.
-            linkViaResponseFile(conf, linkCmd)
+          # Windows's command line limit is about 8K (don't laugh...) so C compilers on
+          # Windows support a feature where the command line can be passed via ``@linkcmd``
+          # to them.
+          linkViaResponseFile(conf, linkCmd)
         else:
           execLinkCmd(conf, linkCmd)
         for cmd in extraCmds:
@@ -949,99 +929,191 @@ proc callCCompiler*(conf: ConfigRef) =
     script.add("\n")
     generateScript(conf, script)
 
+#from json import escapeJson
+import json, std / sha1
+
 template hashNimExe(): string = $secureHashFile(os.getAppFilename())
 
-proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile =
-  # `outFile` is better than `projectName`, as it allows having different json
-  # files for a given source file compiled with different options; it also
-  # works out of the box with `hashMainCompilationParams`.
-  result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json")
-
-const cacheVersion = "D20210525T193831" # update when `BuildCache` spec changes
-type BuildCache = object
-  cacheVersion: string
-  outputFile: string
-  compile: seq[(string, string)]
-  link: seq[string]
-  linkcmd: string
-  extraCmds: seq[string]
-  configFiles: seq[string] # the hash shouldn't be needed
-  stdinInput: bool
-  projectIsCmd: bool
-  cmdInput: string
-  currentDir: string
-  cmdline: string
-  depfiles: seq[(string, string)]
-  nimexe: string
-
 proc writeJsonBuildInstructions*(conf: ConfigRef) =
-  var linkFiles = collect(for it in conf.externalToLink:
-    var it = it
-    if conf.noAbsolutePaths: it = it.extractFilename
-    it.addFileExt(CC[conf.cCompiler].objExt))
-  for it in conf.toCompile: linkFiles.add it.obj.string
-  var bcache = BuildCache(
-    cacheVersion: cacheVersion,
-    outputFile: conf.absOutFile.string,
-    compile: collect(for i, it in conf.toCompile:
-      if CfileFlag.Cached notin it.flags: (it.cname.string, getCompileCFileCmd(conf, it))),
-    link: linkFiles,
-    linkcmd: getLinkCmd(conf, conf.absOutFile, linkFiles.quoteShellCommand),
-    extraCmds: getExtraCmds(conf, conf.absOutFile),
-    stdinInput: conf.projectIsStdin,
-    projectIsCmd: conf.projectIsCmd,
-    cmdInput: conf.cmdInput,
-    configFiles: conf.configFiles.mapIt(it.string),
-    currentDir: getCurrentDir())
-  if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"):
-    bcache.cmdline = conf.commandLine
-    bcache.depfiles = collect(for it in conf.m.fileInfos:
+  template lit(x: untyped) = f.write x
+  template str(x: untyped) =
+    when compiles(escapeJson(x, buf)):
+      buf.setLen 0
+      escapeJson(x, buf)
+      f.write buf
+    else:
+      f.write escapeJson(x)
+
+  proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) =
+    var comma = false
+    for i, it in clist:
+      if CfileFlag.Cached in it.flags: continue
+      let compileCmd = getCompileCFileCmd(conf, it)
+      if comma: lit ",\L" else: comma = true
+      lit "["
+      str it.cname.string
+      lit ", "
+      str compileCmd
+      lit "]"
+
+  proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList;
+                 llist: seq[string]) =
+    var pastStart = false
+    for it in llist:
+      let objfile = if noAbsolutePaths(conf): it.extractFilename
+                    else: it
+      let objstr = addFileExt(objfile, CC[conf.cCompiler].objExt)
+      objfiles.add(' ')
+      objfiles.add(objstr)
+      if pastStart: lit ",\L"
+      str objstr
+      pastStart = true
+
+    for it in clist:
+      let objstr = quoteShell(it.obj)
+      objfiles.add(' ')
+      objfiles.add(objstr)
+      if pastStart: lit ",\L"
+      str objstr
+      pastStart = true
+    lit "\L"
+
+  proc depfiles(conf: ConfigRef; f: File) =
+    var i = 0
+    for it in conf.m.fileInfos:
       let path = it.fullPath.string
       if isAbsolute(path): # TODO: else?
-        (path, $secureHashFile(path)))
-    bcache.nimexe = hashNimExe()
-  conf.jsonBuildFile = conf.jsonBuildInstructionsFile
-  conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty)
+        if i > 0: lit "],\L"
+        lit "["
+        str path
+        lit ", "
+        str $secureHashFile(path)
+        inc i
+    lit "]\L"
 
-proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool =
-  if not fileExists(jsonFile) or not fileExists(conf.absOutFile): return true
-  var bcache: BuildCache
-  try: bcache.fromJson(jsonFile.string.parseFile)
+
+  var buf = newStringOfCap(50)
+
+  let jsonFile = conf.getNimcacheDir / RelativeFile(conf.projectName & ".json")
+  conf.jsonBuildFile = jsonFile
+  let output = conf.absOutFile
+
+  var f: File
+  if open(f, jsonFile.string, fmWrite):
+    lit "{\L"
+    lit "\"outputFile\": "
+    str $output
+
+    lit ",\L\"compile\":[\L"
+    cfiles(conf, f, buf, conf.toCompile, false)
+    lit "],\L\"link\":[\L"
+    var objfiles = ""
+    # XXX add every file here that is to link
+    linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink)
+
+    lit "],\L\"linkcmd\": "
+    str getLinkCmd(conf, output, objfiles)
+
+    lit ",\L\"extraCmds\": "
+    lit $(%* getExtraCmds(conf, conf.absOutFile))
+
+    lit ",\L\"stdinInput\": "
+    lit $(%* conf.projectIsStdin)
+
+    if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"):
+      lit ",\L\"cmdline\": "
+      str conf.commandLine
+      lit ",\L\"depfiles\":[\L"
+      depfiles(conf, f)
+      lit "],\L\"nimexe\": \L"
+      str hashNimExe()
+      lit "\L"
+
+    lit "\L}\L"
+    close(f)
+
+proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile): bool =
+  let jsonFile = toGeneratedFile(conf, projectfile, "json")
+  if not fileExists(jsonFile): return true
+  if not fileExists(conf.absOutFile): return true
+  result = false
+  try:
+    let data = json.parseFile(jsonFile.string)
+    if not data.hasKey("depfiles") or not data.hasKey("cmdline"):
+      return true
+    let oldCmdLine = data["cmdline"].getStr
+    if conf.commandLine != oldCmdLine:
+      return true
+    if hashNimExe() != data["nimexe"].getStr:
+      return true
+    if not data.hasKey("stdinInput"): return true
+    let stdinInput = data["stdinInput"].getBool
+    if conf.projectIsStdin or stdinInput:
+      # could optimize by returning false if stdin input was the same,
+      # but I'm not sure how to get full stding input
+      return true
+
+    let depfilesPairs = data["depfiles"]
+    doAssert depfilesPairs.kind == JArray
+    for p in depfilesPairs:
+      doAssert p.kind == JArray
+      # >= 2 for forwards compatibility with potential later .json files:
+      doAssert p.len >= 2
+      let depFilename = p[0].getStr
+      let oldHashValue = p[1].getStr
+      let newHashValue = $secureHashFile(depFilename)
+      if oldHashValue != newHashValue:
+        return true
   except IOError, OSError, ValueError:
-    stderr.write "Warning: JSON processing failed for: $#\n" % jsonFile.string
-    return true
-  if bcache.currentDir != getCurrentDir() or # fixes bug #16271
-     bcache.configFiles != conf.configFiles.mapIt(it.string) or
-     bcache.cacheVersion != cacheVersion or bcache.outputFile != conf.absOutFile.string or
-     bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or
-     bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true
-  if bcache.stdinInput or conf.projectIsStdin: return true
-    # xxx optimize by returning false if stdin input was the same
-  for (file, hash) in bcache.depfiles:
-    if $secureHashFile(file) != hash: return true
+    echo "Warning: JSON processing failed: ", getCurrentExceptionMsg()
+    result = true
 
-proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) =
-  var bcache: BuildCache
-  try: bcache.fromJson(jsonFile.string.parseFile)
-  except ValueError, KeyError, JsonKindError:
+proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) =
+  let jsonFile = toGeneratedFile(conf, projectfile, "json")
+  try:
+    let data = json.parseFile(jsonFile.string)
+
+    let output = data["outputFile"].getStr
+    createDir output.parentDir
+    let outputCurrent = $conf.absOutFile
+    if output != outputCurrent:
+      # previously, any specified output file would be silently ignored;
+      # simply copying won't work in some cases, for example with `extraCmds`,
+      # so we just make it an error, user should use same command for jsonscript
+      # as was used with --compileOnly.
+      globalError(conf, gCmdLineInfo, "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % [outputCurrent, output, jsonFile.string])
+
+    let toCompile = data["compile"]
+    doAssert toCompile.kind == JArray
+    var cmds: TStringSeq
+    var prettyCmds: TStringSeq
+    let prettyCb = proc (idx: int) =
+      if prettyCmds[idx].len > 0: echo prettyCmds[idx]
+
+    for c in toCompile:
+      doAssert c.kind == JArray
+      doAssert c.len >= 2
+
+      cmds.add(c[1].getStr)
+      prettyCmds.add displayProgressCC(conf, c[0].getStr, c[1].getStr)
+
+    execCmdsInParallel(conf, cmds, prettyCb)
+
+    let linkCmd = data["linkcmd"]
+    doAssert linkCmd.kind == JString
+    execLinkCmd(conf, linkCmd.getStr)
+    if data.hasKey("extraCmds"):
+      let extraCmds = data["extraCmds"]
+      doAssert extraCmds.kind == JArray
+      for cmd in extraCmds:
+        doAssert cmd.kind == JString, $cmd.kind
+        let cmd2 = cmd.getStr
+        execExternalProgram(conf, cmd2, hintExecuting)
+
+  except:
     let e = getCurrentException()
-    conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" %
-      [e.msg, e.getStackTrace(), jsonFile.string]
-  let output = bcache.outputFile
-  createDir output.parentDir
-  let outputCurrent = $conf.absOutFile
-  if output != outputCurrent or bcache.cacheVersion != cacheVersion:
-    globalError(conf, gCmdLineInfo,
-      "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " %
-      [outputCurrent, output, jsonFile.string])
-  var cmds, prettyCmds: TStringSeq
-  let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx])
-  for (name, cmd) in bcache.compile:
-    cmds.add cmd
-    prettyCmds.add displayProgressCC(conf, name, cmd)
-  execCmdsInParallel(conf, cmds, prettyCb)
-  execLinkCmd(conf, bcache.linkcmd)
-  for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting)
+    quit "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() &
+         "error evaluating JSON file: " & jsonFile.string
 
 proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope =
   for it in list:

compiler/filter_tmpl.nim

@@ -22,7 +22,7 @@ type
     info: TLineInfo
     indent, emitPar: int
     x: string                # the current input line
-    outp: PLLStream          # the output will be parsed by parser
+    outp: PLLStream          # the output will be parsed by pnimsyn
     subsChar, nimDirective: char
     emit, conc, toStr: string
     curly, bracket, par: int

compiler/guards.nim

@@ -82,6 +82,26 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
 
 proc interestingCaseExpr*(m: PNode): bool = isLetLocation(m, true)
 
+type
+  Operators* = object
+    opNot*, opContains*, opLe*, opLt*, opAnd*, opOr*, opIsNil*, opEq*: PSym
+    opAdd*, opSub*, opMul*, opDiv*, opLen*: PSym
+
+proc initOperators*(g: ModuleGraph): Operators =
+  result.opLe = createMagic(g, "<=", mLeI)
+  result.opLt = createMagic(g, "<", mLtI)
+  result.opAnd = createMagic(g, "and", mAnd)
+  result.opOr = createMagic(g, "or", mOr)
+  result.opIsNil = createMagic(g, "isnil", mIsNil)
+  result.opEq = createMagic(g, "==", mEqI)
+  result.opAdd = createMagic(g, "+", mAddI)
+  result.opSub = createMagic(g, "-", mSubI)
+  result.opMul = createMagic(g, "*", mMulI)
+  result.opDiv = createMagic(g, "div", mDivI)
+  result.opLen = createMagic(g, "len", mLengthSeq)
+  result.opNot = createMagic(g, "not", mNot)
+  result.opContains = createMagic(g, "contains", mInSet)
+
 proc swapArgs(fact: PNode, newOp: PSym): PNode =
   result = newNodeI(nkCall, fact.info, 3)
   result[0] = newSymNode(newOp)
@@ -384,16 +404,16 @@ proc usefulFact(n: PNode; o: Operators): PNode =
 type
   TModel* = object
     s*: seq[PNode] # the "knowledge base"
-    g*: ModuleGraph
+    o*: Operators
     beSmart*: bool
 
 proc addFact*(m: var TModel, nn: PNode) =
-  let n = usefulFact(nn, m.g.operators)
+  let n = usefulFact(nn, m.o)
   if n != nil:
     if not m.beSmart:
       m.s.add n
     else:
-      let c = canon(n, m.g.operators)
+      let c = canon(n, m.o)
       if c.getMagic == mAnd:
         addFact(m, c[1])
         addFact(m, c[2])
@@ -401,7 +421,7 @@ proc addFact*(m: var TModel, nn: PNode) =
         m.s.add c
 
 proc addFactNeg*(m: var TModel, n: PNode) =
-  let n = n.neg(m.g.operators)
+  let n = n.neg(m.o)
   if n != nil: addFact(m, n)
 
 proc sameOpr(a, b: PSym): bool =
@@ -441,14 +461,8 @@ proc sameTree*(a, b: PNode): bool =
 proc hasSubTree(n, x: PNode): bool =
   if n.sameTree(x): result = true
   else:
-    case n.kind
-    of nkEmpty..nkNilLit:
-      result = n.sameTree(x)
-    of nkFormalParams:
-      discard
-    else:
-      for i in 0..<n.len:
-        if hasSubTree(n[i], x): return true
+    for i in 0..n.safeLen-1:
+      if hasSubTree(n[i], x): return true
 
 proc invalidateFacts*(s: var seq[PNode], n: PNode) =
   # We are able to guard local vars (as opposed to 'let' variables)!
@@ -726,7 +740,7 @@ proc doesImply*(facts: TModel, prop: PNode): TImplication =
       if result != impUnknown: return
 
 proc impliesNotNil*(m: TModel, arg: PNode): TImplication =
-  result = doesImply(m, m.g.operators.opIsNil.buildCall(arg).neg(m.g.operators))
+  result = doesImply(m, m.o.opIsNil.buildCall(arg).neg(m.o))
 
 proc simpleSlice*(a, b: PNode): BiggestInt =
   # returns 'c' if a..b matches (i+c)..(i+c), -1 otherwise. (i)..(i) is matched
@@ -780,7 +794,12 @@ macro `=~`(x: PNode, pat: untyped): bool =
 
   var conds = newTree(nnkBracket)
   m(x, pat, conds)
-  result = nestList(ident"and", conds)
+  when compiles(nestList(ident"and", conds)):
+    result = nestList(ident"and", conds)
+  #elif declared(macros.toNimIdent):
+  #  result = nestList(toNimIdent"and", conds)
+  else:
+    result = nestList(!"and", conds)
 
 proc isMinusOne(n: PNode): bool =
   n.kind in {nkCharLit..nkUInt64Lit} and n.intVal == -1
@@ -814,7 +833,7 @@ proc ple(m: TModel; a, b: PNode): TImplication =
   if b.getMagic in someAdd:
     if zero() <=? b[2] and a <=? b[1]: return impYes
     # x <= y-c  if x+c <= y
-    if b[2] <=? zero() and (canon(m.g.operators.opSub.buildCall(a, b[2]), m.g.operators) <=? b[1]):
+    if b[2] <=? zero() and (canon(m.o.opSub.buildCall(a, b[2]), m.o) <=? b[1]):
       return impYes
 
   #   x+c <= y  if c <= 0 and x <= y
@@ -828,20 +847,20 @@ proc ple(m: TModel; a, b: PNode): TImplication =
   if a.getMagic in someMul and a[2].isValue and a[1].getMagic in someDiv and
       a[1][2].isValue:
     # simplify   (x div 4) * 2 <= y   to  x div (c div d)  <= y
-    if ple(m, buildCall(m.g.operators.opDiv, a[1][1], `|div|`(a[1][2], a[2])), b) == impYes:
+    if ple(m, buildCall(m.o.opDiv, a[1][1], `|div|`(a[1][2], a[2])), b) == impYes:
       return impYes
 
   # x*3 + x == x*4. It follows that:
   # x*3 + y <= x*4  if  y <= x  and 3 <= 4
   if a =~ x*dc + y and b =~ x2*ec:
     if sameTree(x, x2):
-      let ec1 = m.g.operators.opAdd.buildCall(ec, minusOne())
+      let ec1 = m.o.opAdd.buildCall(ec, minusOne())
       if x >=? 1 and ec >=? 1 and dc >=? 1 and dc <=? ec1 and y <=? x:
         return impYes
   elif a =~ x*dc and b =~ x2*ec + y:
     #echo "BUG cam ehrer e ", a, " <=? ", b
     if sameTree(x, x2):
-      let ec1 = m.g.operators.opAdd.buildCall(ec, minusOne())
+      let ec1 = m.o.opAdd.buildCall(ec, minusOne())
       if x >=? 1 and ec >=? 1 and dc >=? 1 and dc <=? ec1 and y <=? zero():
         return impYes
 
@@ -944,12 +963,12 @@ proc pleViaModel(model: TModel; aa, bb: PNode): TImplication =
   var b = bb
   if replacements.len > 0:
     m.s = @[]
-    m.g = model.g
+    m.o = model.o
     # make the other facts consistent:
     for fact in model.s:
       if fact != nil and fact.getMagic notin someEq:
         # XXX 'canon' should not be necessary here, but it is
-        m.s.add applyReplacements(fact, replacements).canon(m.g.operators)
+        m.s.add applyReplacements(fact, replacements).canon(m.o)
     a = applyReplacements(aa, replacements)
     b = applyReplacements(bb, replacements)
   else:
@@ -958,25 +977,24 @@ proc pleViaModel(model: TModel; aa, bb: PNode): TImplication =
   result = pleViaModelRec(m, a, b)
 
 proc proveLe*(m: TModel; a, b: PNode): TImplication =
-  let x = canon(m.g.operators.opLe.buildCall(a, b), m.g.operators)
+  let x = canon(m.o.opLe.buildCall(a, b), m.o)
   #echo "ROOT ", renderTree(x[1]), " <=? ", renderTree(x[2])
   result = ple(m, x[1], x[2])
   if result == impUnknown:
     # try an alternative:  a <= b  iff  not (b < a)  iff  not (b+1 <= a):
-    let y = canon(m.g.operators.opLe.buildCall(m.g.operators.opAdd.buildCall(b, one()), a), m.g.operators)
+    let y = canon(m.o.opLe.buildCall(m.o.opAdd.buildCall(b, one()), a), m.o)
     result = ~ple(m, y[1], y[2])
 
 proc addFactLe*(m: var TModel; a, b: PNode) =
-  m.s.add canon(m.g.operators.opLe.buildCall(a, b), m.g.operators)
+  m.s.add canon(m.o.opLe.buildCall(a, b), m.o)
 
 proc addFactLt*(m: var TModel; a, b: PNode) =
-  let bb = m.g.operators.opAdd.buildCall(b, minusOne())
+  let bb = m.o.opAdd.buildCall(b, minusOne())
   addFactLe(m, a, bb)
 
 proc settype(n: PNode): PType =
-  result = newType(tySet, ItemId(module: -1, item: -1), n.typ.owner)
-  var idgen: IdGenerator
-  addSonSkipIntLit(result, n.typ, idgen)
+  result = newType(tySet, n.typ.owner)
+  addSonSkipIntLit(result, n.typ)
 
 proc buildOf(it, loc: PNode; o: Operators): PNode =
   var s = newNodeI(nkCurly, it.info, it.len-1)
@@ -1002,14 +1020,14 @@ proc buildElse(n: PNode; o: Operators): PNode =
 
 proc addDiscriminantFact*(m: var TModel, n: PNode) =
   var fact = newNodeI(nkCall, n.info, 3)
-  fact[0] = newSymNode(m.g.operators.opEq)
+  fact[0] = newSymNode(m.o.opEq)
   fact[1] = n[0]
   fact[2] = n[1]
   m.s.add fact
 
 proc addAsgnFact*(m: var TModel, key, value: PNode) =
   var fact = newNodeI(nkCall, key.info, 3)
-  fact[0] = newSymNode(m.g.operators.opEq)
+  fact[0] = newSymNode(m.o.opEq)
   fact[1] = key
   fact[2] = value
   m.s.add fact
@@ -1025,7 +1043,7 @@ proc sameSubexprs*(m: TModel; a, b: PNode): bool =
   # However, nil checking requires exactly the same mechanism! But for now
   # we simply use sameTree and live with the unsoundness of the analysis.
   var check = newNodeI(nkCall, a.info, 3)
-  check[0] = newSymNode(m.g.operators.opEq)
+  check[0] = newSymNode(m.o.opEq)
   check[1] = a
   check[2] = b
   result = m.doesImply(check) == impYes
@@ -1033,9 +1051,9 @@ proc sameSubexprs*(m: TModel; a, b: PNode): bool =
 proc addCaseBranchFacts*(m: var TModel, n: PNode, i: int) =
   let branch = n[i]
   if branch.kind == nkOfBranch:
-    m.s.add buildOf(branch, n[0], m.g.operators)
+    m.s.add buildOf(branch, n[0], m.o)
   else:
-    m.s.add n.buildElse(m.g.operators).neg(m.g.operators)
+    m.s.add n.buildElse(m.o).neg(m.o)
 
 proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
   if check[1].kind == nkCurly:
@@ -1053,6 +1071,6 @@ proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
 
 proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef) =
   for i in 1..<n.len:
-    let check = buildProperFieldCheck(n[0], n[i], m.g.operators)
+    let check = buildProperFieldCheck(n[0], n[i], m.o)
     if check != nil and m.doesImply(check) != impYes:
       message(conf, n.info, warnProveField, renderTree(n[0])); break

compiler/hlo.nim

@@ -86,9 +86,9 @@ proc hlo(c: PContext, n: PNode): PNode =
       else:
         result = fitNode(c, n.typ, result, n.info)
       # optimization has been applied so check again:
-      result = commonOptimizations(c.graph, c.idgen, c.module, result)
+      result = commonOptimizations(c.graph, c.module, result)
       result = hlo(c, result)
-      result = commonOptimizations(c.graph, c.idgen, c.module, result)
+      result = commonOptimizations(c.graph, c.module, result)
 
 proc hloBody(c: PContext, n: PNode): PNode =
   # fast exit:

compiler/ic/bitabs.nim

@@ -1,168 +0,0 @@
-## A BiTable is a table that can be seen as an optimized pair
-## of (Table[LitId, Val], Table[Val, LitId]).
-
-import hashes, rodfiles
-
-type
-  LitId* = distinct uint32
-
-  BiTable*[T] = object
-    vals: seq[T] # indexed by LitId
-    keys: seq[LitId]  # indexed by hash(val)
-
-proc nextTry(h, maxHash: Hash): Hash {.inline.} =
-  result = (h + 1) and maxHash
-
-template maxHash(t): untyped = high(t.keys)
-template isFilled(x: LitId): bool = x.uint32 > 0'u32
-
-proc `$`*(x: LitId): string {.borrow.}
-proc `<`*(x, y: LitId): bool {.borrow.}
-proc `<=`*(x, y: LitId): bool {.borrow.}
-proc `==`*(x, y: LitId): bool {.borrow.}
-proc hash*(x: LitId): Hash {.borrow.}
-
-
-proc len*[T](t: BiTable[T]): int = t.vals.len
-
-proc mustRehash(length, counter: int): bool {.inline.} =
-  assert(length > counter)
-  result = (length * 2 < counter * 3) or (length - counter < 4)
-
-const
-  idStart = 256 ##
-  ## Ids do not start with 0 but with this value. The IR needs it.
-  ## TODO: explain why
-
-template idToIdx(x: LitId): int = x.int - idStart
-
-proc hasLitId*[T](t: BiTable[T]; x: LitId): bool =
-  let idx = idToIdx(x)
-  result = idx >= 0 and idx < t.vals.len
-
-proc enlarge[T](t: var BiTable[T]) =
-  var n: seq[LitId]
-  newSeq(n, len(t.keys) * 2)
-  swap(t.keys, n)
-  for i in 0..high(n):
-    let eh = n[i]
-    if isFilled(eh):
-      var j = hash(t.vals[idToIdx eh]) and maxHash(t)
-      while isFilled(t.keys[j]):
-        j = nextTry(j, maxHash(t))
-      t.keys[j] = move n[i]
-
-proc getKeyId*[T](t: BiTable[T]; v: T): LitId =
-  let origH = hash(v)
-  var h = origH and maxHash(t)
-  if t.keys.len != 0:
-    while true:
-      let litId = t.keys[h]
-      if not isFilled(litId): break
-      if t.vals[idToIdx t.keys[h]] == v: return litId
-      h = nextTry(h, maxHash(t))
-  return LitId(0)
-
-proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId =
-  let origH = hash(v)
-  var h = origH and maxHash(t)
-  if t.keys.len != 0:
-    while true:
-      let litId = t.keys[h]
-      if not isFilled(litId): break
-      if t.vals[idToIdx t.keys[h]] == v: return litId
-      h = nextTry(h, maxHash(t))
-    # not found, we need to insert it:
-    if mustRehash(t.keys.len, t.vals.len):
-      enlarge(t)
-      # recompute where to insert:
-      h = origH and maxHash(t)
-      while true:
-        let litId = t.keys[h]
-        if not isFilled(litId): break
-        h = nextTry(h, maxHash(t))
-  else:
-    setLen(t.keys, 16)
-    h = origH and maxHash(t)
-
-  result = LitId(t.vals.len + idStart)
-  t.keys[h] = result
-  t.vals.add v
-
-
-proc `[]`*[T](t: var BiTable[T]; litId: LitId): var T {.inline.} =
-  let idx = idToIdx litId
-  assert idx < t.vals.len
-  result = t.vals[idx]
-
-proc `[]`*[T](t: BiTable[T]; litId: LitId): lent T {.inline.} =
-  let idx = idToIdx litId
-  assert idx < t.vals.len
-  result = t.vals[idx]
-
-proc hash*[T](t: BiTable[T]): Hash =
-  ## as the keys are hashes of the values, we simply use them instead
-  var h: Hash = 0
-  for i, n in pairs t.keys:
-    h = h !& hash((i, n))
-  result = !$h
-
-proc store*[T](f: var RodFile; t: BiTable[T]) =
-  storeSeq(f, t.vals)
-  storeSeq(f, t.keys)
-
-proc load*[T](f: var RodFile; t: var BiTable[T]) =
-  loadSeq(f, t.vals)
-  loadSeq(f, t.keys)
-
-when isMainModule:
-
-  var t: BiTable[string]
-
-  echo getOrIncl(t, "hello")
-
-  echo getOrIncl(t, "hello")
-  echo getOrIncl(t, "hello3")
-  echo getOrIncl(t, "hello4")
-  echo getOrIncl(t, "helloasfasdfdsa")
-  echo getOrIncl(t, "hello")
-  echo getKeyId(t, "hello")
-  echo getKeyId(t, "none")
-
-  for i in 0 ..< 100_000:
-    discard t.getOrIncl($i & "___" & $i)
-
-  for i in 0 ..< 100_000:
-    assert t.getOrIncl($i & "___" & $i).idToIdx == i + 4
-  echo "begin"
-  echo t.vals.len
-
-  echo t.vals[0]
-  echo t.vals[1004]
-
-  echo "middle"
-
-  var tf: BiTable[float]
-
-  discard tf.getOrIncl(0.4)
-  discard tf.getOrIncl(16.4)
-  discard tf.getOrIncl(32.4)
-  echo getKeyId(tf, 32.4)
-
-  var f2 = open("testblah.bin", fmWrite)
-  echo store(f2, tf)
-  f2.close
-
-  var f1 = open("testblah.bin", fmRead)
-
-  var t2: BiTable[float]
-
-  echo f1.load(t2)
-  echo t2.vals.len
-
-  echo getKeyId(t2, 32.4)
-
-  echo "end"
-
-
-  f1.close

compiler/ic/cbackend.nim

@@ -1,175 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2021 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## New entry point into our C/C++ code generator. Ideally
-## somebody would rewrite the old backend (which is 8000 lines of crufty Nim code)
-## to work on packed trees directly and produce the C code as an AST which can
-## then be rendered to text in a very simple manner. Unfortunately nobody wrote
-## this code. So instead we wrap the existing cgen.nim and its friends so that
-## we call directly into the existing code generation logic but avoiding the
-## naive, outdated `passes` design. Thus you will see some
-## `useAliveDataFromDce in flags` checks in the old code -- the old code is
-## also doing cross-module dependency tracking and DCE that we don't need
-## anymore. DCE is now done as prepass over the entire packed module graph.
-
-import std/packedsets, algorithm, tables
-
-import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen,
-  pathutils, extccomp, msgs, modulepaths]
-
-import packed_ast, ic, dce, rodfiles
-
-proc unpackTree(g: ModuleGraph; thisModule: int;
-                tree: PackedTree; n: NodePos): PNode =
-  var decoder = initPackedDecoder(g.config, g.cache)
-  result = loadNodes(decoder, g.packed, thisModule, tree, n)
-
-proc setupBackendModule(g: ModuleGraph; m: var LoadedModule) =
-  if g.backend == nil:
-    g.backend = cgendata.newModuleList(g)
-  assert g.backend != nil
-  var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config)
-  bmod.idgen = idgenFromLoadedModule(m)
-
-proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) =
-  var bmod = BModuleList(g.backend).modules[m.module.position]
-  assert bmod != nil
-  bmod.flags.incl useAliveDataFromDce
-  bmod.alive = move alive[m.module.position]
-
-  for p in allNodes(m.fromDisk.topLevel):
-    let n = unpackTree(g, m.module.position, m.fromDisk.topLevel, p)
-    cgen.genTopLevelStmt(bmod, n)
-
-  finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info))
-  m.fromDisk.backendFlags = cgen.whichInitProcs(bmod)
-
-proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) =
-  for x in mitems(m.fromDisk.emittedTypeInfo):
-    #echo "found type ", x, " for file ", int(origin)
-    g.emittedTypeInfo[x] = origin
-
-proc addFileToLink(config: ConfigRef; m: PSym) =
-  let filename = AbsoluteFile toFullPath(config, m.position.FileIndex)
-  let ext =
-      if config.backend == backendCpp: ".nim.cpp"
-      elif config.backend == backendObjc: ".nim.m"
-      else: ".nim.c"
-  let cfile = changeFileExt(completeCfilePath(config,
-                            mangleModuleName(config, filename).AbsoluteFile), ext)
-  let objFile = completeCfilePath(config, toObjFile(config, cfile))
-  if fileExists(objFile):
-    var cf = Cfile(nimname: m.name.s, cname: cfile,
-                   obj: objFile,
-                   flags: {CfileFlag.Cached})
-    addFileToCompile(config, cf)
-
-when defined(debugDce):
-  import os, std/packedsets
-
-proc storeAliveSymsImpl(asymFile: AbsoluteFile; s: seq[int32]) =
-  var f = rodfiles.create(asymFile.string)
-  f.storeHeader()
-  f.storeSection aliveSymsSection
-  f.storeSeq(s)
-  close f
-
-template prepare {.dirty.} =
-  let asymFile = toRodFile(config, AbsoluteFile toFullPath(config, position.FileIndex), ".alivesyms")
-  var s = newSeqOfCap[int32](alive[position].len)
-  for a in items(alive[position]): s.add int32(a)
-  sort(s)
-
-proc storeAliveSyms(config: ConfigRef; position: int; alive: AliveSyms) =
-  prepare()
-  storeAliveSymsImpl(asymFile, s)
-
-proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool =
-  prepare()
-  var f2 = rodfiles.open(asymFile.string)
-  f2.loadHeader()
-  f2.loadSection aliveSymsSection
-  var oldData: seq[int32]
-  f2.loadSeq(oldData)
-  f2.close
-  if f2.err == ok and oldData == s:
-    result = false
-  else:
-    when defined(debugDce):
-      let oldAsSet = toPackedSet[int32](oldData)
-      let newAsSet = toPackedSet[int32](s)
-      echo "set of live symbols changed ", asymFile.changeFileExt("rod"), " ", position, " ", f2.err
-      echo "in old but not in new ", oldAsSet.difference(newAsSet), " number of entries in old ", oldAsSet.len
-      echo "in new but not in old ", newAsSet.difference(oldAsSet), " number of entries in new ", newAsSet.len
-      #if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0:
-      #  echo "command failed"
-    result = true
-    storeAliveSymsImpl(asymFile, s)
-
-proc genPackedModule(g: ModuleGraph, i: int; alive: var AliveSyms) =
-  # case statement here to enforce exhaustive checks.
-  case g.packed[i].status
-  of undefined:
-    discard "nothing to do"
-  of loading, stored:
-    assert false
-  of storing, outdated:
-    storeAliveSyms(g.config, g.packed[i].module.position, alive)
-    generateCodeForModule(g, g.packed[i], alive)
-    closeRodFile(g, g.packed[i].module)
-  of loaded:
-    if g.packed[i].loadedButAliveSetChanged:
-      generateCodeForModule(g, g.packed[i], alive)
-    else:
-      addFileToLink(g.config, g.packed[i].module)
-      replayTypeInfo(g, g.packed[i], FileIndex(i))
-
-      if g.backend == nil:
-        g.backend = cgendata.newModuleList(g)
-      registerInitProcs(BModuleList(g.backend), g.packed[i].module, g.packed[i].fromDisk.backendFlags)
-
-proc generateCode*(g: ModuleGraph) =
-  ## The single entry point, generate C(++) code for the entire
-  ## Nim program aka `ModuleGraph`.
-  resetForBackend(g)
-  var alive = computeAliveSyms(g.packed, g.config)
-
-  when false:
-    for i in 0..high(g.packed):
-      echo i, " is of status ", g.packed[i].status, " ", toFullPath(g.config, FileIndex(i))
-
-  # First pass: Setup all the backend modules for all the modules that have
-  # changed:
-  for i in 0..high(g.packed):
-    # case statement here to enforce exhaustive checks.
-    case g.packed[i].status
-    of undefined:
-      discard "nothing to do"
-    of loading, stored:
-      assert false
-    of storing, outdated:
-      setupBackendModule(g, g.packed[i])
-    of loaded:
-      # Even though this module didn't change, DCE might trigger a change.
-      # Consider this case: Module A uses symbol S from B and B does not use
-      # S itself. A is then edited not to use S either. Thus we have to
-      # recompile B in order to remove S from the final result.
-      if aliveSymsChanged(g.config, g.packed[i].module.position, alive):
-        g.packed[i].loadedButAliveSetChanged = true
-        setupBackendModule(g, g.packed[i])
-
-  # Second pass: Code generation.
-  let mainModuleIdx = g.config.projectMainIdx2.int
-  # We need to generate the main module last, because only then
-  # all init procs have been registered:
-  for i in 0..high(g.packed):
-    if i != mainModuleIdx:
-      genPackedModule(g, i, alive)
-  if mainModuleIdx >= 0:
-    genPackedModule(g, mainModuleIdx, alive)

compiler/ic/dce.nim

@@ -1,161 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2021 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Dead code elimination (=DCE) for IC.
-
-import std/[intsets, tables]
-import ".." / [ast, options, lineinfos, types]
-
-import packed_ast, ic, bitabs
-
-type
-  AliveSyms* = seq[IntSet]
-  AliveContext* = object ## Purpose is to fill the 'alive' field.
-    stack: seq[(int, TOptions, NodePos)] ## A stack for marking symbols as alive.
-    decoder: PackedDecoder ## We need a PackedDecoder for module ID address translations.
-    thisModule: int  ## The module we're currently analysing for DCE.
-    alive: AliveSyms ## The final result of our computation.
-    options: TOptions
-    compilerProcs: Table[string, (int, int32)]
-
-proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): bool =
-  ## "Exported to C" procs are special (these are marked with '.exportc') because these
-  ## must not be optimized away!
-  let symPtr = unsafeAddr g[c.thisModule].fromDisk.syms[symId]
-  let flags = symPtr.flags
-  # due to a bug/limitation in the lambda lifting, unused inner procs
-  # are not transformed correctly; issue (#411). However, the whole purpose here
-  # is to eliminate unused procs. So there is no special logic required for this case.
-  if sfCompileTime notin flags:
-    if ({sfExportc, sfCompilerProc} * flags != {}) or
-        (symPtr.kind == skMethod):
-      result = true
-      # XXX: This used to be a condition to:
-      #  (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or
-    if sfCompilerProc in flags:
-      c.compilerProcs[g[c.thisModule].fromDisk.strings[symPtr.name]] = (c.thisModule, symId)
-
-template isNotGeneric(n: NodePos): bool = ithSon(tree, n, genericParamsPos).kind == nkEmpty
-
-proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: int32) =
-  ## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will
-  ## be analysed.
-  if not c.alive[module].containsOrIncl(item):
-    var body = g[module].fromDisk.syms[item].ast
-    if body != emptyNodeId:
-      let opt = g[module].fromDisk.syms[item].options
-      if g[module].fromDisk.syms[item].kind in routineKinds:
-        body = NodeId ithSon(g[module].fromDisk.bodies, NodePos body, bodyPos)
-      c.stack.add((module, opt, NodePos(body)))
-
-    when false:
-      let nid = g[module].fromDisk.syms[item].name
-      if nid != LitId(0):
-        let name = g[module].fromDisk.strings[nid]
-        if name in ["nimFrame", "callDepthLimitReached"]:
-          echo "I was called! ", name, " body exists: ", body != emptyNodeId, " ", module, " ", item
-
-proc requestCompilerProc(c: var AliveContext; g: PackedModuleGraph; name: string) =
-  let (module, item) = c.compilerProcs[name]
-  followLater(c, g, module, item)
-
-proc loadTypeKind(t: PackedItemId; c: AliveContext; g: PackedModuleGraph; toSkip: set[TTypeKind]): TTypeKind =
-  template kind(t: ItemId): TTypeKind = g[t.module].fromDisk.types[t.item].kind
-
-  var t2 = translateId(t, g, c.thisModule, c.decoder.config)
-  result = t2.kind
-  while result in toSkip:
-    t2 = translateId(g[t2.module].fromDisk.types[t2.item].types[^1], g, t2.module, c.decoder.config)
-    result = t2.kind
-
-proc rangeCheckAnalysis(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) =
-  ## Replicates the logic of `ccgexprs.genRangeChck`.
-  ## XXX Refactor so that the duplicated logic is avoided. However, for now it's not clear
-  ## the approach has enough merit.
-  var dest = loadTypeKind(n.typ, c, g, abstractVar)
-  if optRangeCheck notin c.options or dest in {tyUInt..tyUInt64}:
-    discard "no need to generate a check because it was disabled"
-  else:
-    let n0t = loadTypeKind(n.firstSon.typ, c, g, {})
-    if n0t in {tyUInt, tyUInt64}:
-      c.requestCompilerProc(g, "raiseRangeErrorNoArgs")
-    else:
-      let raiser =
-        case loadTypeKind(n.typ, c, g, abstractVarRange)
-        of tyUInt..tyUInt64, tyChar: "raiseRangeErrorU"
-        of tyFloat..tyFloat128: "raiseRangeErrorF"
-        else: "raiseRangeErrorI"
-      c.requestCompilerProc(g, raiser)
-
-proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) =
-  ## Marks the symbols we encounter when we traverse the AST at `tree[n]` as alive, unless
-  ## it is purely in a declarative context (type section etc.).
-  case n.kind
-  of nkNone..pred(nkSym), succ(nkSym)..nkNilLit:
-    discard "ignore non-sym atoms"
-  of nkSym:
-    # This symbol is alive and everything its body references.
-    followLater(c, g, c.thisModule, n.operand)
-  of nkModuleRef:
-    let (n1, n2) = sons2(tree, n)
-    assert n1.kind == nkInt32Lit
-    assert n2.kind == nkInt32Lit
-    let m = n1.litId
-    let item = n2.operand
-    let otherModule = toFileIndexCached(c.decoder, g, c.thisModule, m).int
-    followLater(c, g, otherModule, item)
-  of nkMacroDef, nkTemplateDef, nkTypeSection, nkTypeOfExpr,
-     nkCommentStmt, nkIncludeStmt,
-     nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
-     nkFromStmt, nkStaticStmt:
-    discard
-  of nkVarSection, nkLetSection, nkConstSection:
-    # XXX ignore the defining local variable name?
-    for son in sonsReadonly(tree, n):
-      aliveCode(c, g, tree, son)
-  of nkChckRangeF, nkChckRange64, nkChckRange:
-    rangeCheckAnalysis(c, g, tree, n)
-  of nkProcDef, nkConverterDef, nkMethodDef, nkFuncDef, nkIteratorDef:
-    if n.firstSon.kind == nkSym and isNotGeneric(n):
-      let item = n.firstSon.operand
-      if isExportedToC(c, g, item):
-        # This symbol is alive and everything its body references.
-        followLater(c, g, c.thisModule, item)
-  else:
-    for son in sonsReadonly(tree, n):
-      aliveCode(c, g, tree, son)
-
-proc followNow(c: var AliveContext; g: PackedModuleGraph) =
-  ## Mark all entries in the stack. Marking can add more entries
-  ## to the stack but eventually we have looked at every alive symbol.
-  while c.stack.len > 0:
-    let (modId, opt, ast) = c.stack.pop()
-    c.thisModule = modId
-    c.options = opt
-    aliveCode(c, g, g[modId].fromDisk.bodies, ast)
-
-proc computeAliveSyms*(g: PackedModuleGraph; conf: ConfigRef): AliveSyms =
-  ## Entry point for our DCE algorithm.
-  var c = AliveContext(stack: @[], decoder: PackedDecoder(config: conf),
-                       thisModule: -1, alive: newSeq[IntSet](g.len),
-                       options: conf.options)
-  for i in countdown(high(g), 0):
-    if g[i].status != undefined:
-      c.thisModule = i
-      for p in allNodes(g[i].fromDisk.topLevel):
-        aliveCode(c, g, g[i].fromDisk.topLevel, p)
-
-  followNow(c, g)
-  result = move(c.alive)
-
-proc isAlive*(a: AliveSyms; module: int, item: int32): bool =
-  ## Backends use this to query if a symbol is `alive` which means
-  ## we need to produce (C/C++/etc) code for it.
-  result = a[module].contains(item)
-

compiler/ic/design.rst

@@ -1,56 +0,0 @@
-====================================
-  Incremental Recompilations
-====================================
-
-We split the Nim compiler into a frontend and a backend.
-The frontend produces a set of `.rod` files. Every `.nim` module
-produces its own `.rod` file.
-
-- The IR must be a faithful representation of the AST in memory.
-- The backend can do its own caching but doesn't have to. In the
-  current implementation the backend also caches its results.
-
-Advantage of the "set of files" vs the previous global database:
-- By construction, we either read from the `.rod` file or from the
-  `.nim` file, there can be no inconsistency. There can also be no
-  partial updates.
-- No dependency to external packages (SQLite). SQLite simply is too
-  slow and the old way of serialization was too slow too. We use a
-  format designed for Nim and expect to base further tools on this
-  file format.
-
-References to external modules must be (moduleId, symId) pairs.
-The symbol IDs are module specific. This way no global ID increment
-mechanism needs to be implemented that we could get wrong. ModuleIds
-are rod-file specific too.
-
-
-
-Global state
-------------
-
-There is no global state.
-
-Rod File Format
----------------
-
-It's a simple binary file format. `rodfiles.nim` contains some details.
-
-
-Backend
--------
-
-Nim programmers have to come to enjoy whole-program dead code elimination,
-by default. Since this is a "whole program" optimization, it does break
-modularity. However, thanks to the packed AST representation we can perform
-this global analysis without having to unpack anything. This is basically
-a mark&sweep GC algorithm:
-
-- Start with the top level statements. Every symbol that is referenced
-  from a top level statement is not "dead" and needs to be compiled by
-  the backend.
-- Every symbol referenced from a referenced symbol also has to be
-  compiled.
-
-Caching logic: Only if the set of alive symbols is different from the
-last run, the module has to be regenerated.

compiler/ic/integrity.nim

@@ -1,149 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2021 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Integrity checking for a set of .rod files.
-## The set must cover a complete Nim project.
-
-import sets
-import ".." / [ast, modulegraphs]
-import packed_ast, bitabs, ic
-
-type
-  CheckedContext = object
-    g: ModuleGraph
-    thisModule: int32
-    checkedSyms: HashSet[ItemId]
-    checkedTypes: HashSet[ItemId]
-
-proc checkType(c: var CheckedContext; typeId: PackedItemId)
-proc checkForeignSym(c: var CheckedContext; symId: PackedItemId)
-proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos)
-
-proc checkTypeObj(c: var CheckedContext; typ: PackedType) =
-  for child in typ.types:
-    checkType(c, child)
-  if typ.n != emptyNodeId:
-    checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos typ.n)
-  if typ.sym != nilItemId:
-    checkForeignSym(c, typ.sym)
-  if typ.owner != nilItemId:
-    checkForeignSym(c, typ.owner)
-  checkType(c, typ.typeInst)
-
-proc checkType(c: var CheckedContext; typeId: PackedItemId) =
-  if typeId == nilItemId: return
-  let itemId = translateId(typeId, c.g.packed, c.thisModule, c.g.config)
-  if not c.checkedTypes.containsOrIncl(itemId):
-    let oldThisModule = c.thisModule
-    c.thisModule = itemId.module
-    checkTypeObj c, c.g.packed[itemId.module].fromDisk.types[itemId.item]
-    c.thisModule = oldThisModule
-
-proc checkSym(c: var CheckedContext; s: PackedSym) =
-  if s.name != LitId(0):
-    assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId s.name
-  checkType c, s.typ
-  if s.ast != emptyNodeId:
-    checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos s.ast)
-  if s.owner != nilItemId:
-    checkForeignSym(c, s.owner)
-
-proc checkLocalSym(c: var CheckedContext; item: int32) =
-  let itemId = ItemId(module: c.thisModule, item: item)
-  if not c.checkedSyms.containsOrIncl(itemId):
-    checkSym c, c.g.packed[c.thisModule].fromDisk.syms[item]
-
-proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) =
-  let itemId = translateId(symId, c.g.packed, c.thisModule, c.g.config)
-  if not c.checkedSyms.containsOrIncl(itemId):
-    let oldThisModule = c.thisModule
-    c.thisModule = itemId.module
-    checkSym c, c.g.packed[itemId.module].fromDisk.syms[itemId.item]
-    c.thisModule = oldThisModule
-
-proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) =
-  if tree[n.int].typeId != nilItemId:
-    checkType(c, tree[n.int].typeId)
-  case n.kind
-  of nkEmpty, nkNilLit, nkType, nkNilRodNode:
-    discard
-  of nkIdent:
-    assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId
-  of nkSym:
-    checkLocalSym(c, tree.nodes[n.int].operand)
-  of directIntLit:
-    discard
-  of externIntLit, nkFloatLit..nkFloat128Lit:
-    assert c.g.packed[c.thisModule].fromDisk.numbers.hasLitId n.litId
-  of nkStrLit..nkTripleStrLit:
-    assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId
-  of nkModuleRef:
-    let (n1, n2) = sons2(tree, n)
-    assert n1.kind == nkInt32Lit
-    assert n2.kind == nkInt32Lit
-    checkForeignSym(c, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand))
-  else:
-    for n0 in sonsReadonly(tree, n):
-      checkNode(c, tree, n0)
-
-proc checkTree(c: var CheckedContext; t: PackedTree) =
-  for p in allNodes(t): checkNode(c, t, p)
-
-proc checkLocalSymIds(c: var CheckedContext; m: PackedModule; symIds: seq[int32]) =
-  for symId in symIds:
-    assert symId >= 0 and symId < m.syms.len, $symId & " " & $m.syms.len
-
-proc checkModule(c: var CheckedContext; m: PackedModule) =
-  # We check that:
-  # - Every symbol references existing types and symbols.
-  # - Every tree node references existing types and symbols.
-  for i in 0..high(m.syms):
-    checkLocalSym c, int32(i)
-
-  checkTree c, m.toReplay
-  checkTree c, m.topLevel
-
-  for e in m.exports:
-    assert e[1] >= 0 and e[1] < m.syms.len
-    assert e[0] == m.syms[e[1]].name
-
-  for e in m.compilerProcs:
-    assert e[1] >= 0 and e[1] < m.syms.len
-    assert e[0] == m.syms[e[1]].name
-
-  checkLocalSymIds c, m, m.converters
-  checkLocalSymIds c, m, m.methods
-  checkLocalSymIds c, m, m.trmacros
-  checkLocalSymIds c, m, m.pureEnums
-  #[
-    To do: Check all these fields:
-
-    reexports*: seq[(LitId, PackedItemId)]
-    macroUsages*: seq[(PackedItemId, PackedLineInfo)]
-
-    typeInstCache*: seq[(PackedItemId, PackedItemId)]
-    procInstCache*: seq[PackedInstantiation]
-    attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)]
-    methodsPerType*: seq[(PackedItemId, int, PackedItemId)]
-    enumToStringProcs*: seq[(PackedItemId, PackedItemId)]
-  ]#
-
-proc checkIntegrity*(g: ModuleGraph) =
-  var c = CheckedContext(g: g)
-  for i in 0..high(g.packed):
-    # case statement here to enforce exhaustive checks.
-    case g.packed[i].status
-    of undefined:
-      discard "nothing to do"
-    of loading:
-      assert false, "cannot check integrity: Module still loading"
-    of stored, storing, outdated, loaded:
-      c.thisModule = int32 i
-      checkModule(c, g.packed[i].fromDisk)
-

compiler/ic/navigator.nim

@@ -1,167 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2021 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Supports the "nim check --ic:on --defusages:FILE,LINE,COL"
-## IDE-like features. It uses the set of .rod files to accomplish
-## its task. The set must cover a complete Nim project.
-
-import sets
-
-from os import nil
-from std/private/miscdollars import toLocation
-
-import ".." / [ast, modulegraphs, msgs, options]
-import packed_ast, bitabs, ic
-
-type
-  NavContext = object
-    g: ModuleGraph
-    thisModule: int32
-    trackPos: PackedLineInfo
-    alreadyEmitted: HashSet[string]
-    outputSep: char # for easier testing, use short filenames and spaces instead of tabs.
-
-proc isTracked(current, trackPos: PackedLineInfo, tokenLen: int): bool =
-  if current.file == trackPos.file and current.line == trackPos.line:
-    let col = trackPos.col
-    if col >= current.col and col < current.col+tokenLen:
-      return true
-
-proc searchLocalSym(c: var NavContext; s: PackedSym; info: PackedLineInfo): bool =
-  result = s.name != LitId(0) and
-    isTracked(info, c.trackPos, c.g.packed[c.thisModule].fromDisk.strings[s.name].len)
-
-proc searchForeignSym(c: var NavContext; s: ItemId; info: PackedLineInfo): bool =
-  let name = c.g.packed[s.module].fromDisk.syms[s.item].name
-  result = name != LitId(0) and
-    isTracked(info, c.trackPos, c.g.packed[s.module].fromDisk.strings[name].len)
-
-const
-  EmptyItemId = ItemId(module: -1'i32, item: -1'i32)
-
-proc search(c: var NavContext; tree: PackedTree): ItemId =
-  # We use the linear representation here directly:
-  for i in 0..high(tree.nodes):
-    case tree.nodes[i].kind
-    of nkSym:
-      let item = tree.nodes[i].operand
-      if searchLocalSym(c, c.g.packed[c.thisModule].fromDisk.syms[item], tree.nodes[i].info):
-        return ItemId(module: c.thisModule, item: item)
-    of nkModuleRef:
-      if tree.nodes[i].info.line == c.trackPos.line and tree.nodes[i].info.file == c.trackPos.file:
-        let (n1, n2) = sons2(tree, NodePos i)
-        assert n1.kind == nkInt32Lit
-        assert n2.kind == nkInt32Lit
-        let pId = PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)
-        let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config)
-        if searchForeignSym(c, itemId, tree.nodes[i].info):
-          return itemId
-    else: discard
-  return EmptyItemId
-
-proc isDecl(tree: PackedTree; n: NodePos): bool =
-  # XXX This is not correct yet.
-  const declarativeNodes = procDefs + {nkMacroDef, nkTemplateDef,
-    nkLetSection, nkVarSection, nkUsingStmt, nkConstSection, nkTypeSection,
-    nkIdentDefs, nkEnumTy, nkVarTuple}
-  result = n.int >= 0 and tree[n.int].kind in declarativeNodes
-
-proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) =
-  var m = ""
-  var file = c.g.packed[c.thisModule].fromDisk.strings[info.file]
-  if c.outputSep == ' ':
-    file = os.extractFilename file
-  toLocation(m, file, info.line.int, info.col.int + ColOffset)
-  if not c.alreadyEmitted.containsOrIncl(m):
-    msgWriteln c.g.config, (if isDecl: "def" else: "usage") & c.outputSep & m
-
-proc list(c: var NavContext; tree: PackedTree; sym: ItemId) =
-  for i in 0..high(tree.nodes):
-    case tree.nodes[i].kind
-    of nkSym:
-      let item = tree.nodes[i].operand
-      if sym.item == item and sym.module == c.thisModule:
-        usage(c, tree.nodes[i].info, isDecl(tree, parent(NodePos i)))
-    of nkModuleRef:
-      let (n1, n2) = sons2(tree, NodePos i)
-      assert n1.kind == nkInt32Lit
-      assert n2.kind == nkInt32Lit
-      let pId = PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)
-      let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config)
-      if itemId.item == sym.item and sym.module == itemId.module:
-        usage(c, tree.nodes[i].info, isDecl(tree, parent(NodePos i)))
-    else: discard
-
-proc searchForIncludeFile(g: ModuleGraph; fullPath: string): int =
-  for i in 0..high(g.packed):
-    for k in 1..high(g.packed[i].fromDisk.includes):
-      # we start from 1 because the first "include" file is
-      # the module's filename.
-      if os.cmpPaths(g.packed[i].fromDisk.strings[g.packed[i].fromDisk.includes[k][0]], fullPath) == 0:
-        return i
-  return -1
-
-proc nav(g: ModuleGraph) =
-  # translate the track position to a packed position:
-  let unpacked = g.config.m.trackPos
-  var mid = unpacked.fileIndex.int
-
-  let fullPath = toFullPath(g.config, unpacked.fileIndex)
-
-  if g.packed[mid].status == undefined:
-    # check if 'mid' is an include file of some other module:
-    mid = searchForIncludeFile(g, fullPath)
-
-  if mid < 0:
-    localError(g.config, unpacked, "unknown file name: " & fullPath)
-    return
-
-  let fileId = g.packed[mid].fromDisk.strings.getKeyId(fullPath)
-
-  if fileId == LitId(0):
-    internalError(g.config, unpacked, "cannot find a valid file ID")
-    return
-
-  var c = NavContext(
-    g: g,
-    thisModule: int32 mid,
-    trackPos: PackedLineInfo(line: unpacked.line, col: unpacked.col, file: fileId),
-    outputSep: if isDefined(g.config, "nimIcNavigatorTests"): ' ' else: '\t'
-  )
-  var symId = search(c, g.packed[mid].fromDisk.topLevel)
-  if symId == EmptyItemId:
-    symId = search(c, g.packed[mid].fromDisk.bodies)
-
-  if symId == EmptyItemId:
-    localError(g.config, unpacked, "no symbol at this position")
-    return
-
-  for i in 0..high(g.packed):
-    # case statement here to enforce exhaustive checks.
-    case g.packed[i].status
-    of undefined:
-      discard "nothing to do"
-    of loading:
-      assert false, "cannot check integrity: Module still loading"
-    of stored, storing, outdated, loaded:
-      c.thisModule = int32 i
-      list(c, g.packed[i].fromDisk.topLevel, symId)
-      list(c, g.packed[i].fromDisk.bodies, symId)
-
-proc navDefinition*(g: ModuleGraph) = nav(g)
-proc navUsages*(g: ModuleGraph) = nav(g)
-proc navDefusages*(g: ModuleGraph) = nav(g)
-
-proc writeRodFiles*(g: ModuleGraph) =
-  for i in 0..high(g.packed):
-    case g.packed[i].status
-    of undefined, loading, stored, loaded:
-      discard "nothing to do"
-    of storing, outdated:
-      closeRodFile(g, g.packed[i].module)

compiler/ic/packed_ast.nim

@@ -1,417 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2020 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Packed AST representation, mostly based on a seq of nodes.
-## For IC support. Far future: Rewrite the compiler passes to
-## use this representation directly in all the transformations,
-## it is superior.
-
-import hashes, tables, strtabs
-import bitabs
-import ".." / [ast, options]
-
-type
-  SymId* = distinct int32
-  ModuleId* = distinct int32
-  NodePos* = distinct int
-
-  NodeId* = distinct int32
-
-  PackedItemId* = object
-    module*: LitId       # 0 if it's this module
-    item*: int32         # same as the in-memory representation
-
-const
-  nilItemId* = PackedItemId(module: LitId(0), item: -1.int32)
-
-const
-  emptyNodeId* = NodeId(-1)
-
-type
-  PackedLineInfo* = object
-    line*: uint16
-    col*: int16
-    file*: LitId
-
-  PackedLib* = object
-    kind*: TLibKind
-    generated*: bool
-    isOverriden*: bool
-    name*: LitId
-    path*: NodeId
-
-  PackedSym* = object
-    kind*: TSymKind
-    name*: LitId
-    typ*: PackedItemId
-    flags*: TSymFlags
-    magic*: TMagic
-    info*: PackedLineInfo
-    ast*: NodeId
-    owner*: PackedItemId
-    guard*: PackedItemId
-    bitsize*: int
-    alignment*: int # for alignment
-    options*: TOptions
-    position*: int
-    offset*: int
-    externalName*: LitId # instead of TLoc
-    locFlags*: TLocFlags
-    annex*: PackedLib
-    when hasFFI:
-      cname*: LitId
-    constraint*: NodeId
-
-  PackedType* = object
-    kind*: TTypeKind
-    callConv*: TCallingConvention
-    #nodekind*: TNodeKind
-    flags*: TTypeFlags
-    types*: seq[PackedItemId]
-    n*: NodeId
-    #nodeflags*: TNodeFlags
-    sym*: PackedItemId
-    owner*: PackedItemId
-    size*: BiggestInt
-    align*: int16
-    paddingAtEnd*: int16
-    lockLevel*: TLockLevel # lock level as required for deadlock checking
-    # not serialized: loc*: TLoc because it is backend-specific
-    typeInst*: PackedItemId
-    nonUniqueId*: int32
-
-  PackedNode* = object     # 28 bytes
-    kind*: TNodeKind
-    flags*: TNodeFlags
-    operand*: int32  # for kind in {nkSym, nkSymDef}: SymId
-                     # for kind in {nkStrLit, nkIdent, nkNumberLit}: LitId
-                     # for kind in nkInt32Lit: direct value
-                     # for non-atom kinds: the number of nodes (for easy skipping)
-    typeId*: PackedItemId
-    info*: PackedLineInfo
-
-  PackedTree* = object ## usually represents a full Nim module
-    nodes*: seq[PackedNode]
-
-  PackedInstantiation* = object
-    key*, sym*: PackedItemId
-    concreteTypes*: seq[PackedItemId]
-
-proc `==`*(a, b: SymId): bool {.borrow.}
-proc hash*(a: SymId): Hash {.borrow.}
-
-proc `==`*(a, b: NodePos): bool {.borrow.}
-#proc `==`*(a, b: PackedItemId): bool {.borrow.}
-proc `==`*(a, b: NodeId): bool {.borrow.}
-
-proc newTreeFrom*(old: PackedTree): PackedTree =
-  result.nodes = @[]
-  when false: result.sh = old.sh
-
-when false:
-  proc declareSym*(tree: var PackedTree; kind: TSymKind;
-                  name: LitId; info: PackedLineInfo): SymId =
-    result = SymId(tree.sh.syms.len)
-    tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info)
-
-  proc litIdFromName*(tree: PackedTree; name: string): LitId =
-    result = tree.sh.strings.getOrIncl(name)
-
-  proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) =
-    tree.nodes.add PackedNode(kind: kind, info: info,
-                              operand: int32 getOrIncl(tree.sh.strings, token))
-
-  proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) =
-    tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
-
-proc throwAwayLastNode*(tree: var PackedTree) =
-  tree.nodes.setLen(tree.nodes.len-1)
-
-proc addIdent*(tree: var PackedTree; s: LitId; info: PackedLineInfo) =
-  tree.nodes.add PackedNode(kind: nkIdent, operand: int32(s), info: info)
-
-proc addSym*(tree: var PackedTree; s: int32; info: PackedLineInfo) =
-  tree.nodes.add PackedNode(kind: nkSym, operand: s, info: info)
-
-proc addModuleId*(tree: var PackedTree; s: ModuleId; info: PackedLineInfo) =
-  tree.nodes.add PackedNode(kind: nkInt32Lit, operand: int32(s), info: info)
-
-proc addSymDef*(tree: var PackedTree; s: SymId; info: PackedLineInfo) =
-  tree.nodes.add PackedNode(kind: nkSym, operand: int32(s), info: info)
-
-proc isAtom*(tree: PackedTree; pos: int): bool {.inline.} = tree.nodes[pos].kind <= nkNilLit
-
-proc copyTree*(dest: var PackedTree; tree: PackedTree; n: NodePos) =
-  # and this is why the IR is superior. We can copy subtrees
-  # via a linear scan.
-  let pos = n.int
-  let L = if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
-  let d = dest.nodes.len
-  dest.nodes.setLen(d + L)
-  for i in 0..<L:
-    dest.nodes[d+i] = tree.nodes[pos+i]
-
-when false:
-  proc copySym*(dest: var PackedTree; tree: PackedTree; s: SymId): SymId =
-    result = SymId(dest.sh.syms.len)
-    assert int(s) < tree.sh.syms.len
-    let oldSym = tree.sh.syms[s.int]
-    dest.sh.syms.add oldSym
-
-type
-  PatchPos = distinct int
-
-when false:
-  proc prepare*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo): PatchPos =
-    result = PatchPos tree.nodes.len
-    tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
-
-proc prepare*(tree: var PackedTree; kind: TNodeKind; flags: TNodeFlags; typeId: PackedItemId; info: PackedLineInfo): PatchPos =
-  result = PatchPos tree.nodes.len
-  tree.nodes.add PackedNode(kind: kind, flags: flags, operand: 0, info: info,
-                            typeId: typeId)
-
-proc prepare*(dest: var PackedTree; source: PackedTree; sourcePos: NodePos): PatchPos =
-  result = PatchPos dest.nodes.len
-  dest.nodes.add source.nodes[sourcePos.int]
-
-proc patch*(tree: var PackedTree; pos: PatchPos) =
-  let pos = pos.int
-  assert tree.nodes[pos].kind > nkNilLit
-  let distance = int32(tree.nodes.len - pos)
-  tree.nodes[pos].operand = distance
-
-proc len*(tree: PackedTree): int {.inline.} = tree.nodes.len
-
-proc `[]`*(tree: PackedTree; i: int): lent PackedNode {.inline.} =
-  tree.nodes[i]
-
-proc nextChild(tree: PackedTree; pos: var int) {.inline.} =
-  if tree.nodes[pos].kind > nkNilLit:
-    assert tree.nodes[pos].operand > 0
-    inc pos, tree.nodes[pos].operand
-  else:
-    inc pos
-
-iterator sonsReadonly*(tree: PackedTree; n: NodePos): NodePos =
-  var pos = n.int
-  assert tree.nodes[pos].kind > nkNilLit
-  let last = pos + tree.nodes[pos].operand
-  inc pos
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-iterator sons*(dest: var PackedTree; tree: PackedTree; n: NodePos): NodePos =
-  let patchPos = prepare(dest, tree, n)
-  for x in sonsReadonly(tree, n): yield x
-  patch dest, patchPos
-
-iterator isons*(dest: var PackedTree; tree: PackedTree;
-                n: NodePos): (int, NodePos) =
-  var i = 0
-  for ch0 in sons(dest, tree, n):
-    yield (i, ch0)
-    inc i
-
-iterator sonsFrom1*(tree: PackedTree; n: NodePos): NodePos =
-  var pos = n.int
-  assert tree.nodes[pos].kind > nkNilLit
-  let last = pos + tree.nodes[pos].operand
-  inc pos
-  if pos < last:
-    nextChild tree, pos
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-iterator sonsWithoutLast2*(tree: PackedTree; n: NodePos): NodePos =
-  var count = 0
-  for child in sonsReadonly(tree, n):
-    inc count
-  var pos = n.int
-  assert tree.nodes[pos].kind > nkNilLit
-  let last = pos + tree.nodes[pos].operand
-  inc pos
-  while pos < last and count > 2:
-    yield NodePos pos
-    dec count
-    nextChild tree, pos
-
-proc parentImpl(tree: PackedTree; n: NodePos): NodePos =
-  # finding the parent of a node is rather easy:
-  var pos = n.int - 1
-  while pos >= 0 and (isAtom(tree, pos) or (pos + tree.nodes[pos].operand - 1 < n.int)):
-    dec pos
-  #assert pos >= 0, "node has no parent"
-  result = NodePos(pos)
-
-template parent*(n: NodePos): NodePos = parentImpl(tree, n)
-
-proc hasXsons*(tree: PackedTree; n: NodePos; x: int): bool =
-  var count = 0
-  if tree.nodes[n.int].kind > nkNilLit:
-    for child in sonsReadonly(tree, n): inc count
-  result = count == x
-
-proc hasAtLeastXsons*(tree: PackedTree; n: NodePos; x: int): bool =
-  if tree.nodes[n.int].kind > nkNilLit:
-    var count = 0
-    for child in sonsReadonly(tree, n):
-      inc count
-      if count >= x: return true
-  return false
-
-proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} =
-  NodePos(n.int+1)
-proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} =
-  tree.nodes[n.int].kind
-proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} =
-  LitId tree.nodes[n.int].operand
-proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} =
-  tree.nodes[n.int].info
-
-template typ*(n: NodePos): PackedItemId =
-  tree.nodes[n.int].typeId
-template flags*(n: NodePos): TNodeFlags =
-  tree.nodes[n.int].flags
-
-template operand*(n: NodePos): int32 =
-  tree.nodes[n.int].operand
-
-proc span*(tree: PackedTree; pos: int): int {.inline.} =
-  if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
-
-proc sons2*(tree: PackedTree; n: NodePos): (NodePos, NodePos) =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  result = (NodePos a, NodePos b)
-
-proc sons3*(tree: PackedTree; n: NodePos): (NodePos, NodePos, NodePos) =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  let c = b + span(tree, b)
-  result = (NodePos a, NodePos b, NodePos c)
-
-proc ithSon*(tree: PackedTree; n: NodePos; i: int): NodePos =
-  if tree.nodes[n.int].kind > nkNilLit:
-    var count = 0
-    for child in sonsReadonly(tree, n):
-      if count == i: return child
-      inc count
-  assert false, "node has no i-th child"
-
-when false:
-  proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} =
-    tree.sh.strings[lit]
-
-template kind*(n: NodePos): TNodeKind = tree.nodes[n.int].kind
-template info*(n: NodePos): PackedLineInfo = tree.nodes[n.int].info
-template litId*(n: NodePos): LitId = LitId tree.nodes[n.int].operand
-
-template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].operand
-
-proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
-
-when false:
-  # xxx `nkStrLit` or `nkStrLit..nkTripleStrLit:` below?
-  proc strLit*(tree: PackedTree; n: NodePos): lent string =
-    assert n.kind == nkStrLit
-    result = tree.sh.strings[LitId tree.nodes[n.int].operand]
-
-  proc strVal*(tree: PackedTree; n: NodePos): string =
-    assert n.kind == nkStrLit
-    result = tree.sh.strings[LitId tree.nodes[n.int].operand]
-    #result = cookedStrLit(raw)
-
-  proc filenameVal*(tree: PackedTree; n: NodePos): string =
-    case n.kind
-    of nkStrLit:
-      result = strVal(tree, n)
-    of nkIdent:
-      result = tree.sh.strings[n.litId]
-    of nkSym:
-      result = tree.sh.strings[tree.sh.syms[int n.symId].name]
-    else:
-      result = ""
-
-  proc identAsStr*(tree: PackedTree; n: NodePos): lent string =
-    assert n.kind == nkIdent
-    result = tree.sh.strings[LitId tree.nodes[n.int].operand]
-
-const
-  externIntLit* = {nkCharLit,
-    nkIntLit,
-    nkInt8Lit,
-    nkInt16Lit,
-    nkInt64Lit,
-    nkUIntLit,
-    nkUInt8Lit,
-    nkUInt16Lit,
-    nkUInt32Lit,
-    nkUInt64Lit} # nkInt32Lit is missing by design!
-
-  externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt64Lit}
-  externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit}
-  directIntLit* = nkInt32Lit
-
-when false:
-  proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
-    if n.kind == nkIdent:
-      result = n.litId
-    elif n.kind == nkSym:
-      result = tree.sh.syms[int n.symId].name
-    else:
-      result = LitId(0)
-
-  template identId*(n: NodePos): LitId = identIdImpl(tree, n)
-
-template copyInto*(dest, n, body) =
-  let patchPos = prepare(dest, tree, n)
-  body
-  patch dest, patchPos
-
-template copyIntoKind*(dest, kind, info, body) =
-  let patchPos = prepare(dest, kind, info)
-  body
-  patch dest, patchPos
-
-when false:
-  proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool =
-    let litId = tree.sh.strings.getKeyId(pragma)
-    if litId == LitId(0):
-      return false
-    assert n.kind == nkPragma
-    for ch0 in sonsReadonly(tree, n):
-      if ch0.kind == nkExprColonExpr:
-        if ch0.firstSon.identId == litId:
-          return true
-      elif ch0.identId == litId:
-        return true
-
-proc getNodeId*(tree: PackedTree): NodeId {.inline.} = NodeId tree.nodes.len
-
-when false:
-  proc produceError*(dest: var PackedTree; tree: PackedTree; n: NodePos; msg: string) =
-    let patchPos = prepare(dest, nkError, n.info)
-    dest.add nkStrLit, msg, n.info
-    copyTree(dest, tree, n)
-    patch dest, patchPos
-
-iterator allNodes*(tree: PackedTree): NodePos =
-  var p = 0
-  while p < tree.len:
-    yield NodePos(p)
-    let s = span(tree, p)
-    inc p, s
-
-proc toPackedItemId*(item: int32): PackedItemId {.inline.} =
-  PackedItemId(module: LitId(0), item: item)

compiler/ic/replayer.nim

@@ -1,147 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2020 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Module that contains code to replay global VM state changes and pragma
-## state like ``{.compile: "foo.c".}``. For IC (= Incremental compilation)
-## support.
-
-import ".." / [ast, modulegraphs, trees, extccomp, btrees,
-  msgs, lineinfos, pathutils, options, cgmeth]
-
-import tables
-
-import packed_ast, ic, bitabs
-
-proc replayStateChanges*(module: PSym; g: ModuleGraph) =
-  let list = module.ast
-  assert list != nil
-  assert list.kind == nkStmtList
-  for n in list:
-    assert n.kind == nkReplayAction
-    # Fortunately only a tiny subset of the available pragmas need to
-    # be replayed here. This is always a subset of ``pragmas.stmtPragmas``.
-    if n.len >= 2:
-      internalAssert g.config, n[0].kind == nkStrLit and n[1].kind == nkStrLit
-      case n[0].strVal
-      of "hint": message(g.config, n.info, hintUser, n[1].strVal)
-      of "warning": message(g.config, n.info, warnUser, n[1].strVal)
-      of "error": localError(g.config, n.info, errUser, n[1].strVal)
-      of "compile":
-        internalAssert g.config, n.len == 4 and n[2].kind == nkStrLit
-        let cname = AbsoluteFile n[1].strVal
-        var cf = Cfile(nimname: splitFile(cname).name, cname: cname,
-                       obj: AbsoluteFile n[2].strVal,
-                       flags: {CfileFlag.External},
-                       customArgs: n[3].strVal)
-        extccomp.addExternalFileToCompile(g.config, cf)
-      of "link":
-        extccomp.addExternalFileToLink(g.config, AbsoluteFile n[1].strVal)
-      of "passl":
-        extccomp.addLinkOption(g.config, n[1].strVal)
-      of "passc":
-        extccomp.addCompileOption(g.config, n[1].strVal)
-      of "localpassc":
-        extccomp.addLocalCompileOption(g.config, n[1].strVal, toFullPathConsiderDirty(g.config, module.info.fileIndex))
-      of "cppdefine":
-        options.cppDefine(g.config, n[1].strVal)
-      of "inc":
-        let destKey = n[1].strVal
-        let by = n[2].intVal
-        let v = getOrDefault(g.cacheCounters, destKey)
-        g.cacheCounters[destKey] = v+by
-      of "put":
-        let destKey = n[1].strVal
-        let key = n[2].strVal
-        let val = n[3]
-        if not contains(g.cacheTables, destKey):
-          g.cacheTables[destKey] = initBTree[string, PNode]()
-        if not contains(g.cacheTables[destKey], key):
-          g.cacheTables[destKey].add(key, val)
-        else:
-          internalError(g.config, n.info, "key already exists: " & key)
-      of "incl":
-        let destKey = n[1].strVal
-        let val = n[2]
-        if not contains(g.cacheSeqs, destKey):
-          g.cacheSeqs[destKey] = newTree(nkStmtList, val)
-        else:
-          block search:
-            for existing in g.cacheSeqs[destKey]:
-              if exprStructuralEquivalent(existing, val, strictSymEquality=true):
-                break search
-            g.cacheSeqs[destKey].add val
-      of "add":
-        let destKey = n[1].strVal
-        let val = n[2]
-        if not contains(g.cacheSeqs, destKey):
-          g.cacheSeqs[destKey] = newTree(nkStmtList, val)
-        else:
-          g.cacheSeqs[destKey].add val
-      else:
-        internalAssert g.config, false
-
-proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
-  ## We remember the generic instantiations a module performed
-  ## in order to to avoid the code bloat that generic code tends
-  ## to imply. This is cheaper than deduplication of identical
-  ## generic instantiations. However, deduplication is more
-  ## powerful and general and I hope to implement it soon too
-  ## (famous last words).
-  assert g.packed[module].status == loaded
-  for it in g.packed[module].fromDisk.typeInstCache:
-    let key = translateId(it[0], g.packed, module, g.config)
-    g.typeInstCache.mgetOrPut(key, @[]).add LazyType(id: FullId(module: module, packed: it[1]), typ: nil)
-
-  for it in mitems(g.packed[module].fromDisk.procInstCache):
-    let key = translateId(it.key, g.packed, module, g.config)
-    let sym = translateId(it.sym, g.packed, module, g.config)
-    var concreteTypes = newSeq[FullId](it.concreteTypes.len)
-    for i in 0..high(it.concreteTypes):
-      let tmp = translateId(it.concreteTypes[i], g.packed, module, g.config)
-      concreteTypes[i] = FullId(module: tmp.module, packed: it.concreteTypes[i])
-
-    g.procInstCache.mgetOrPut(key, @[]).add LazyInstantiation(
-      module: module, sym: FullId(module: sym.module, packed: it.sym),
-      concreteTypes: concreteTypes, inst: nil)
-
-  for it in mitems(g.packed[module].fromDisk.methodsPerType):
-    let key = translateId(it[0], g.packed, module, g.config)
-    let col = it[1]
-    let tmp = translateId(it[2], g.packed, module, g.config)
-    let symId = FullId(module: tmp.module, packed: it[2])
-    g.methodsPerType.mgetOrPut(key, @[]).add (col, LazySym(id: symId, sym: nil))
-
-  for it in mitems(g.packed[module].fromDisk.enumToStringProcs):
-    let key = translateId(it[0], g.packed, module, g.config)
-    let tmp = translateId(it[1], g.packed, module, g.config)
-    let symId = FullId(module: tmp.module, packed: it[1])
-    g.enumToStringProcs[key] = LazySym(id: symId, sym: nil)
-
-  for it in mitems(g.packed[module].fromDisk.methods):
-    let sym = loadSymFromId(g.config, g.cache, g.packed, module,
-                            PackedItemId(module: LitId(0), item: it))
-    methodDef(g, g.idgen, sym)
-
-  when false:
-    # not used anymore:
-    for it in mitems(g.packed[module].fromDisk.compilerProcs):
-      let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it[1]))
-      g.lazyCompilerprocs[g.packed[module].fromDisk.sh.strings[it[0]]] = symId
-
-  for it in mitems(g.packed[module].fromDisk.converters):
-    let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it))
-    g.ifaces[module].converters.add LazySym(id: symId, sym: nil)
-
-  for it in mitems(g.packed[module].fromDisk.trmacros):
-    let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it))
-    g.ifaces[module].patterns.add LazySym(id: symId, sym: nil)
-
-  for it in mitems(g.packed[module].fromDisk.pureEnums):
-    let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it))
-    g.ifaces[module].pureEnums.add LazySym(id: symId, sym: nil)

compiler/ic/rodfiles.nim

@@ -1,247 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2020 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Low level binary format used by the compiler to store and load various AST
-## and related data.
-##
-## NB: this is incredibly low level and if you're interested in how the
-##     compiler works and less a storage format, you're probably looking for
-##     the `ic` or `packed_ast` modules to understand the logical format.
-
-from typetraits import supportsCopyMem
-
-## Overview
-## ========
-## `RodFile` represents a Rod File (versioned binary format), and the
-## associated data for common interactions such as IO and error tracking
-## (`RodFileError`). The file format broken up into sections (`RodSection`)
-## and preceeded by a header (see: `cookie`). The precise layout, section
-## ordering and data following the section are determined by the user. See
-## `ic.loadRodFile`.
-##
-## A basic but "wrong" example of the lifecycle:
-## ---------------------------------------------
-## 1. `create` or `open`        - create a new one or open an existing
-## 2. `storeHeader`             - header info
-## 3. `storePrim` or `storeSeq` - save your stuff
-## 4. `close`                   - and we're done
-##
-## Now read the bits below to understand what's missing.
-##
-## ### Issues with the Example
-## Missing Sections:
-## This is a low level API, so headers and sections need to be stored and
-## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` &
-## `loadSection`, respectively.
-##
-## No Error Handling:
-## The API is centered around IO and prone to error, each operation checks or
-## sets the `RodFile.err` field. A user of this API needs to handle these
-## appropriately.
-##
-## API Notes
-## =========
-##
-## Valid inputs for Rod files
-## --------------------------
-## ASTs, hopes, dreams, and anything as long as it and any children it may have
-## support `copyMem`. This means anything that is not a pointer and that does not contain a pointer. At a glance these are:
-## * string
-## * objects & tuples (fields are recursed)
-## * sequences AKA `seq[T]`
-##
-## Note on error handling style
-## ----------------------------
-## A flag based approach is used where operations no-op in case of a
-## preexisting error and set the flag if they encounter one.
-##
-## Misc
-## ----
-## * 'Prim' is short for 'primitive', as in a non-sequence type
-
-type
-  RodSection* = enum
-    versionSection
-    configSection
-    stringsSection
-    checkSumsSection
-    depsSection
-    numbersSection
-    exportsSection
-    hiddenSection
-    reexportsSection
-    compilerProcsSection
-    trmacrosSection
-    convertersSection
-    methodsSection
-    pureEnumsSection
-    macroUsagesSection
-    toReplaySection
-    topLevelSection
-    bodiesSection
-    symsSection
-    typesSection
-    typeInstCacheSection
-    procInstCacheSection
-    attachedOpsSection
-    methodsPerTypeSection
-    enumToStringProcsSection
-    typeInfoSection  # required by the backend
-    backendFlagsSection
-    aliveSymsSection # beware, this is stored in a `.alivesyms` file.
-
-  RodFileError* = enum
-    ok, tooBig, cannotOpen, ioFailure, wrongHeader, wrongSection, configMismatch,
-    includeFileChanged
-
-  RodFile* = object
-    f*: File
-    currentSection*: RodSection # for error checking
-    err*: RodFileError # little experiment to see if this works
-                       # better than exceptions.
-
-const
-  RodVersion = 1
-  cookie = [byte(0), byte('R'), byte('O'), byte('D'),
-            byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)]
-
-proc setError(f: var RodFile; err: RodFileError) {.inline.} =
-  f.err = err
-  #raise newException(IOError, "IO error")
-
-proc storePrim*(f: var RodFile; s: string) =
-  ## Stores a string.
-  ## The len is prefixed to allow for later retreival.
-  if f.err != ok: return
-  if s.len >= high(int32):
-    setError f, tooBig
-    return
-  var lenPrefix = int32(s.len)
-  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    setError f, ioFailure
-  else:
-    if s.len != 0:
-      if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
-        setError f, ioFailure
-
-proc storePrim*[T](f: var RodFile; x: T) =
-  ## Stores a non-sequence/string `T`.
-  ## If `T` doesn't support `copyMem` and is an object or tuple then the fields
-  ## are written -- the user from context will need to know which `T` to load.
-  if f.err != ok: return
-  when supportsCopyMem(T):
-    if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
-      setError f, ioFailure
-  elif T is tuple:
-    for y in fields(x):
-      storePrim(f, y)
-  elif T is object:
-    for y in fields(x):
-      when y is seq:
-        storeSeq(f, y)
-      else:
-        storePrim(f, y)
-  else:
-    {.error: "unsupported type for 'storePrim'".}
-
-proc storeSeq*[T](f: var RodFile; s: seq[T]) =
-  ## Stores a sequence of `T`s, with the len as a prefix for later retrieval.
-  if f.err != ok: return
-  if s.len >= high(int32):
-    setError f, tooBig
-    return
-  var lenPrefix = int32(s.len)
-  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    setError f, ioFailure
-  else:
-    for i in 0..<s.len:
-      storePrim(f, s[i])
-
-proc loadPrim*(f: var RodFile; s: var string) =
-  ## Read a string, the length was stored as a prefix
-  if f.err != ok: return
-  var lenPrefix = int32(0)
-  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    setError f, ioFailure
-  else:
-    s = newString(lenPrefix)
-    if lenPrefix > 0:
-      if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
-        setError f, ioFailure
-
-proc loadPrim*[T](f: var RodFile; x: var T) =
-  ## Load a non-sequence/string `T`.
-  if f.err != ok: return
-  when supportsCopyMem(T):
-    if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
-      setError f, ioFailure
-  elif T is tuple:
-    for y in fields(x):
-      loadPrim(f, y)
-  elif T is object:
-    for y in fields(x):
-      when y is seq:
-        loadSeq(f, y)
-      else:
-        loadPrim(f, y)
-  else:
-    {.error: "unsupported type for 'loadPrim'".}
-
-proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
-  ## `T` must be compatible with `copyMem`, see `loadPrim`
-  if f.err != ok: return
-  var lenPrefix = int32(0)
-  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    setError f, ioFailure
-  else:
-    s = newSeq[T](lenPrefix)
-    for i in 0..<lenPrefix:
-      loadPrim(f, s[i])
-
-proc storeHeader*(f: var RodFile) =
-  ## stores the header which is described by `cookie`.
-  if f.err != ok: return
-  if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len:
-    setError f, ioFailure
-
-proc loadHeader*(f: var RodFile) =
-  ## Loads the header which is described by `cookie`.
-  if f.err != ok: return
-  var thisCookie: array[cookie.len, byte]
-  if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len:
-    setError f, ioFailure
-  elif thisCookie != cookie:
-    setError f, wrongHeader
-
-proc storeSection*(f: var RodFile; s: RodSection) =
-  ## update `currentSection` and writes the bytes value of s.
-  if f.err != ok: return
-  assert f.currentSection < s
-  f.currentSection = s
-  storePrim(f, s)
-
-proc loadSection*(f: var RodFile; expected: RodSection) =
-  ## read the bytes value of s, sets and error if the section is incorrect.
-  if f.err != ok: return
-  var s: RodSection
-  loadPrim(f, s)
-  if expected != s and f.err == ok:
-    setError f, wrongSection
-
-proc create*(filename: string): RodFile =
-  ## create the file and open it for writing
-  if not open(result.f, filename, fmWrite):
-    setError result, cannotOpen
-
-proc close*(f: var RodFile) = close(f.f)
-
-proc open*(filename: string): RodFile =
-  ## open the file for reading
-  if not open(result.f, filename, fmRead):
-    setError result, cannotOpen

compiler/idents.nim

@@ -15,9 +15,12 @@ import
   hashes, wordrecg
 
 type
-  PIdent* = ref TIdent
-  TIdent*{.acyclic.} = object
+  TIdObj* = object of RootObj
     id*: int # unique id; use this for comparisons and not the pointers
+
+  PIdObj* = ref TIdObj
+  PIdent* = ref TIdent
+  TIdent*{.acyclic.} = object of TIdObj
     s*: string
     next*: PIdent             # for hash-table chaining
     h*: Hash                 # hash value of s
@@ -113,8 +116,3 @@ proc newIdentCache*(): IdentCache =
 proc whichKeyword*(id: PIdent): TSpecialWord =
   if id.id < 0: result = wInvalid
   else: result = TSpecialWord(id.id)
-
-proc hash*(x: PIdent): Hash {.inline.} = x.h
-proc `==`*(a, b: PIdent): bool {.inline.} =
-  if a.isNil or b.isNil: result = system.`==`(a, b)
-  else: result = a.id == b.id

compiler/idgen.nim

@@ -0,0 +1,59 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2012 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module contains a simple persistent id generator.
+
+import idents, strutils, options, pathutils
+
+var gFrontEndId*: int
+
+const
+  debugIds* = false
+
+when debugIds:
+  import intsets
+
+  var usedIds = initIntSet()
+
+proc registerID*(id: PIdObj) =
+  when debugIds:
+    if id.id == -1 or containsOrIncl(usedIds, id.id):
+      internalError("ID already used: " & $id.id)
+
+proc getID*(): int {.inline.} =
+  result = gFrontEndId
+  inc(gFrontEndId)
+
+proc setId*(id: int) {.inline.} =
+  gFrontEndId = max(gFrontEndId, id + 1)
+
+proc idSynchronizationPoint*(idRange: int) =
+  gFrontEndId = (gFrontEndId div idRange + 1) * idRange + 1
+
+proc toGid(conf: ConfigRef; f: AbsoluteFile): string =
+  # we used to use ``f.addFileExt("gid")`` (aka ``$project.gid``), but this
+  # will cause strange bugs if multiple projects are in the same folder, so
+  # we simply use a project independent name:
+  result = options.completeGeneratedFilePath(conf, AbsoluteFile"nim.gid").string
+
+proc saveMaxIds*(conf: ConfigRef; project: AbsoluteFile) =
+  var f = open(toGid(conf, project), fmWrite)
+  f.writeLine($gFrontEndId)
+  f.close()
+
+proc loadMaxIds*(conf: ConfigRef; project: AbsoluteFile) =
+  var f: File
+  if open(f, toGid(conf, project), fmRead):
+    var line = newStringOfCap(20)
+    if f.readLine(line):
+      var frontEndId = parseInt(line)
+      if f.readLine(line):
+        # var backEndId = parseInt(line)
+        gFrontEndId = max(gFrontEndId, frontEndId)
+    f.close()

compiler/importer.nim

@@ -11,9 +11,7 @@
 
 import
   intsets, ast, astalgo, msgs, options, idents, lookups,
-  semdata, modulepaths, sigmatch, lineinfos, sets,
-  modulegraphs, wordrecg, tables
-from strutils import `%`
+  semdata, modulepaths, sigmatch, lineinfos, sets
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
@@ -22,67 +20,30 @@ proc readExceptSet*(c: PContext, n: PNode): IntSet =
     let ident = lookups.considerQuotedIdent(c, n[i])
     result.incl(ident.id)
 
-proc declarePureEnumField*(c: PContext; s: PSym) =
-  # XXX Remove the outer 'if' statement and see what breaks.
-  var amb = false
-  if someSymFromImportTable(c, s.name, amb) == nil:
-    strTableAdd(c.pureEnumFields, s)
-    when false:
-      let checkB = strTableGet(c.pureEnumFields, s.name)
-      if checkB == nil:
-        strTableAdd(c.pureEnumFields, s)
-    when false:
+proc importPureEnumField*(c: PContext; s: PSym) =
+  let check = strTableGet(c.importTable.symbols, s.name)
+  if check == nil:
+    let checkB = strTableGet(c.pureEnumFields, s.name)
+    if checkB == nil:
+      strTableAdd(c.pureEnumFields, s)
+    else:
       # mark as ambiguous:
       incl(c.ambiguousSymbols, checkB.id)
       incl(c.ambiguousSymbols, s.id)
 
-proc importPureEnumField(c: PContext; s: PSym) =
-  var amb = false
-  if someSymFromImportTable(c, s.name, amb) == nil:
-    strTableAdd(c.pureEnumFields, s)
-    when false:
-      let checkB = strTableGet(c.pureEnumFields, s.name)
-      if checkB == nil:
-        strTableAdd(c.pureEnumFields, s)
-    when false:
-      # mark as ambiguous:
-      incl(c.ambiguousSymbols, checkB.id)
-      incl(c.ambiguousSymbols, s.id)
-
-proc importPureEnumFields(c: PContext; s: PSym; etyp: PType) =
-  assert sfPure in s.flags
-  for j in 0..<etyp.n.len:
-    var e = etyp.n[j].sym
-    if e.kind != skEnumField:
-      internalError(c.config, s.info, "rawImportSymbol")
-      # BUGFIX: because of aliases for enums the symbol may already
-      # have been put into the symbol table
-      # BUGFIX: but only iff they are the same symbols!
-    for check in importedItems(c, e.name):
-      if check.id == e.id:
-        e = nil
-        break
-    if e != nil:
-      importPureEnumField(c, e)
-
-proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) =
+proc rawImportSymbol(c: PContext, s, origin: PSym) =
   # This does not handle stubs, because otherwise loading on demand would be
   # pointless in practice. So importing stubs is fine here!
   # check if we have already a symbol of the same name:
-  when false:
-    var check = someSymFromImportTable(c, s.name)
-    if check != nil and check.id != s.id:
-      if s.kind notin OverloadableSyms or check.kind notin OverloadableSyms:
-        # s and check need to be qualified:
-        incl(c.ambiguousSymbols, s.id)
-        incl(c.ambiguousSymbols, check.id)
+  var check = strTableGet(c.importTable.symbols, s.name)
+  if check != nil and check.id != s.id:
+    if s.kind notin OverloadableSyms or check.kind notin OverloadableSyms:
+      # s and check need to be qualified:
+      incl(c.ambiguousSymbols, s.id)
+      incl(c.ambiguousSymbols, check.id)
   # thanks to 'export' feature, it could be we import the same symbol from
-  # multiple sources, so we need to call 'strTableAdd' here:
-  when false:
-    # now lazy. Speeds up the compiler and is a prerequisite for IC.
-    strTableAdd(c.importTable.symbols, s)
-  else:
-    importSet.incl s.id
+  # multiple sources, so we need to call 'StrTableAdd' here:
+  strTableAdd(c.importTable.symbols, s)
   if s.kind == skType:
     var etyp = s.typ
     if etyp.kind in {tyBool, tyEnum}:
@@ -93,42 +54,27 @@ proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) =
           # BUGFIX: because of aliases for enums the symbol may already
           # have been put into the symbol table
           # BUGFIX: but only iff they are the same symbols!
-        for check in importedItems(c, e.name):
+        var it: TIdentIter
+        check = initIdentIter(it, c.importTable.symbols, e.name)
+        while check != nil:
           if check.id == e.id:
             e = nil
             break
+          check = nextIdentIter(it, c.importTable.symbols)
         if e != nil:
           if sfPure notin s.flags:
-            rawImportSymbol(c, e, origin, importSet)
+            rawImportSymbol(c, e, origin)
           else:
             importPureEnumField(c, e)
   else:
-    if s.kind == skConverter: addConverter(c, LazySym(sym: s))
-    if hasPattern(s): addPattern(c, LazySym(sym: s))
+    if s.kind == skConverter: addConverter(c, s)
+    if hasPattern(s): addPattern(c, s)
   if s.owner != origin:
     c.exportIndirections.incl((origin.id, s.id))
 
-proc splitPragmas(c: PContext, n: PNode): (PNode, seq[TSpecialWord]) =
-  template bail = globalError(c.config, n.info, "invalid pragma")
-  if n.kind == nkPragmaExpr:
-    if n.len == 2 and n[1].kind == nkPragma:
-      result[0] = n[0]
-      for ni in n[1]:
-        if ni.kind == nkIdent: result[1].add whichKeyword(ni.ident)
-        else: bail()
-    else: bail()
-  else:
-    result[0] = n
-    if result[0].safeLen > 0:
-      (result[0][^1], result[1]) = splitPragmas(c, result[0][^1])
-
-proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
-  let (n, kws) = splitPragmas(c, n)
-  if kws.len > 0:
-    globalError(c.config, n.info, "unexpected pragma")
-
+proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
   let ident = lookups.considerQuotedIdent(c, n)
-  let s = someSym(c.graph, fromMod, ident)
+  let s = strTableGet(fromMod.tab, ident)
   if s == nil:
     errorUndeclaredIdentifier(c, n.info, ident.s)
   else:
@@ -138,77 +84,34 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
     # for an enumeration we have to add all identifiers
     if multiImport:
       # for a overloadable syms add all overloaded routines
-      var it: ModuleIter
-      var e = initModuleIter(it, c.graph, fromMod, s.name)
+      var it: TIdentIter
+      var e = initIdentIter(it, fromMod.tab, s.name)
       while e != nil:
         if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3")
         if s.kind in ExportableSymKinds:
-          rawImportSymbol(c, e, fromMod, importSet)
-        e = nextModuleIter(it, c.graph)
+          rawImportSymbol(c, e, fromMod)
+        e = nextIdentIter(it, fromMod.tab)
     else:
-      rawImportSymbol(c, s, fromMod, importSet)
-    suggestSym(c.graph, n.info, s, c.graph.usageSym, false)
-
-proc addImport(c: PContext; im: sink ImportedModule) =
-  for i in 0..high(c.imports):
-    if c.imports[i].m == im.m:
-      # we have already imported the module: Check which import
-      # is more "powerful":
-      case c.imports[i].mode
-      of importAll: discard "already imported all symbols"
-      of importSet:
-        case im.mode
-        of importAll, importExcept:
-          # XXX: slightly wrong semantics for 'importExcept'...
-          # But we should probably change the spec and disallow this case.
-          c.imports[i] = im
-        of importSet:
-          # merge the import sets:
-          c.imports[i].imported.incl im.imported
-      of importExcept:
-        case im.mode
-        of importAll:
-          c.imports[i] = im
-        of importSet:
-          discard
-        of importExcept:
-          var cut = initIntSet()
-          # only exclude what is consistent between the two sets:
-          for j in im.exceptSet:
-            if j in c.imports[i].exceptSet:
-              cut.incl j
-          c.imports[i].exceptSet = cut
-      return
-  c.imports.add im
-
-template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} =
-  for it in mitems c.graph.ifaces[fromMod.position].converters:
-    if filter:
-      loadPackedSym(c.graph, it)
-      if sfExported in it.sym.flags:
-        addConverter(c, it)
-  for it in mitems c.graph.ifaces[fromMod.position].patterns:
-    if filter:
-      loadPackedSym(c.graph, it)
-      if sfExported in it.sym.flags:
-        addPattern(c, it)
-  for it in mitems c.graph.ifaces[fromMod.position].pureEnums:
-    if filter:
-      loadPackedSym(c.graph, it)
-      importPureEnumFields(c, it.sym, it.sym.typ)
+      rawImportSymbol(c, s, fromMod)
+    suggestSym(c.config, n.info, s, c.graph.usageSym, false)
 
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
-  c.addImport ImportedModule(m: fromMod, mode: importExcept, exceptSet: exceptSet)
-  addUnnamedIt(c, fromMod, it.sym.name.id notin exceptSet)
+  var i: TTabIter
+  var s = initTabIter(i, fromMod.tab)
+  while s != nil:
+    if s.kind != skModule:
+      if s.kind != skEnumField:
+        if s.kind notin ExportableSymKinds:
+          internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s)
+        if exceptSet.isNil or s.name.id notin exceptSet:
+          rawImportSymbol(c, s, fromMod)
+    s = nextIter(i, fromMod.tab)
 
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
-  c.addImport ImportedModule(m: fromMod, mode: importAll)
-  addUnnamedIt(c, fromMod, true)
-  when false:
-    var exceptSet: IntSet
-    importAllSymbolsExcept(c, fromMod, exceptSet)
+  var exceptSet: IntSet
+  importAllSymbolsExcept(c, fromMod, exceptSet)
 
-proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; importSet: var IntSet) =
+proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym) =
   if n.isNil: return
   case n.kind
   of nkExportStmt:
@@ -218,61 +121,27 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; im
       if s.kind == skModule:
         importAllSymbolsExcept(c, s, exceptSet)
       elif exceptSet.isNil or s.name.id notin exceptSet:
-        rawImportSymbol(c, s, fromMod, importSet)
+        rawImportSymbol(c, s, fromMod)
   of nkExportExceptStmt:
     localError(c.config, n.info, "'export except' not implemented")
   else:
     for i in 0..n.safeLen-1:
-      importForwarded(c, n[i], exceptSet, fromMod, importSet)
+      importForwarded(c, n[i], exceptSet, fromMod)
 
-proc addUnique[T](x: var seq[T], y: sink T) {.noSideEffect.} =
-  for i in 0..high(x):
-    if x[i] == y: return
-  x.add y
-    
-proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool): PSym =
+proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
   result = realModule
-  template createModuleAliasImpl(ident): untyped =
-    createModuleAlias(realModule, nextSymId c.idgen, ident, n.info, c.config.options)
+  c.unusedImports.add((realModule, n.info))
   if n.kind != nkImportAs: discard
   elif n.len != 2 or n[1].kind != nkIdent:
     localError(c.config, n.info, "module alias must be an identifier")
   elif n[1].ident.id != realModule.name.id:
     # some misguided guy will write 'import abc.foo as foo' ...
-    result = createModuleAliasImpl(n[1].ident)
-  if result == realModule:
-    # avoids modifying `realModule`, see D20201209T194412 for `import {.all.}`
-    result = createModuleAliasImpl(realModule.name)
-  if importHidden:
-    result.options.incl optImportHidden
-  c.unusedImports.add((result, n.info))
-  c.importModuleMap[result.id] = realModule.id
-  c.importModuleLookup.mgetOrPut(result.name.id, @[]).addUnique realModule.id
+    result = createModuleAlias(realModule, n[1].ident, realModule.info,
+                               c.config.options)
 
-proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] =
-  var ret: typeof(result)
-  proc processPragma(n2: PNode): PNode =
-    let (result2, kws) = splitPragmas(c, n2)
-    result = result2
-    for ai in kws:
-      case ai
-      of wImportHidden: ret.importHidden = true
-      else: globalError(c.config, n.info, "invalid pragma, expected: " & ${wImportHidden})
-
-  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
-    ret.node = newNodeI(nkImportAs, n.info)
-    ret.node.add n[1].processPragma
-    ret.node.add n[2]
-  else:
-    ret.node = n.processPragma
-  return ret
-
-proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym =
-  let transf = transformImportAs(c, n)
-  n = transf.node
+proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
   let f = checkModuleName(c.config, n)
   if f != InvalidFileIdx:
-    addImportFileDep(c, f)
     let L = c.graph.importStack.len
     let recursion = c.graph.importStack.find(f)
     c.graph.importStack.add f
@@ -285,65 +154,64 @@ proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym =
                 toFullPath(c.config, c.graph.importStack[i+1])
       c.recursiveDep = err
 
-    var realModule: PSym
     discard pushOptionEntry(c)
-    realModule = c.graph.importModuleCallback(c.graph, c.module, f)
-    result = importModuleAs(c, n, realModule, transf.importHidden)
+    result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f))
     popOptionEntry(c)
 
     #echo "set back to ", L
     c.graph.importStack.setLen(L)
     # we cannot perform this check reliably because of
-    # test: modules/import_in_config) # xxx is that still true?
-    if realModule == c.module:
-      localError(c.config, n.info, "module '$1' cannot import itself" % realModule.name.s)
-    if sfDeprecated in realModule.flags:
-      var prefix = ""
-      if realModule.constraint != nil: prefix = realModule.constraint.strVal & "; "
-      message(c.config, n.info, warnDeprecated, prefix & realModule.name.s & " is deprecated")
-    suggestSym(c.graph, n.info, result, c.graph.usageSym, false)
+    # test: modules/import_in_config)
+    when true:
+      if result.info.fileIndex == c.module.info.fileIndex and
+          result.info.fileIndex == n.info.fileIndex:
+        localError(c.config, n.info, "A module cannot import itself")
+    if sfDeprecated in result.flags:
+      if result.constraint != nil:
+        message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated")
+      else:
+        message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated")
+    suggestSym(c.config, n.info, result, c.graph.usageSym, false)
     importStmtResult.add newSymNode(result, n.info)
     #newStrNode(toFullPath(c.config, f), n.info)
 
-proc afterImport(c: PContext, m: PSym) =
-  # fixes bug #17510, for re-exported symbols
-  let realModuleId = c.importModuleMap[m.id]
-  for s in allSyms(c.graph, m):
-    if s.owner.id != realModuleId:
-      c.exportIndirections.incl((m.id, s.id))
+proc transformImportAs(c: PContext; n: PNode): PNode =
+  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
+    result = newNodeI(nkImportAs, n.info)
+    result.add n[1]
+    result.add n[2]
+  else:
+    result = n
 
 proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
-  var it = it
+  let it = transformImportAs(c, it)
   let m = myImportModule(c, it, importStmtResult)
   if m != nil:
     # ``addDecl`` needs to be done before ``importAllSymbols``!
     addDecl(c, m, it.info) # add symbol to symbol table of module
     importAllSymbols(c, m)
     #importForwarded(c, m.ast, emptySet, m)
-    afterImport(c, m)
 
 proc evalImport*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   for i in 0..<n.len:
     let it = n[i]
-    if it.kind in {nkInfix, nkPrefix} and it[^1].kind == nkBracket:
-      let lastPos = it.len - 1
-      var imp = copyNode(it)
-      newSons(imp, it.len)
-      for i in 0 ..< lastPos: imp[i] = it[i]
-      imp[lastPos] = imp[0] # dummy entry, replaced in the loop
-      for x in it[lastPos]:
+    if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
+      let sep = it[0]
+      let dir = it[1]
+      var imp = newNodeI(nkInfix, it.info)
+      imp.add sep
+      imp.add dir
+      imp.add sep # dummy entry, replaced in the loop
+      for x in it[2]:
         # transform `a/b/[c as d]` to `/a/b/c as d`
         if x.kind == nkInfix and x[0].ident.s == "as":
-          var impAs = copyNode(x)
-          newSons(impAs, 3)
-          impAs[0] = x[0]
-          imp[lastPos] = x[1]
+          let impAs = copyTree(x)
+          imp[2] = x[1]
           impAs[1] = imp
-          impAs[2] = x[2]
-          impMod(c, impAs, result)
+          impMod(c, imp, result)
         else:
-          imp[lastPos] = x
+          imp[2] = x
           impMod(c, imp, result)
     else:
       impMod(c, it, result)
@@ -351,25 +219,22 @@ proc evalImport*(c: PContext, n: PNode): PNode =
 proc evalFrom*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
+  n[0] = transformImportAs(c, n[0])
   var m = myImportModule(c, n[0], result)
   if m != nil:
     n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
-
-    var im = ImportedModule(m: m, mode: importSet, imported: initIntSet())
     for i in 1..<n.len:
       if n[i].kind != nkNilLit:
-        importSymbol(c, n[i], m, im.imported)
-    c.addImport im
-    afterImport(c, m)
+        importSymbol(c, n[i], m)
 
 proc evalImportExcept*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
+  n[0] = transformImportAs(c, n[0])
   var m = myImportModule(c, n[0], result)
   if m != nil:
     n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
     importAllSymbolsExcept(c, m, readExceptSet(c, n))
     #importForwarded(c, m.ast, exceptSet, m)
-    afterImport(c, m)

compiler/incremental.nim

@@ -0,0 +1,198 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2018 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Basic type definitions the module graph needs in order to support
+## incremental compilations.
+
+const nimIncremental* = defined(nimIncremental)
+
+import options, lineinfos
+
+when nimIncremental:
+  import ast, msgs, intsets, btrees, db_sqlite, std / sha1, pathutils
+  from strutils import parseInt
+  from os import isAbsolute
+
+  type
+    Writer* = object
+      sstack*: seq[PSym]          # a stack of symbols to process
+      tstack*: seq[PType]         # a stack of types to process
+      tmarks*, smarks*: IntSet
+      forwardedSyms*: seq[PSym]
+
+    Reader* = object
+      syms*: BTree[int, PSym]
+      types*: BTree[int, PType]
+
+    IncrementalCtx* = object
+      db*: DbConn
+      w*: Writer
+      r*: Reader
+      configChanged*: bool
+
+  proc init*(incr: var IncrementalCtx) =
+    incr.w.sstack = @[]
+    incr.w.tstack = @[]
+    incr.w.tmarks = initIntSet()
+    incr.w.smarks = initIntSet()
+    incr.w.forwardedSyms = @[]
+    incr.r.syms = initBTree[int, PSym]()
+    incr.r.types = initBTree[int, PType]()
+
+
+  proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: AbsoluteFile): string =
+    result = msgs.getHash(conf, fileIdx)
+    if result.len == 0 and isAbsolute(string fullpath):
+      result = $secureHashFile(string fullpath)
+      msgs.setHash(conf, fileIdx, result)
+
+  proc toDbFileId*(incr: var IncrementalCtx; conf: ConfigRef; fileIdx: FileIndex): int =
+    if fileIdx == FileIndex(-1): return -1
+    let fullpath = toFullPath(conf, fileIdx)
+    let row = incr.db.getRow(sql"select id, fullhash from filenames where fullpath = ?",
+      fullpath)
+    let id = row[0]
+    let fullhash = hashFileCached(conf, fileIdx, AbsoluteFile fullpath)
+    if id.len == 0:
+      result = int incr.db.insertID(sql"insert into filenames(nimid, fullpath, fullhash) values (?, ?, ?)",
+        int(fileIdx), fullpath, fullhash)
+    else:
+      if row[1] != fullhash:
+        incr.db.exec(sql"update filenames set fullhash = ? where fullpath = ?", fullhash, fullpath)
+      result = parseInt(id)
+
+  proc fromDbFileId*(incr: var IncrementalCtx; conf: ConfigRef; dbId: int): FileIndex =
+    if dbId == -1: return FileIndex(-1)
+    let fullpath = incr.db.getValue(sql"select fullpath from filenames where id = ?", dbId)
+    doAssert fullpath.len > 0, "cannot find file name for DB ID " & $dbId
+    result = fileInfoIdx(conf, AbsoluteFile fullpath)
+
+
+  proc addModuleDep*(incr: var IncrementalCtx; conf: ConfigRef;
+                     module, fileIdx: FileIndex;
+                     isIncludeFile: bool) =
+    if conf.symbolFiles != v2Sf: return
+
+    let a = toDbFileId(incr, conf, module)
+    let b = toDbFileId(incr, conf, fileIdx)
+
+    incr.db.exec(sql"insert into deps(module, dependency, isIncludeFile) values (?, ?, ?)",
+      a, b, ord(isIncludeFile))
+
+  # --------------- Database model ---------------------------------------------
+
+  proc createDb*(db: DbConn) =
+    db.exec(sql"""
+      create table if not exists controlblock(
+        idgen integer not null
+      );
+    """)
+
+    db.exec(sql"""
+      create table if not exists config(
+        config varchar(8000) not null
+      );
+    """)
+
+    db.exec(sql"""
+      create table if not exists filenames(
+        id integer primary key,
+        nimid integer not null,
+        fullpath varchar(8000) not null,
+        fullHash varchar(256) not null
+      );
+    """)
+    db.exec sql"create index if not exists FilenameIx on filenames(fullpath);"
+
+    db.exec(sql"""
+      create table if not exists modules(
+        id integer primary key,
+        nimid integer not null,
+        fullpath varchar(8000) not null,
+        interfHash varchar(256) not null,
+        fullHash varchar(256) not null,
+
+        created timestamp not null default (DATETIME('now'))
+      );""")
+    db.exec(sql"""create unique index if not exists SymNameIx on modules(fullpath);""")
+
+    db.exec(sql"""
+      create table if not exists deps(
+        id integer primary key,
+        module integer not null,
+        dependency integer not null,
+        isIncludeFile integer not null,
+        foreign key (module) references filenames(id),
+        foreign key (dependency) references filenames(id)
+      );""")
+    db.exec(sql"""create index if not exists DepsIx on deps(module);""")
+
+    db.exec(sql"""
+      create table if not exists types(
+        id integer primary key,
+        nimid integer not null,
+        module integer not null,
+        data blob not null,
+        foreign key (module) references module(id)
+      );
+    """)
+    db.exec sql"create index TypeByModuleIdx on types(module);"
+    db.exec sql"create index TypeByNimIdIdx on types(nimid);"
+
+    db.exec(sql"""
+      create table if not exists syms(
+        id integer primary key,
+        nimid integer not null,
+        module integer not null,
+        name varchar(256) not null,
+        data blob not null,
+        exported int not null,
+        foreign key (module) references module(id)
+      );
+    """)
+    db.exec sql"create index if not exists SymNameIx on syms(name);"
+    db.exec sql"create index SymByNameAndModuleIdx on syms(name, module);"
+    db.exec sql"create index SymByModuleIdx on syms(module);"
+    db.exec sql"create index SymByNimIdIdx on syms(nimid);"
+
+
+    db.exec(sql"""
+      create table if not exists toplevelstmts(
+        id integer primary key,
+        position integer not null,
+        module integer not null,
+        data blob not null,
+        foreign key (module) references module(id)
+      );
+    """)
+    db.exec sql"create index TopLevelStmtByModuleIdx on toplevelstmts(module);"
+    db.exec sql"create index TopLevelStmtByPositionIdx on toplevelstmts(position);"
+
+    db.exec(sql"""
+      create table if not exists statics(
+        id integer primary key,
+        module integer not null,
+        data blob not null,
+        foreign key (module) references module(id)
+      );
+    """)
+    db.exec sql"create index StaticsByModuleIdx on toplevelstmts(module);"
+    db.exec sql"insert into controlblock(idgen) values (0)"
+
+
+else:
+  type
+    IncrementalCtx* = object
+
+  template init*(incr: IncrementalCtx) = discard
+
+  template addModuleDep*(incr: var IncrementalCtx; conf: ConfigRef;
+                     module, fileIdx: FileIndex;
+                     isIncludeFile: bool) =
+    discard

compiler/injectdestructors.nim

@@ -15,23 +15,14 @@
 
 import
   intsets, strtabs, ast, astalgo, msgs, renderer, magicsys, types, idents,
-  strutils, options, dfa, lowerings, tables, modulegraphs,
+  strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
   lineinfos, parampatterns, sighashes, liftdestructors, optimizer,
   varpartitions
 
 from trees import exprStructuralEquivalent, getRoot
 
 type
-  Con = object
-    owner: PSym
-    g: ControlFlowGraph
-    graph: ModuleGraph
-    inLoop, inSpawn, inLoopCond: int
-    uninit: IntSet # set of uninit'ed vars
-    uninitComputed: bool
-    idgen: IdGenerator
-
-  Scope = object # we do scope-based memory management.
+  Scope = object  # well we do scope-based memory management. \
     # a scope is comparable to an nkStmtListExpr like
     # (try: statements; dest = y(); finally: destructors(); dest)
     vars: seq[PSym]
@@ -40,19 +31,22 @@ type
     needsTry: bool
     parent: ptr Scope
 
+type
+  Con = object
+    owner: PSym
+    g: ControlFlowGraph
+    graph: ModuleGraph
+    otherRead: PNode
+    inLoop, inSpawn, inLoopCond: int
+    uninit: IntSet # set of uninit'ed vars
+    uninitComputed: bool
+
   ProcessMode = enum
     normal
     consumed
     sinkArg
 
 const toDebug {.strdefine.} = ""
-when toDebug.len > 0:
-  var shouldDebug = false
-
-template dbg(body) =
-  when toDebug.len > 0:
-    if shouldDebug:
-      body
 
 proc hasDestructor(c: Con; t: PType): bool {.inline.} =
   result = ast.hasDestructor(t)
@@ -61,8 +55,13 @@ proc hasDestructor(c: Con; t: PType): bool {.inline.} =
     if not result and c.graph.config.selectedGC in {gcArc, gcOrc}:
       assert(not containsGarbageCollectedRef(t))
 
+template dbg(body) =
+  when toDebug.len > 0:
+    if c.owner.name.s == toDebug or toDebug == "always":
+      body
+
 proc getTemp(c: var Con; s: var Scope; typ: PType; info: TLineInfo): PNode =
-  let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), nextSymId c.idgen, c.owner, info)
+  let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), c.owner, info)
   sym.typ = typ
   s.vars.add(sym)
   result = newSymNode(sym)
@@ -73,123 +72,123 @@ proc nestedScope(parent: var Scope): Scope =
 proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode
 proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope; isDecl = false): PNode
 
-import sets, hashes
+proc isLastRead(location: PNode; cfg: ControlFlowGraph; otherRead: var PNode; pc, until: int): int =
+  var pc = pc
+  while pc < cfg.len and pc < until:
+    case cfg[pc].kind
+    of def:
+      if instrTargets(cfg[pc].n, location) == Full:
+        # the path leads to a redefinition of 's' --> abandon it.
+        return high(int)
+      elif instrTargets(cfg[pc].n, location) == Partial:
+        # only partially writes to 's' --> can't sink 's', so this def reads 's'
+        otherRead = cfg[pc].n
+        return -1
+      inc pc
+    of use:
+      if instrTargets(cfg[pc].n, location) != None:
+        otherRead = cfg[pc].n
+        return -1
+      inc pc
+    of goto:
+      pc += cfg[pc].dest
+    of fork:
+      # every branch must lead to the last read of the location:
+      var variantA = pc + 1
+      var variantB = pc + cfg[pc].dest
+      while variantA != variantB:
+        if min(variantA, variantB) < 0: return -1
+        if max(variantA, variantB) >= cfg.len or min(variantA, variantB) >= until:
+          break
+        if variantA < variantB:
+          variantA = isLastRead(location, cfg, otherRead, variantA, min(variantB, until))
+        else:
+          variantB = isLastRead(location, cfg, otherRead, variantB, min(variantA, until))
+      pc = min(variantA, variantB)
+  return pc
 
-proc hash(n: PNode): Hash = hash(cast[pointer](n))
-
-proc aliasesCached(cache: var Table[(PNode, PNode), AliasKind], obj, field: PNode): AliasKind =
-  let key = (obj, field)
-  if not cache.hasKey(key):
-    cache[key] = aliases(obj, field)
-  cache[key]
-
-type
-  State = ref object
-    lastReads: IntSet
-    potentialLastReads: IntSet
-    notLastReads: IntSet
-    alreadySeen: HashSet[PNode]
-
-proc preprocessCfg(cfg: var ControlFlowGraph) =
-  for i in 0..<cfg.len:
-    if cfg[i].kind in {goto, fork} and i + cfg[i].dest > cfg.len:
-      cfg[i].dest = cfg.len - i
-
-proc mergeStates(a: var State, b: sink State) =
-  # Inplace for performance:
-  #   lastReads = a.lastReads + b.lastReads
-  #   potentialLastReads = (a.potentialLastReads + b.potentialLastReads) - (a.notLastReads + b.notLastReads)
-  #   notLastReads = a.notLastReads + b.notLastReads
-  #   alreadySeen = a.alreadySeen + b.alreadySeen
-  # b is never nil
-  if a == nil:
-    a = b
+proc isCursor(n: PNode; c: Con): bool =
+  case n.kind
+  of nkSym:
+    sfCursor in n.sym.flags
+  of nkDotExpr:
+    isCursor(n[1], c)
+  of nkCheckedFieldExpr:
+    isCursor(n[0], c)
   else:
-    a.lastReads.incl b.lastReads
-    a.potentialLastReads.incl b.potentialLastReads
-    a.potentialLastReads.excl a.notLastReads
-    a.potentialLastReads.excl b.notLastReads
-    a.notLastReads.incl b.notLastReads
-    a.alreadySeen.incl b.alreadySeen
-
-proc computeLastReadsAndFirstWrites(cfg: ControlFlowGraph) =
-  var cache = initTable[(PNode, PNode), AliasKind]()
-  template aliasesCached(obj, field: PNode): AliasKind =
-    aliasesCached(cache, obj, field)
-
-  var cfg = cfg
-  preprocessCfg(cfg)
-
-  var states = newSeq[State](cfg.len + 1)
-  states[0] = State()
-
-  for pc in 0..<cfg.len:
-    template state: State = states[pc]
-    if state != nil:
-      case cfg[pc].kind
-      of def:
-        var potentialLastReadsCopy = state.potentialLastReads
-        for r in potentialLastReadsCopy:
-          if cfg[pc].n.aliasesCached(cfg[r].n) == yes:
-            # the path leads to a redefinition of 's' --> sink 's'.
-            state.lastReads.incl r
-            state.potentialLastReads.excl r
-          elif cfg[r].n.aliasesCached(cfg[pc].n) != no:
-            # only partially writes to 's' --> can't sink 's', so this def reads 's'
-            # or maybe writes to 's' --> can't sink 's'
-            cfg[r].n.comment = '\n' & $pc
-            state.potentialLastReads.excl r
-            state.notLastReads.incl r
-
-        var alreadySeenThisNode = false
-        for s in state.alreadySeen:
-          if cfg[pc].n.aliasesCached(s) != no or s.aliasesCached(cfg[pc].n) != no:
-            alreadySeenThisNode = true; break
-        if alreadySeenThisNode: cfg[pc].n.flags.excl nfFirstWrite
-        else: cfg[pc].n.flags.incl nfFirstWrite
-
-        state.alreadySeen.incl cfg[pc].n
-
-        mergeStates(states[pc + 1], move(states[pc]))
-      of use:
-        var potentialLastReadsCopy = state.potentialLastReads
-        for r in potentialLastReadsCopy:
-          if cfg[pc].n.aliasesCached(cfg[r].n) != no or cfg[r].n.aliasesCached(cfg[pc].n) != no:
-            cfg[r].n.comment = '\n' & $pc
-            state.potentialLastReads.excl r
-            state.notLastReads.incl r
-
-        state.potentialLastReads.incl pc
-
-        state.alreadySeen.incl cfg[pc].n
-
-        mergeStates(states[pc + 1], move(states[pc]))
-      of goto:
-        mergeStates(states[pc + cfg[pc].dest], move(states[pc]))
-      of fork:
-        var copy = State()
-        copy[] = states[pc][]
-        mergeStates(states[pc + cfg[pc].dest], copy)
-        mergeStates(states[pc + 1], move(states[pc]))
-
-  let lastReads = (states[^1].lastReads + states[^1].potentialLastReads) - states[^1].notLastReads
-  var lastReadTable: Table[PNode, seq[int]]
-  for position, node in cfg:
-    if node.kind == use:
-      lastReadTable.mgetOrPut(node.n, @[]).add position
-  for node, positions in lastReadTable:
-    block checkIfAllPosLastRead:
-      for p in positions:
-        if p notin lastReads: break checkIfAllPosLastRead
-      node.flags.incl nfLastRead
+    false
 
 proc isLastRead(n: PNode; c: var Con): bool =
+  # first we need to search for the instruction that belongs to 'n':
+  var instr = -1
   let m = dfa.skipConvDfa(n)
-  (m.kind == nkSym and sfSingleUsedTemp in m.sym.flags) or nfLastRead in m.flags
+  if m.kind == nkSym and sfSingleUsedTemp in m.sym.flags: return true
+
+  for i in 0..<c.g.len:
+    # This comparison is correct and MUST not be ``instrTargets``:
+    if c.g[i].kind == use and c.g[i].n == m:
+      if instr < 0:
+        instr = i
+        break
+
+  dbg: echo "starting point for ", n, " is ", instr, " ", n.kind
+
+  if instr < 0: return false
+  # we go through all paths beginning from 'instr+1' and need to
+  # ensure that we don't find another 'use X' instruction.
+  if instr+1 >= c.g.len: return true
+
+  c.otherRead = nil
+  result = isLastRead(n, c.g, c.otherRead, instr+1, int.high) >= 0
+  dbg: echo "ugh ", c.otherRead.isNil, " ", result
+
+proc isFirstWrite(location: PNode; cfg: ControlFlowGraph; pc, until: int): int =
+  var pc = pc
+  while pc < until:
+    case cfg[pc].kind
+    of def:
+      if instrTargets(cfg[pc].n, location) != None:
+        # a definition of 's' before ours makes ours not the first write
+        return -1
+      inc pc
+    of use:
+      if instrTargets(cfg[pc].n, location) != None:
+        return -1
+      inc pc
+    of goto:
+      pc += cfg[pc].dest
+    of fork:
+      # every branch must not contain a def/use of our location:
+      var variantA = pc + 1
+      var variantB = pc + cfg[pc].dest
+      while variantA != variantB:
+        if min(variantA, variantB) < 0: return -1
+        if max(variantA, variantB) > until:
+          break
+        if variantA < variantB:
+          variantA = isFirstWrite(location, cfg, variantA, min(variantB, until))
+        else:
+          variantB = isFirstWrite(location, cfg, variantB, min(variantA, until))
+      pc = min(variantA, variantB)
+  return pc
 
 proc isFirstWrite(n: PNode; c: var Con): bool =
+  # first we need to search for the instruction that belongs to 'n':
+  var instr = -1
   let m = dfa.skipConvDfa(n)
-  nfFirstWrite in m.flags
+
+  for i in countdown(c.g.len-1, 0): # We search backwards here to treat loops correctly
+    if c.g[i].kind == def and c.g[i].n == m:
+      if instr < 0:
+        instr = i
+        break
+
+  if instr < 0: return false
+  # we go through all paths going to 'instr' and need to
+  # ensure that we don't find another 'def/use X' instruction.
+  if instr == 0: return true
+
+  result = isFirstWrite(n, c.g, 0, instr) >= 0
 
 proc initialized(code: ControlFlowGraph; pc: int,
                  init, uninit: var IntSet; until: int): int =
@@ -229,17 +228,6 @@ proc initialized(code: ControlFlowGraph; pc: int,
       inc pc
   return pc
 
-proc isCursor(n: PNode): bool =
-  case n.kind
-  of nkSym:
-    sfCursor in n.sym.flags
-  of nkDotExpr:
-    isCursor(n[1])
-  of nkCheckedFieldExpr:
-    isCursor(n[0])
-  else:
-    false
-
 template isUnpackedTuple(n: PNode): bool =
   ## we move out all elements of unpacked tuples,
   ## hence unpacked tuples themselves don't need to be destroyed
@@ -251,9 +239,9 @@ proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
     m.add "; requires a copy because it's not the last read of '"
     m.add renderTree(ri)
     m.add '\''
-    if ri.comment.startsWith('\n'):
+    if c.otherRead != nil:
       m.add "; another read is done here: "
-      m.add c.graph.config $ c.g[parseInt(ri.comment[1..^1])].n.info
+      m.add c.graph.config $ c.otherRead.info
     elif ri.kind == nkSym and ri.sym.kind == skParam and not isSinkType(ri.sym.typ):
       m.add "; try to make "
       m.add renderTree(ri)
@@ -262,28 +250,28 @@ proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
   m.add c.owner.name.s
   localError(c.graph.config, ri.info, errGenerated, m)
 
-proc makePtrType(c: var Con, baseType: PType): PType =
-  result = newType(tyPtr, nextTypeId c.idgen, c.owner)
-  addSonSkipIntLit(result, baseType, c.idgen)
+proc makePtrType(c: Con, baseType: PType): PType =
+  result = newType(tyPtr, c.owner)
+  addSonSkipIntLit(result, baseType)
 
-proc genOp(c: var Con; op: PSym; dest: PNode): PNode =
+proc genOp(c: Con; op: PSym; dest: PNode): PNode =
   let addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
   addrExp.add(dest)
   result = newTree(nkCall, newSymNode(op), addrExp)
 
-proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode =
-  var op = getAttachedOp(c.graph, t, kind)
-  if op == nil or op.ast.isGenericRoutine:
+proc genOp(c: Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode =
+  var op = t.attachedOps[kind]
+  if op == nil or op.ast[genericParamsPos].kind != nkEmpty:
     # give up and find the canonical type instead:
-    let h = sighashes.hashType(t, c.graph.config, {CoType, CoConsiderOwned, CoDistinct})
+    let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
     let canon = c.graph.canonTypes.getOrDefault(h)
     if canon != nil:
-      op = getAttachedOp(c.graph, canon, kind)
+      op = canon.attachedOps[kind]
   if op == nil:
     #echo dest.typ.id
     globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &
       "' operator not found for type " & typeToString(t))
-  elif op.ast.isGenericRoutine:
+  elif op.ast[genericParamsPos].kind != nkEmpty:
     globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &
       "' operator is generic")
   dbg:
@@ -292,16 +280,16 @@ proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode
   if sfError in op.flags: checkForErrorPragma(c, t, ri, AttachedOpToStr[kind])
   c.genOp(op, dest)
 
-proc genDestroy(c: var Con; dest: PNode): PNode =
+proc genDestroy(c: Con; dest: PNode): PNode =
   let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink})
   result = c.genOp(t, attachedDestructor, dest, nil)
 
 proc canBeMoved(c: Con; t: PType): bool {.inline.} =
   let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
   if optOwnedRefs in c.graph.config.globalOptions:
-    result = t.kind != tyRef and getAttachedOp(c.graph, t, attachedSink) != nil
+    result = t.kind != tyRef and t.attachedOps[attachedSink] != nil
   else:
-    result = getAttachedOp(c.graph, t, attachedSink) != nil
+    result = t.attachedOps[attachedSink] != nil
 
 proc isNoInit(dest: PNode): bool {.inline.} =
   result = dest.kind == nkSym and sfNoInit in dest.sym.flags
@@ -314,7 +302,7 @@ proc genSink(c: var Con; dest, ri: PNode, isDecl = false): PNode =
     result = newTree(nkFastAsgn, dest, ri)
   else:
     let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink})
-    if getAttachedOp(c.graph, t, attachedSink) != nil:
+    if t.attachedOps[attachedSink] != nil:
       result = c.genOp(t, attachedSink, dest, ri)
       result.add ri
     else:
@@ -322,55 +310,17 @@ proc genSink(c: var Con; dest, ri: PNode, isDecl = false): PNode =
       # and copyMem(dest, source). This is efficient.
       result = newTree(nkStmtList, c.genDestroy(dest), newTree(nkFastAsgn, dest, ri))
 
-proc isCriticalLink(dest: PNode): bool {.inline.} =
-  #[
-  Lins's idea that only "critical" links can introduce a cycle. This is
-  critical for the performance gurantees that we strive for: If you
-  traverse a data structure, no tracing will be performed at all.
-  ORC is about this promise: The GC only touches the memory that the
-  mutator touches too.
-
-  These constructs cannot possibly create cycles::
-
-    local = ...
-
-    new(x)
-    dest = ObjectConstructor(field: noalias(dest))
-
-  But since 'ObjectConstructor' is already moved into 'dest' all we really have
-  to look for is assignments to local variables.
-  ]#
-  result = dest.kind != nkSym
-
-proc finishCopy(c: var Con; result, dest: PNode; isFromSink: bool) =
-  if c.graph.config.selectedGC == gcOrc:
-    let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink, tyDistinct})
-    if cyclicType(c.graph, t):
-      result.add boolLit(c.graph, result.info, isFromSink or isCriticalLink(dest))
-
-proc genMarkCyclic(c: var Con; result, dest: PNode) =
-  if c.graph.config.selectedGC == gcOrc:
-    let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink, tyDistinct})
-    if cyclicType(c.graph, t):
-      if t.kind == tyRef:
-        result.add callCodegenProc(c.graph, "nimMarkCyclic", dest.info, dest)
-      else:
-        let xenv = genBuiltin(c.graph, c.idgen, mAccessEnv, "accessEnv", dest)
-        xenv.typ = getSysType(c.graph, dest.info, tyPointer)
-        result.add callCodegenProc(c.graph, "nimMarkCyclic", dest.info, xenv)
-
-proc genCopyNoCheck(c: var Con; dest, ri: PNode): PNode =
+proc genCopyNoCheck(c: Con; dest, ri: PNode): PNode =
   let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink})
   result = c.genOp(t, attachedAsgn, dest, ri)
-  assert ri.typ != nil
 
 proc genCopy(c: var Con; dest, ri: PNode): PNode =
   let t = dest.typ
   if tfHasOwned in t.flags and ri.kind != nkNilLit:
     # try to improve the error message here:
+    if c.otherRead == nil: discard isLastRead(ri, c)
     c.checkForErrorPragma(t, ri, "=copy")
   result = c.genCopyNoCheck(dest, ri)
-  assert ri.typ != nil
 
 proc genDiscriminantAsgn(c: var Con; s: var Scope; n: PNode): PNode =
   # discriminator is ordinal value that doesn't need sink destroy
@@ -388,35 +338,35 @@ proc genDiscriminantAsgn(c: var Con; s: var Scope; n: PNode): PNode =
   let objType = leDotExpr[0].typ
 
   if hasDestructor(c, objType):
-    if getAttachedOp(c.graph, objType, attachedDestructor) != nil and
-        sfOverriden in getAttachedOp(c.graph, objType, attachedDestructor).flags:
+    if objType.attachedOps[attachedDestructor] != nil and
+        sfOverriden in objType.attachedOps[attachedDestructor].flags:
       localError(c.graph.config, n.info, errGenerated, """Assignment to discriminant for objects with user defined destructor is not supported, object must have default destructor.
 It is best to factor out piece of object that needs custom destructor into separate object or not use discriminator assignment""")
       result.add newTree(nkFastAsgn, le, tmp)
       return
 
     # generate: if le != tmp: `=destroy`(le)
-    let branchDestructor = produceDestructorForDiscriminator(c.graph, objType, leDotExpr[1].sym, n.info, c.idgen)
+    let branchDestructor = produceDestructorForDiscriminator(c.graph, objType, leDotExpr[1].sym, n.info)
     let cond = newNodeIT(nkInfix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
     cond.add newSymNode(getMagicEqSymForType(c.graph, le.typ, n.info))
     cond.add le
     cond.add tmp
     let notExpr = newNodeIT(nkPrefix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
-    notExpr.add newSymNode(createMagic(c.graph, c.idgen, "not", mNot))
+    notExpr.add newSymNode(createMagic(c.graph, "not", mNot))
     notExpr.add cond
     result.add newTree(nkIfStmt, newTree(nkElifBranch, notExpr, c.genOp(branchDestructor, le)))
   result.add newTree(nkFastAsgn, le, tmp)
 
 proc genWasMoved(c: var Con, n: PNode): PNode =
   result = newNodeI(nkCall, n.info)
-  result.add(newSymNode(createMagic(c.graph, c.idgen, "wasMoved", mWasMoved)))
+  result.add(newSymNode(createMagic(c.graph, "wasMoved", mWasMoved)))
   result.add copyTree(n) #mWasMoved does not take the address
   #if n.kind != nkSym:
   #  message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")")
 
 proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode =
   result = newNodeI(nkCall, info)
-  result.add(newSymNode(createMagic(c.graph, c.idgen, "default", mDefault)))
+  result.add(newSymNode(createMagic(c.graph, "default", mDefault)))
   result.typ = t
 
 proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode =
@@ -427,7 +377,7 @@ proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode =
   else:
     result = newNodeIT(nkStmtListExpr, n.info, n.typ)
 
-    var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), nextSymId c.idgen, c.owner, n.info)
+    var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), c.owner, n.info)
     temp.typ = n.typ
     var v = newNodeI(nkLetSection, n.info)
     let tempAsNode = newSymNode(temp)
@@ -439,9 +389,7 @@ proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode =
     v.add(vpart)
 
     result.add v
-    let nn = skipConv(n)
-    c.genMarkCyclic(result, nn)
-    let wasMovedCall = c.genWasMoved(nn)
+    let wasMovedCall = c.genWasMoved(skipConv(n))
     result.add wasMovedCall
     result.add tempAsNode
 
@@ -456,7 +404,6 @@ proc passCopyToSink(n: PNode; c: var Con; s: var Scope): PNode =
     result.add c.genWasMoved(tmp)
     var m = c.genCopy(tmp, n)
     m.add p(n, c, s, normal)
-    c.finishCopy(m, n, isFromSink = true)
     result.add m
     if isLValue(n) and not isCapturedVar(n) and n.typ.skipTypes(abstractInst).kind != tyRef and c.inSpawn == 0:
       message(c.graph.config, n.info, hintPerformance,
@@ -535,19 +482,23 @@ proc cycleCheck(n: PNode; c: var Con) =
       message(c.graph.config, n.info, warnCycleCreated, msg)
       break
 
-proc pVarTopLevel(v: PNode; c: var Con; s: var Scope; res: PNode) =
+proc pVarTopLevel(v: PNode; c: var Con; s: var Scope; ri, res: PNode) =
   # move the variable declaration to the top of the frame:
   s.vars.add v.sym
   if isUnpackedTuple(v):
     if c.inLoop > 0:
       # unpacked tuple needs reset at every loop iteration
       res.add newTree(nkFastAsgn, v, genDefaultCall(v.typ, c, v.info))
-  elif sfThread notin v.sym.flags and sfCursor notin v.sym.flags:
+  elif sfThread notin v.sym.flags:
     # do not destroy thread vars for now at all for consistency.
     if sfGlobal in v.sym.flags and s.parent == nil: #XXX: Rethink this logic (see tarcmisc.test2)
       c.graph.globalDestructors.add c.genDestroy(v)
     else:
       s.final.add c.genDestroy(v)
+  if ri.kind == nkEmpty and c.inLoop > 0:
+    res.add moveOrCopy(v, genDefaultCall(v.typ, c, v.info), c, s, isDecl = true)
+  elif ri.kind != nkEmpty:
+    res.add moveOrCopy(v, ri, c, s, isDecl = true)
 
 proc processScope(c: var Con; s: var Scope; ret: PNode): PNode =
   result = newNodeI(nkStmtList, ret.info)
@@ -573,14 +524,13 @@ proc processScope(c: var Con; s: var Scope; ret: PNode): PNode =
 
 template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: untyped): PNode =
   assert not ret.typ.isEmptyType
-  var result = newNodeIT(nkStmtListExpr, ret.info, ret.typ)
+  var result = newNodeI(nkStmtListExpr, ret.info)
   # There is a possibility to do this check: s.wasMoved.len > 0 or s.final.len > 0
   # later and use it to eliminate the temporary when theres no need for it, but its
   # tricky because you would have to intercept moveOrCopy at a certain point
   let tmp = c.getTemp(s.parent[], ret.typ, ret.info)
   tmp.sym.flags.incl sfSingleUsedTemp
   let cpy = if hasDestructor(c, ret.typ):
-              s.parent[].final.add c.genDestroy(tmp)
               moveOrCopy(tmp, ret, c, s, isDecl = true)
             else:
               newTree(nkFastAsgn, tmp, p(ret, c, s, normal))
@@ -706,11 +656,6 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false) =
   of nkWhen: # This should be a "when nimvm" node.
     result = copyTree(n)
     result[1][0] = processCall(n[1][0], s)
-  of nkPragmaBlock:
-    result = shallowCopy(n)
-    for i in 0 ..< n.len-1:
-      result[i] = p(n[i], c, s, normal)
-    result[^1] = maybeVoid(n[^1], s)
   else: assert(false)
 
 proc pRaiseStmt(n: PNode, c: var Con; s: var Scope): PNode =
@@ -723,7 +668,6 @@ proc pRaiseStmt(n: PNode, c: var Con; s: var Scope): PNode =
       let tmp = c.getTemp(s, n[0].typ, n.info)
       var m = c.genCopyNoCheck(tmp, n[0])
       m.add p(n[0], c, s, normal)
-      c.finishCopy(m, n[0], isFromSink = false)
       result = newTree(nkStmtList, c.genWasMoved(tmp), m)
       var toDisarm = n[0]
       if toDisarm.kind == nkStmtListExpr: toDisarm = toDisarm.lastSon
@@ -740,7 +684,7 @@ proc pRaiseStmt(n: PNode, c: var Con; s: var Scope): PNode =
 
 proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
   if n.kind in {nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkIfStmt,
-                nkIfExpr, nkCaseStmt, nkWhen, nkWhileStmt, nkParForStmt, nkTryStmt, nkPragmaBlock}:
+                nkIfExpr, nkCaseStmt, nkWhen, nkWhileStmt, nkParForStmt, nkTryStmt}:
     template process(child, s): untyped = p(child, c, s, mode)
     handleNestedTempl(n, process)
   elif mode == sinkArg:
@@ -753,7 +697,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
          nkCallKinds + nkLiterals:
       result = p(n, c, s, consumed)
     elif ((n.kind == nkSym and isSinkParam(n.sym)) or isAnalysableFieldAccess(n, c.owner)) and
-        isLastRead(n, c) and not (n.kind == nkSym and isCursor(n)):
+        isLastRead(n, c) and not (n.kind == nkSym and isCursor(n, c)):
       # Sinked params can be consumed only once. We need to reset the memory
       # to disable the destructor which we have not elided
       result = destructiveMoveVar(n, c, s)
@@ -779,7 +723,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       result = passCopyToSink(n, c, s)
   else:
     case n.kind
-    of nkBracket, nkTupleConstr, nkClosure, nkCurly:
+    of nkBracket, nkObjConstr, nkTupleConstr, nkClosure, nkCurly:
       # Let C(x) be the construction, 'x' the vector of arguments.
       # C(x) either owns 'x' or it doesn't.
       # If C(x) owns its data, we must consume C(x).
@@ -790,34 +734,18 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       # don't destroy it"
       # but if C(x) is a ref it MUST own its data since we must destroy it
       # so then we have no choice but to use 'sinkArg'.
-      let m = if mode == normal: normal
-              else: sinkArg
-
-      result = copyTree(n)
-      for i in ord(n.kind == nkClosure)..<n.len:
-        if n[i].kind == nkExprColonExpr:
-          result[i][1] = p(n[i][1], c, s, m)
-        elif n[i].kind == nkRange:
-          result[i][0] = p(n[i][0], c, s, m)
-          result[i][1] = p(n[i][1], c, s, m)
-        else:
-          result[i] = p(n[i], c, s, m)
-    of nkObjConstr:
-      # see also the remark about `nkTupleConstr`.
-      let t = n.typ.skipTypes(abstractInst)
-      let isRefConstr = t.kind == tyRef
+      let isRefConstr = n.kind == nkObjConstr and n.typ.skipTypes(abstractInst).kind == tyRef
       let m = if isRefConstr: sinkArg
               elif mode == normal: normal
               else: sinkArg
 
       result = copyTree(n)
-      for i in 1..<n.len:
+      for i in ord(n.kind in {nkObjConstr, nkClosure})..<n.len:
         if n[i].kind == nkExprColonExpr:
-          let field = lookupFieldAgain(t, n[i][0].sym)
-          if field != nil and sfCursor in field.flags:
-            result[i][1] = p(n[i][1], c, s, normal)
-          else:
-            result[i][1] = p(n[i][1], c, s, m)
+          result[i][1] = p(n[i][1], c, s, m)
+        elif n[i].kind == nkRange:
+          result[i][0] = p(n[i][0], c, s, m)
+          result[i][1] = p(n[i][1], c, s, m)
         else:
           result[i] = p(n[i], c, s, m)
       if mode == normal and isRefConstr:
@@ -873,21 +801,19 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       for it in n:
         var ri = it[^1]
         if it.kind == nkVarTuple and hasDestructor(c, ri.typ):
-          let x = lowerTupleUnpacking(c.graph, it, c.idgen, c.owner)
+          let x = lowerTupleUnpacking(c.graph, it, c.owner)
           result.add p(x, c, s, consumed)
-        elif it.kind == nkIdentDefs and hasDestructor(c, it[0].typ):
+        elif it.kind == nkIdentDefs and hasDestructor(c, it[0].typ) and not isCursor(it[0], c):
           for j in 0..<it.len-2:
             let v = it[j]
             if v.kind == nkSym:
               if sfCompileTime in v.sym.flags: continue
-              pVarTopLevel(v, c, s, result)
-            if ri.kind != nkEmpty:
-              result.add moveOrCopy(v, ri, c, s, isDecl = v.kind == nkSym)
-            elif ri.kind == nkEmpty and c.inLoop > 0:
-              let skipInit = v.kind == nkDotExpr and # Closure var
-                             sfNoInit in v[1].sym.flags
-              if not skipInit:
-                result.add moveOrCopy(v, genDefaultCall(v.typ, c, v.info), c, s, isDecl = v.kind == nkSym)
+              pVarTopLevel(v, c, s, ri, result)
+            else:
+              if ri.kind == nkEmpty and c.inLoop > 0:
+                ri = genDefaultCall(v.typ, c, v.info)
+              if ri.kind != nkEmpty:
+                result.add moveOrCopy(v, ri, c, s, isDecl = false)
         else: # keep the var but transform 'ri':
           var v = copyNode(n)
           var itCopy = copyNode(it)
@@ -897,7 +823,8 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
           v.add itCopy
           result.add v
     of nkAsgn, nkFastAsgn:
-      if hasDestructor(c, n[0].typ) and n[1].kind notin {nkProcDef, nkDo, nkLambda}:
+      if hasDestructor(c, n[0].typ) and n[1].kind notin {nkProcDef, nkDo, nkLambda} and
+          not isCursor(n[0], c):
         if n[0].kind in {nkDotExpr, nkCheckedFieldExpr}:
           cycleCheck(n, c)
         assert n[1].kind notin {nkAsgn, nkFastAsgn}
@@ -920,7 +847,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
        nkTypeOfExpr, nkMixinStmt, nkBindStmt:
       result = n
 
-    of nkStringToCString, nkCStringToString, nkChckRangeF, nkChckRange64, nkChckRange:
+    of nkStringToCString, nkCStringToString, nkChckRangeF, nkChckRange64, nkChckRange, nkPragmaBlock:
       result = shallowCopy(n)
       for i in 0 ..< n.len:
         result[i] = p(n[i], c, s, normal)
@@ -1000,129 +927,64 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
     else:
       internalError(c.graph.config, n.info, "cannot inject destructors to node kind: " & $n.kind)
 
-proc sameLocation*(a, b: PNode): bool =
-  proc sameConstant(a, b: PNode): bool =
-    a.kind in nkLiterals and b.kind in nkLiterals and a.intVal == b.intVal
-
-  const nkEndPoint = {nkSym, nkDotExpr, nkCheckedFieldExpr, nkBracketExpr}
-  if a.kind in nkEndPoint and b.kind in nkEndPoint:
-    if a.kind == b.kind:
-      case a.kind
-      of nkSym: a.sym == b.sym
-      of nkDotExpr, nkCheckedFieldExpr: sameLocation(a[0], b[0]) and sameLocation(a[1], b[1])
-      of nkBracketExpr: sameLocation(a[0], b[0]) and sameConstant(a[1], b[1])
-      else: false
-    else: false
-  else:
-    case a.kind
-    of nkSym, nkDotExpr, nkCheckedFieldExpr, nkBracketExpr:
-      # Reached an endpoint, flip to recurse the other side.
-      sameLocation(b, a)
-    of nkAddr, nkHiddenAddr, nkDerefExpr, nkHiddenDeref:
-      # We don't need to check addr/deref levels or differentiate between the two,
-      # since pointers don't have hooks :) (e.g: var p: ptr pointer; p[] = addr p)
-      sameLocation(a[0], b)
-    of nkObjDownConv, nkObjUpConv: sameLocation(a[0], b)
-    of nkHiddenStdConv, nkHiddenSubConv: sameLocation(a[1], b)
-    else: false
-
-proc genFieldAccessSideEffects(c: var Con; dest, ri: PNode, isDecl: bool): PNode =
-  # with side effects
-  var temp = newSym(skLet, getIdent(c.graph.cache, "bracketTmp"), nextSymId c.idgen, c.owner, ri[1].info)
-  temp.typ = ri[1].typ
-  var v = newNodeI(nkLetSection, ri[1].info)
-  let tempAsNode = newSymNode(temp)
-
-  var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
-  vpart[0] = tempAsNode
-  vpart[1] = newNodeI(nkEmpty, tempAsNode.info)
-  vpart[2] = ri[1]
-  v.add(vpart)
-
-  var newAccess = copyNode(ri)
-  newAccess.add ri[0]
-  newAccess.add tempAsNode
-
-  var snk = c.genSink(dest, newAccess, isDecl)
-  result = newTree(nkStmtList, v, snk, c.genWasMoved(newAccess))
-
 proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, isDecl = false): PNode =
-  if sameLocation(dest, ri):
-    # rule (self-assignment-removal):
-    result = newNodeI(nkEmpty, dest.info)
-  elif isCursor(dest):
-    case ri.kind:
-    of nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt:
-      template process(child, s): untyped = moveOrCopy(dest, child, c, s, isDecl)
-      # We know the result will be a stmt so we use that fact to optimize
-      handleNestedTempl(ri, process, willProduceStmt = true)
+  case ri.kind
+  of nkCallKinds:
+    result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
+  of nkBracketExpr:
+    if isUnpackedTuple(ri[0]):
+      # unpacking of tuple: take over the elements
+      result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
+    elif isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c) and
+        not aliases(dest, ri):
+      # Rule 3: `=sink`(x, z); wasMoved(z)
+      var snk = c.genSink(dest, ri, isDecl)
+      result = newTree(nkStmtList, snk, c.genWasMoved(ri))
     else:
-      result = newTree(nkFastAsgn, dest, p(ri, c, s, normal))
+      result = c.genCopy(dest, ri)
+      result.add p(ri, c, s, consumed)
+  of nkBracket:
+    # array constructor
+    if ri.len > 0 and isDangerousSeq(ri.typ):
+      result = c.genCopy(dest, ri)
+      result.add p(ri, c, s, consumed)
+    else:
+      result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
+  of nkObjConstr, nkTupleConstr, nkClosure, nkCharLit..nkNilLit:
+    result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
+  of nkSym:            
+    if dest.kind == nkSym and dest.sym == ri.sym:
+      # rule (self-assignment-removal):
+      result = newNodeI(nkEmpty, dest.info)
+    elif isSinkParam(ri.sym) and isLastRead(ri, c):
+      # Rule 3: `=sink`(x, z); wasMoved(z)
+      let snk = c.genSink(dest, ri, isDecl)
+      result = newTree(nkStmtList, snk, c.genWasMoved(ri))
+    elif ri.sym.kind != skParam and ri.sym.owner == c.owner and
+        isLastRead(ri, c) and canBeMoved(c, dest.typ) and not isCursor(ri, c):
+      # Rule 3: `=sink`(x, z); wasMoved(z)
+      let snk = c.genSink(dest, ri, isDecl)
+      result = newTree(nkStmtList, snk, c.genWasMoved(ri))
+    else:
+      result = c.genCopy(dest, ri)
+      result.add p(ri, c, s, consumed)
+  of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv:
+    result = c.genSink(dest, p(ri, c, s, sinkArg), isDecl)
+  of nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt:
+    template process(child, s): untyped = moveOrCopy(dest, child, c, s, isDecl)
+    # We know the result will be a stmt so we use that fact to optimize
+    handleNestedTempl(ri, process, willProduceStmt = true)
+  of nkRaiseStmt:
+    result = pRaiseStmt(ri, c, s)
   else:
-    let ri2 = if ri.kind == nkWhen: ri[1][0] else: ri
-    case ri2.kind
-    of nkCallKinds:
-      result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
-    of nkBracketExpr:
-      if isUnpackedTuple(ri[0]):
-        # unpacking of tuple: take over the elements
-        result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
-      elif isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c):
-        if aliases(dest, ri) == no:
-          # Rule 3: `=sink`(x, z); wasMoved(z)
-          if isAtom(ri[1]):
-            var snk = c.genSink(dest, ri, isDecl)
-            result = newTree(nkStmtList, snk, c.genWasMoved(ri))
-          else:
-            result = genFieldAccessSideEffects(c, dest, ri, isDecl)
-        else:
-          result = c.genSink(dest, destructiveMoveVar(ri, c, s), isDecl)
-      else:
-        result = c.genCopy(dest, ri)
-        result.add p(ri, c, s, consumed)
-        c.finishCopy(result, dest, isFromSink = false)
-    of nkBracket:
-      # array constructor
-      if ri.len > 0 and isDangerousSeq(ri.typ):
-        result = c.genCopy(dest, ri)
-        result.add p(ri, c, s, consumed)
-        c.finishCopy(result, dest, isFromSink = false)
-      else:
-        result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
-    of nkObjConstr, nkTupleConstr, nkClosure, nkCharLit..nkNilLit:
-      result = c.genSink(dest, p(ri, c, s, consumed), isDecl)
-    of nkSym:
-      if isSinkParam(ri.sym) and isLastRead(ri, c):
-        # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, isDecl)
-        result = newTree(nkStmtList, snk, c.genWasMoved(ri))
-      elif ri.sym.kind != skParam and ri.sym.owner == c.owner and
-          isLastRead(ri, c) and canBeMoved(c, dest.typ) and not isCursor(ri):
-        # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, isDecl)
-        result = newTree(nkStmtList, snk, c.genWasMoved(ri))
-      else:
-        result = c.genCopy(dest, ri)
-        result.add p(ri, c, s, consumed)
-        c.finishCopy(result, dest, isFromSink = false)
-    of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv, nkCast:
-      result = c.genSink(dest, p(ri, c, s, sinkArg), isDecl)
-    of nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt:
-      template process(child, s): untyped = moveOrCopy(dest, child, c, s, isDecl)
-      # We know the result will be a stmt so we use that fact to optimize
-      handleNestedTempl(ri, process, willProduceStmt = true)
-    of nkRaiseStmt:
-      result = pRaiseStmt(ri, c, s)
+    if isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c) and
+        canBeMoved(c, dest.typ):
+      # Rule 3: `=sink`(x, z); wasMoved(z)
+      let snk = c.genSink(dest, ri, isDecl)
+      result = newTree(nkStmtList, snk, c.genWasMoved(ri))
     else:
-      if isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c) and
-          canBeMoved(c, dest.typ):
-        # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, isDecl)
-        result = newTree(nkStmtList, snk, c.genWasMoved(ri))
-      else:
-        result = c.genCopy(dest, ri)
-        result.add p(ri, c, s, consumed)
-        c.finishCopy(result, dest, isFromSink = false)
+      result = c.genCopy(dest, ri)
+      result.add p(ri, c, s, consumed)
 
 proc computeUninit(c: var Con) =
   if not c.uninitComputed:
@@ -1150,21 +1012,17 @@ proc injectDefaultCalls(n: PNode, c: var Con) =
     for i in 0..<n.safeLen:
       injectDefaultCalls(n[i], c)
 
-proc injectDestructorCalls*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; n: PNode): PNode =
-  when toDebug.len > 0:
-    shouldDebug = toDebug == owner.name.s or toDebug == "always"
+proc injectDestructorCalls*(g: ModuleGraph; owner: PSym; n: PNode): PNode =
   if sfGeneratedOp in owner.flags or (owner.kind == skIterator and isInlineIterator(owner.typ)):
     return n
-  var c = Con(owner: owner, graph: g, g: constructCfg(owner, n), idgen: idgen)
+  var c = Con(owner: owner, graph: g, g: constructCfg(owner, n))
   dbg:
     echo "\n### ", owner.name.s, ":\nCFG:"
     echoCfg(c.g)
     echo n
 
   if optCursorInference in g.config.options:
-    computeCursors(owner, n, g)
-
-  computeLastReadsAndFirstWrites(c.g)
+    computeCursors(owner, n, g.config)
 
   var scope: Scope
   let body = p(n, c, scope, normal)

compiler/installer.ini

@@ -13,7 +13,6 @@ Platforms: """
   netbsd: i386;amd64
   openbsd: i386;amd64;arm;arm64
   dragonfly: i386;amd64
-  crossos: amd64
   haiku: i386;amd64
   android: i386;arm;arm64
   nintendoswitch: arm64
@@ -77,6 +76,7 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/fusion"
 
 Files: "tests"
 
@@ -120,6 +120,7 @@ Files: "bin/nim"
 InstallScript: "yes"
 UninstallScript: "yes"
 Files: "bin/nim-gdb"
+Files: "bin/nim-gdb.bash"
 
 
 [InnoSetup]
@@ -141,5 +142,5 @@ shortDesc: "The Nim Compiler"
 licenses: "bin/nim,MIT;lib/*,MIT;"
 
 [nimble]
-pkgName: "nim"
+pkgName: "compiler"
 pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt;doc/nimdoc.css"

compiler/int128.nim

@@ -1,13 +1,13 @@
 ## This module is for compiler internal use only. For reliable error
 ## messages and range checks, the compiler needs a data type that can
-## hold all from `low(BiggestInt)` to `high(BiggestUInt)`, This
+## hold all from ``low(BiggestInt)`` to ``high(BiggestUInt)``, This
 ## type is for that purpose.
 
 from math import trunc
 
 type
   Int128* = object
-    udata: array[4, uint32]
+    udata: array[4,uint32]
 
 template sdata(arg: Int128, idx: int): int32 =
   # udata and sdata was supposed to be in a union, but unions are
@@ -17,12 +17,12 @@ template sdata(arg: Int128, idx: int): int32 =
 # encoding least significant int first (like LittleEndian)
 
 const
-  Zero* = Int128(udata: [0'u32, 0, 0, 0])
-  One* = Int128(udata: [1'u32, 0, 0, 0])
-  Ten* = Int128(udata: [10'u32, 0, 0, 0])
-  Min = Int128(udata: [0'u32, 0, 0, 0x80000000'u32])
-  Max = Int128(udata: [high(uint32), high(uint32), high(uint32), uint32(high(int32))])
-  NegOne* = Int128(udata: [0xffffffff'u32, 0xffffffff'u32, 0xffffffff'u32, 0xffffffff'u32])
+  Zero* = Int128(udata: [0'u32,0,0,0])
+  One* = Int128(udata: [1'u32,0,0,0])
+  Ten* = Int128(udata: [10'u32,0,0,0])
+  Min = Int128(udata: [0'u32,0,0,0x80000000'u32])
+  Max = Int128(udata: [high(uint32),high(uint32),high(uint32),uint32(high(int32))])
+  NegOne* = Int128(udata: [0xffffffff'u32,0xffffffff'u32,0xffffffff'u32,0xffffffff'u32])
 
 template low*(t: typedesc[Int128]): Int128 = Min
 template high*(t: typedesc[Int128]): Int128 = Max
@@ -57,10 +57,10 @@ template isNegative(arg: Int128): bool =
 template isNegative(arg: int32): bool =
   arg < 0
 
-proc bitconcat(a, b: uint32): uint64 =
+proc bitconcat(a,b: uint32): uint64 =
   (uint64(a) shl 32) or uint64(b)
 
-proc bitsplit(a: uint64): (uint32, uint32) =
+proc bitsplit(a: uint64): (uint32,uint32) =
   (cast[uint32](a shr 32), cast[uint32](a))
 
 proc toInt64*(arg: Int128): int64 =
@@ -176,6 +176,7 @@ proc toHex*(arg: Int128): string =
   result.addToHex(arg)
 
 proc inc*(a: var Int128, y: uint32 = 1) =
+  let input = a
   a.udata[0] += y
   if unlikely(a.udata[0] < y):
     a.udata[1].inc
@@ -185,7 +186,7 @@ proc inc*(a: var Int128, y: uint32 = 1) =
         a.udata[3].inc
         doAssert(a.sdata(3) != low(int32), "overflow")
 
-proc cmp*(a, b: Int128): int =
+proc cmp*(a,b: Int128): int =
   let tmp1 = cmp(a.sdata(3), b.sdata(3))
   if tmp1 != 0: return tmp1
   let tmp2 = cmp(a.udata[2], b.udata[2])
@@ -195,13 +196,13 @@ proc cmp*(a, b: Int128): int =
   let tmp4 = cmp(a.udata[0], b.udata[0])
   return tmp4
 
-proc `<`*(a, b: Int128): bool =
-  cmp(a, b) < 0
+proc `<`*(a,b: Int128): bool =
+  cmp(a,b) < 0
 
-proc `<=`*(a, b: Int128): bool =
-  cmp(a, b) <= 0
+proc `<=`*(a,b: Int128): bool =
+  cmp(a,b) <= 0
 
-proc `==`*(a, b: Int128): bool =
+proc `==`*(a,b: Int128): bool =
   if a.udata[0] != b.udata[0]: return false
   if a.udata[1] != b.udata[1]: return false
   if a.udata[2] != b.udata[2]: return false
@@ -220,19 +221,19 @@ proc bitnot*(a: Int128): Int128 =
   result.udata[2] = not a.udata[2]
   result.udata[3] = not a.udata[3]
 
-proc bitand*(a, b: Int128): Int128 =
+proc bitand*(a,b: Int128): Int128 =
   result.udata[0] = a.udata[0] and b.udata[0]
   result.udata[1] = a.udata[1] and b.udata[1]
   result.udata[2] = a.udata[2] and b.udata[2]
   result.udata[3] = a.udata[3] and b.udata[3]
 
-proc bitor*(a, b: Int128): Int128 =
+proc bitor*(a,b: Int128): Int128 =
   result.udata[0] = a.udata[0] or b.udata[0]
   result.udata[1] = a.udata[1] or b.udata[1]
   result.udata[2] = a.udata[2] or b.udata[2]
   result.udata[3] = a.udata[3] or b.udata[3]
 
-proc bitxor*(a, b: Int128): Int128 =
+proc bitxor*(a,b: Int128): Int128 =
   result.udata[0] = a.udata[0] xor b.udata[0]
   result.udata[1] = a.udata[1] xor b.udata[1]
   result.udata[2] = a.udata[2] xor b.udata[2]
@@ -287,7 +288,7 @@ proc `shl`*(a: Int128, b: int): Int128 =
     result.udata[2] = 0
     result.udata[3] = a.udata[0] shl (b and 31)
 
-proc `+`*(a, b: Int128): Int128 =
+proc `+`*(a,b: Int128): Int128 =
   let tmp0 = uint64(a.udata[0]) + uint64(b.udata[0])
   result.udata[0] = cast[uint32](tmp0)
   let tmp1 = uint64(a.udata[1]) + uint64(b.udata[1]) + (tmp0 shr 32)
@@ -304,7 +305,7 @@ proc `-`*(a: Int128): Int128 =
   result = bitnot(a)
   result.inc
 
-proc `-`*(a, b: Int128): Int128 =
+proc `-`*(a,b: Int128): Int128 =
   a + (-b)
 
 proc `-=`*(a: var Int128, b: Int128) =
@@ -341,7 +342,7 @@ proc `*`*(a: Int128, b: int32): Int128 =
 proc `*=`*(a: var Int128, b: int32): Int128 =
   result = result * b
 
-proc makeInt128(high, low: uint64): Int128 =
+proc makeInt128(high,low: uint64): Int128 =
   result.udata[0] = cast[uint32](low)
   result.udata[1] = cast[uint32](low shr 32)
   result.udata[2] = cast[uint32](high)
@@ -353,7 +354,7 @@ proc high64(a: Int128): uint64 =
 proc low64(a: Int128): uint64 =
   bitconcat(a.udata[1], a.udata[0])
 
-proc `*`*(lhs, rhs: Int128): Int128 =
+proc `*`*(lhs,rhs: Int128): Int128 =
   let
     a = cast[uint64](lhs.udata[0])
     b = cast[uint64](lhs.udata[1])
@@ -388,7 +389,7 @@ proc fastLog2*(a: Int128): int =
   if a.udata[1] != 0:
     return 32 + fastLog2(a.udata[1])
   if a.udata[0] != 0:
-    return fastLog2(a.udata[0])
+    return      fastLog2(a.udata[0])
 
 proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
   assert(divisor != Zero)
@@ -440,12 +441,12 @@ proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
   else:
     result.remainder = dividend
 
-proc `div`*(a, b: Int128): Int128 =
-  let (a, b) = divMod(a, b)
+proc `div`*(a,b: Int128): Int128 =
+  let (a,b) = divMod(a,b)
   return a
 
-proc `mod`*(a, b: Int128): Int128 =
-  let (a, b) = divMod(a, b)
+proc `mod`*(a,b: Int128): Int128 =
+  let (a,b) = divMod(a,b)
   return b
 
 proc addInt128*(result: var string; value: Int128) =
@@ -476,7 +477,7 @@ proc `$`*(a: Int128): string =
 
 proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =
   assert(pos < arg.len)
-  assert(arg[pos] in {'-', '0'..'9'})
+  assert(arg[pos] in {'-','0'..'9'})
 
   var isNegative = false
   var pos = pos
@@ -496,13 +497,13 @@ proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =
 # fluff
 
 proc `<`*(a: Int128, b: BiggestInt): bool =
-  cmp(a, toInt128(b)) < 0
+  cmp(a,toInt128(b)) < 0
 
 proc `<`*(a: BiggestInt, b: Int128): bool =
   cmp(toInt128(a), b) < 0
 
 proc `<=`*(a: Int128, b: BiggestInt): bool =
-  cmp(a, toInt128(b)) <= 0
+  cmp(a,toInt128(b)) <= 0
 
 proc `<=`*(a: BiggestInt, b: Int128): bool =
   cmp(toInt128(a), b) <= 0
@@ -538,7 +539,7 @@ proc toFloat64*(arg: Int128): float64 =
 
 proc ldexp(x: float64, exp: cint): float64 {.importc: "ldexp", header: "<math.h>".}
 
-template bitor(a, b, c: Int128): Int128 = bitor(bitor(a, b), c)
+template bitor(a,b,c: Int128): Int128 = bitor(bitor(a,b), c)
 
 proc toInt128*(arg: float64): Int128 =
   let isNegative = arg < 0

compiler/isolation_check.nim

@@ -11,7 +11,7 @@
 ## https://github.com/nim-lang/RFCs/issues/244 for more details.
 
 import
-  ast, types, renderer, intsets
+  ast, types, renderer, idents, intsets
 
 proc canAlias(arg, ret: PType; marker: var IntSet): bool
 
@@ -64,7 +64,7 @@ proc canAlias(arg, ret: PType; marker: var IntSet): bool =
   else:
     result = false
 
-proc isValueOnlyType(t: PType): bool =
+proc isValueOnlyType(t: PType): bool = 
   # t doesn't contain pointers and references
   proc wrap(t: PType): bool {.nimcall.} = t.kind in {tyRef, tyPtr, tyVar, tyLent}
   result = not types.searchTypeFor(t, wrap)
@@ -77,17 +77,6 @@ proc canAlias*(arg, ret: PType): bool =
     var marker = initIntSet()
     result = canAlias(arg, ret, marker)
 
-proc containsVariable(n: PNode): bool =
-  case n.kind
-  of nodesToIgnoreSet:
-    result = false
-  of nkSym:
-    result = n.sym.kind in {skForVar, skParam, skVar, skLet, skConst, skResult, skTemp}
-  else:
-    for ch in n:
-      if containsVariable(ch): return true
-    result = false
-
 proc checkIsolate*(n: PNode): bool =
   if types.containsTyRef(n.typ):
     # XXX Maybe require that 'n.typ' is acyclic. This is not much
@@ -96,10 +85,7 @@ proc checkIsolate*(n: PNode): bool =
     of nkCharLit..nkNilLit:
       result = true
     of nkCallKinds:
-      # XXX: as long as we don't update the analysis while examining arguments
-      #      we can do an early check of the return type, otherwise this is a
-      #      bug and needs to be moved below
-      if tfNoSideEffect notin n[0].typ.flags:
+      if n[0].typ.flags * {tfGcSafe, tfNoSideEffect} == {}:
         return false
       for i in 1..<n.len:
         if checkIsolate(n[i]):
@@ -107,23 +93,14 @@ proc checkIsolate*(n: PNode): bool =
         else:
           let argType = n[i].typ
           if argType != nil and not isCompileTimeOnly(argType) and containsTyRef(argType):
-            if argType.canAlias(n.typ) or containsVariable(n[i]):
-              # bug #19013: Alias information is not enough, we need to check for potential
-              # "overlaps". I claim the problem can only happen by reading again from a location
-              # that materialized which is only possible if a variable that contains a `ref`
-              # is involved.
+            if argType.canAlias(n.typ):
               return false
       result = true
     of nkIfStmt, nkIfExpr:
       for it in n:
         result = checkIsolate(it.lastSon)
         if not result: break
-    of nkCaseStmt:
-      for i in 1..<n.len:
-        result = checkIsolate(n[i].lastSon)
-        if not result: break
-    of nkObjConstr:
-      result = true
+    of nkCaseStmt, nkObjConstr:
       for i in 1..<n.len:
         result = checkIsolate(n[i].lastSon)
         if not result: break
@@ -146,3 +123,4 @@ proc checkIsolate*(n: PNode): bool =
   else:
     # no ref, no cry:
     result = true
+

compiler/jsgen.nim

@@ -33,10 +33,12 @@ import
   nversion, msgs, idents, types,
   ropes, passes, ccgutils, wordrecg, renderer,
   cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils,
-  transf, injectdestructors, sourcemap, astmsgs
+  transf, injectdestructors, sourcemap
 
 import json, sets, math, tables, intsets, strutils
 
+from modulegraphs import ModuleGraph, PPassContext
+
 type
   TJSGen = object of PPassContext
     module: PSym
@@ -178,7 +180,7 @@ const
 proc mapType(typ: PType): TJSTypeKind =
   let t = skipTypes(typ, abstractInst)
   case t.kind
-  of tyVar, tyRef, tyPtr:
+  of tyVar, tyRef, tyPtr, tyLent:
     if skipTypes(t.lastSon, abstractInst).kind in MappedToObject:
       result = etyObject
     else:
@@ -186,8 +188,7 @@ proc mapType(typ: PType): TJSTypeKind =
   of tyPointer:
     # treat a tyPointer like a typed pointer to an array of bytes
     result = etyBaseIndex
-  of tyRange, tyDistinct, tyOrdinal, tyProxy, tyLent:
-    # tyLent is no-op as JS has pass-by-reference semantics
+  of tyRange, tyDistinct, tyOrdinal, tyProxy:
     result = mapType(t[0])
   of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar: result = etyInt
   of tyBool: result = etyBool
@@ -209,8 +210,8 @@ proc mapType(typ: PType): TJSTypeKind =
     if t.n != nil: result = mapType(lastSon t)
     else: result = etyNone
   of tyProc: result = etyProc
-  of tyCstring: result = etyString
-  of tyConcept, tyIterable: doAssert false
+  of tyCString: result = etyString
+  of tyOptDeprecated: doAssert false
 
 proc mapType(p: PProc; typ: PType): TJSTypeKind =
   result = mapType(typ)
@@ -259,7 +260,7 @@ proc mangleName(m: BModule, s: PSym): Rope =
       if m.config.hcrOn:
         # When hot reloading is enabled, we must ensure that the names
         # of functions and types will be preserved across rebuilds:
-        result.add(idOrSig(s, m.module.name.s, m.sigConflicts, m.config))
+        result.add(idOrSig(s, m.module.name.s, m.sigConflicts))
       else:
         result.add("_")
         result.add(rope(s.id))
@@ -321,7 +322,7 @@ proc isSimpleExpr(p: PProc; n: PNode): bool =
 
 proc getTemp(p: PProc, defineInLocals: bool = true): Rope =
   inc(p.unique)
-  result = "Temporary$1" % [rope(p.unique)]
+  result = "Tmp$1" % [rope(p.unique)]
   if defineInLocals:
     p.locals.add(p.indentLine("var $1;$n" % [result]))
 
@@ -488,12 +489,12 @@ proc maybeMakeTempAssignable(p: PProc, n: PNode; x: TCompRes): tuple[a, tmp: Rop
       if optBoundsCheck in p.options:
         useMagic(p, "chckIndx")
         if first == 0: # save a couple chars
-          index.res = "chckIndx($1, 0, ($2).length - 1)" % [index.res, tmp1]
+          index.res = "chckIndx($1, 0, ($2).length-1)" % [index.res, tmp1]
         else:
-          index.res = "chckIndx($1, $2, ($3).length + ($2) - 1) - ($2)" % [
+          index.res = "chckIndx($1, $2, ($3).length+($2)-1)-($2)" % [
             index.res, rope(first), tmp1]
       elif first != 0:
-        index.res = "($1) - ($2)" % [index.res, rope(first)]
+        index.res = "($1)-($2)" % [index.res, rope(first)]
       else:
         discard # index.res = index.res
       let (n1, tmp2) = maybeMakeTemp(p, n[1], index)
@@ -602,16 +603,16 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mMulF64: applyFormat("($1 * $2)", "($1 * $2)")
   of mDivF64: applyFormat("($1 / $2)", "($1 / $2)")
   of mShrI: applyFormat("", "")
-  of mShlI:
+  of mShlI: 
     if n[1].typ.size <= 4:
       applyFormat("($1 << $2)", "($1 << $2)")
     else:
-      applyFormat("($1 * Math.pow(2, $2))", "($1 * Math.pow(2, $2))")
-  of mAshrI:
+      applyFormat("($1 * Math.pow(2,$2))", "($1 * Math.pow(2,$2))")
+  of mAshrI: 
     if n[1].typ.size <= 4:
       applyFormat("($1 >> $2)", "($1 >> $2)")
     else:
-      applyFormat("Math.floor($1 / Math.pow(2, $2))", "Math.floor($1 / Math.pow(2, $2))")
+      applyFormat("Math.floor($1 / Math.pow(2,$2))", "Math.floor($1 / Math.pow(2,$2))")
   of mBitandI: applyFormat("($1 & $2)", "($1 & $2)")
   of mBitorI: applyFormat("($1 | $2)", "($1 | $2)")
   of mBitxorI: applyFormat("($1 ^ $2)", "($1 ^ $2)")
@@ -655,10 +656,13 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mUnaryMinusF64: applyFormat("-($1)", "-($1)")
   of mCharToStr: applyFormat("nimCharToStr($1)", "nimCharToStr($1)")
   of mBoolToStr: applyFormat("nimBoolToStr($1)", "nimBoolToStr($1)")
-  of mIntToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")")
-  of mInt64ToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")")
+  of mIntToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
+  of mInt64ToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
+  of mFloatToStr:
+    useMagic(p, "nimFloatToString")
+    applyFormat "cstrToNimstr(nimFloatToString($1))"
   of mCStrToStr: applyFormat("cstrToNimstr($1)", "cstrToNimstr($1)")
-  of mStrToStr, mUnown, mIsolate, mFinished: applyFormat("$1", "$1")
+  of mStrToStr, mUnown, mIsolate: applyFormat("$1", "$1")
   else:
     assert false, $op
 
@@ -667,10 +671,7 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mAddU: binaryUintExpr(p, n, r, "+")
   of mSubU: binaryUintExpr(p, n, r, "-")
   of mMulU: binaryUintExpr(p, n, r, "*")
-  of mDivU:
-    binaryUintExpr(p, n, r, "/")
-    if n[1].typ.skipTypes(abstractRange).size == 8:
-      r.res = "Math.trunc($1)" % [r.res]
+  of mDivU: binaryUintExpr(p, n, r, "/")
   of mDivI:
     arithAux(p, n, r, op)
   of mModI:
@@ -679,8 +680,10 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
     var x, y: TCompRes
     gen(p, n[1], x)
     gen(p, n[2], y)
-    r.res = "($1 >>> $2)" % [x.rdLoc, y.rdLoc]
-  of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr, mStrToStr, mEnumToStr:
+    let trimmer = unsignedTrimmer(n[1].typ.skipTypes(abstractRange).size)
+    r.res = "(($1 $2) >>> $3)" % [x.rdLoc, trimmer, y.rdLoc]
+  of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr,
+      mCStrToStr, mStrToStr, mEnumToStr:
     arithAux(p, n, r, op)
   of mEqRef:
     if mapType(n[1].typ) != etyBaseIndex:
@@ -699,7 +702,7 @@ proc hasFrameInfo(p: PProc): bool =
       ((p.prc == nil) or not (sfPure in p.prc.flags))
 
 proc lineDir(config: ConfigRef, info: TLineInfo, line: int): Rope =
-  ropes.`%`("/* line $2 \"$1\" */$n",
+  ropes.`%`("// line $2 \"$1\"$n",
          [rope(toFullPath(config, info)), rope(line)])
 
 proc genLineDir(p: PProc, n: PNode) =
@@ -720,9 +723,9 @@ proc genWhileStmt(p: PProc, n: PNode) =
   p.blocks[^1].id = -p.unique
   p.blocks[^1].isLoop = true
   let labl = p.unique.rope
-  lineF(p, "Label$1: while (true) {$n", [labl])
+  lineF(p, "L$1: while (true) {$n", [labl])
   p.nested: gen(p, n[0], cond)
-  lineF(p, "if (!$1) break Label$2;$n",
+  lineF(p, "if (!$1) break L$2;$n",
        [cond.res, labl])
   p.nested: genStmt(p, n[1])
   lineF(p, "}$n", [labl])
@@ -745,8 +748,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   #  try {
   #    stmts;
   #    --excHandler;
-  #  } catch (EXCEPTION) {
-  #    var prevJSError = lastJSError; lastJSError = EXCEPTION;
+  #  } catch (EXC) {
+  #    var prevJSError = lastJSError; lastJSError = EXC;
   #    framePtr = tmpFramePtr;
   #    --excHandler;
   #    if (e.typ && e.typ == NTI433 || e.typ == NTI2321) {
@@ -771,16 +774,18 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   if catchBranchesExist:
     p.body.add("++excHandler;\L")
   var tmpFramePtr = rope"F"
+  if optStackTrace notin p.options:
+    tmpFramePtr = p.getTemp(true)
+    line(p, tmpFramePtr & " = framePtr;\L")
   lineF(p, "try {$n", [])
   var a: TCompRes
   gen(p, n[0], a)
   moveInto(p, a, r)
   var generalCatchBranchExists = false
   if catchBranchesExist:
-    p.body.addf("--excHandler;$n} catch (EXCEPTION) {$n var prevJSError = lastJSError;$n" &
-        " lastJSError = EXCEPTION;$n --excHandler;$n", [])
-    if hasFrameInfo(p):
-      line(p, "framePtr = $1;$n" % [tmpFramePtr])
+    p.body.addf("--excHandler;$n} catch (EXC) {$n var prevJSError = lastJSError;$n" &
+        " lastJSError = EXC;$n --excHandler;$n", [])
+    line(p, "framePtr = $1;$n" % [tmpFramePtr])
   while i < n.len and n[i].kind == nkExceptBranch:
     if n[i].len == 1:
       # general except section:
@@ -839,8 +844,7 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
       line(p, "}\L")
     lineF(p, "lastJSError = prevJSError;$n")
   line(p, "} finally {\L")
-  if hasFrameInfo(p):
-    line(p, "framePtr = $1;$n" % [tmpFramePtr])
+  line(p, "framePtr = $1;$n" % [tmpFramePtr])
   if i < n.len and n[i].kind == nkFinally:
     genStmt(p, n[i][0])
   line(p, "}\L")
@@ -884,7 +888,7 @@ proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
           var v = copyNode(e[0])
           inc(totalRange, int(e[1].intVal - v.intVal))
           if totalRange > 65535:
-            localError(p.config, n.info,
+            localError(p.config, n.info, 
                        "Your case statement contains too many branches, consider using if/else instead!")
           while v.intVal <= e[1].intVal:
             gen(p, v, cond)
@@ -922,12 +926,12 @@ proc genBlock(p: PProc, n: PNode, r: var TCompRes) =
     sym.loc.k = locOther
     sym.position = idx+1
   let labl = p.unique
-  lineF(p, "Label$1: do {$n", [labl.rope])
+  lineF(p, "L$1: do {$n", [labl.rope])
   setLen(p.blocks, idx + 1)
   p.blocks[idx].id = - p.unique # negative because it isn't used yet
   gen(p, n[1], r)
   setLen(p.blocks, idx)
-  lineF(p, "} while (false);$n", [labl.rope])
+  lineF(p, "} while(false);$n", [labl.rope])
 
 proc genBreakStmt(p: PProc, n: PNode) =
   var idx: int
@@ -945,7 +949,7 @@ proc genBreakStmt(p: PProc, n: PNode) =
     if idx < 0 or not p.blocks[idx].isLoop:
       internalError(p.config, n.info, "no loop to break")
   p.blocks[idx].id = abs(p.blocks[idx].id) # label is used
-  lineF(p, "break Label$1;$n", [rope(p.blocks[idx].id)])
+  lineF(p, "break L$1;$n", [rope(p.blocks[idx].id)])
 
 proc genAsmOrEmitStmt(p: PProc, n: PNode) =
   genLineDir(p, n)
@@ -1040,14 +1044,14 @@ proc needsNoCopy(p: PProc; y: PNode): bool =
   return y.kind in nodeKindsNeedNoCopy or
         ((mapType(y.typ) != etyBaseIndex or (y.kind == nkSym and y.sym.kind == skParam)) and
           (skipTypes(y.typ, abstractInst).kind in
-            {tyRef, tyPtr, tyLent, tyVar, tyCstring, tyProc, tyOwned} + IntegralTypes))
+            {tyRef, tyPtr, tyLent, tyVar, tyCString, tyProc, tyOwned} + IntegralTypes))
 
 proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
   var a, b: TCompRes
   var xtyp = mapType(p, x.typ)
 
   # disable `[]=` for cstring
-  if x.kind == nkBracketExpr and x.len >= 2 and x[0].typ.skipTypes(abstractInst).kind == tyCstring:
+  if x.kind == nkBracketExpr and x.len >= 2 and x[0].typ.skipTypes(abstractInst).kind == tyCString:
     localError(p.config, x.info, "cstring doesn't support `[]=` operator")
 
   gen(p, x, a)
@@ -1060,24 +1064,19 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
     xtyp = etySeq
   case xtyp
   of etySeq:
-    if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
+    if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
       lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
       lineF(p, "$1 = nimCopy(null, $2, $3);$n",
                [a.rdLoc, b.res, genTypeInfo(p, y.typ)])
   of etyObject:
-    if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
+    if x.typ.kind in {tyVar} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
       lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
-      # supports proc getF(): var T
-      if x.kind in {nkHiddenDeref, nkDerefExpr} and x[0].kind in nkCallKinds:
-          lineF(p, "nimCopy($1, $2, $3);$n",
-                [a.res, b.res, genTypeInfo(p, x.typ)])
-      else:
-        lineF(p, "$1 = nimCopy($1, $2, $3);$n",
-              [a.res, b.res, genTypeInfo(p, x.typ)])
+      lineF(p, "nimCopy($1, $2, $3);$n",
+               [a.res, b.res, genTypeInfo(p, y.typ)])
   of etyBaseIndex:
     if a.typ != etyBaseIndex or b.typ != etyBaseIndex:
       if y.kind == nkCall:
@@ -1092,18 +1091,10 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
         lineF(p, "$# = [$#, $#];$n", [a.res, b.address, b.res])
         lineF(p, "$1 = $2;$n", [a.address, b.res])
         lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
-      elif a.typ == etyBaseIndex:
-        # array indexing may not map to var type
-        if b.address != nil:
-          lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res])
-        else:
-          lineF(p, "$1 = $2;$n", [a.address, b.res])
       else:
         internalError(p.config, x.info, $("genAsgn", b.typ, a.typ))
-    elif b.address != nil:
-      lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res])
     else:
-      lineF(p, "$1 = $2;$n", [a.address, b.res])
+      lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res])
   else:
     lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
 
@@ -1217,13 +1208,12 @@ proc genCheckedFieldOp(p: PProc, n: PNode, addrTyp: PType, r: var TCompRes) =
   let tmp = p.getTemp()
   lineF(p, "var $1 = $2;$n", tmp, obj.res)
 
-  useMagic(p, "raiseFieldError2")
+  useMagic(p, "raiseFieldError")
   useMagic(p, "makeNimstrLit")
-  useMagic(p, "reprDiscriminant") # no need to offset by firstOrd unlike for cgen
-  let msg = genFieldDefect(p.config, field.name.s, disc)
-  lineF(p, "if ($1[$2.$3]$4undefined) { raiseFieldError2(makeNimstrLit($5), reprDiscriminant($2.$3, $6)); }$n",
+  let msg = genFieldDefect(field, disc)
+  lineF(p, "if ($1[$2.$3]$4undefined) { raiseFieldError(makeNimstrLit($5)); }$n",
     setx.res, tmp, disc.loc.r, if negCheck: ~"!==" else: ~"===",
-    makeJSString(msg), genTypeInfo(p, disc.typ))
+    makeJSString(msg))
 
   if addrTyp != nil and mapType(p, addrTyp) == etyBaseIndex:
     r.typ = etyBaseIndex
@@ -1251,12 +1241,12 @@ proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) =
   if optBoundsCheck in p.options:
     useMagic(p, "chckIndx")
     if first == 0: # save a couple chars
-      r.res = "chckIndx($1, 0, ($2).length - 1)" % [b.res, tmp]
+      r.res = "chckIndx($1, 0, ($2).length-1)" % [b.res, tmp]
     else:
-      r.res = "chckIndx($1, $2, ($3).length + ($2) - 1) - ($2)" % [
+      r.res = "chckIndx($1, $2, ($3).length+($2)-1)-($2)" % [
         b.res, rope(first), tmp]
   elif first != 0:
-    r.res = "($1) - ($2)" % [b.res, rope(first)]
+    r.res = "($1)-($2)" % [b.res, rope(first)]
   else:
     r.res = b.res
   r.kind = resExpr
@@ -1265,14 +1255,14 @@ proc genArrayAccess(p: PProc, n: PNode, r: var TCompRes) =
   var ty = skipTypes(n[0].typ, abstractVarRange)
   if ty.kind in {tyRef, tyPtr, tyLent, tyOwned}: ty = skipTypes(ty.lastSon, abstractVarRange)
   case ty.kind
-  of tyArray, tyOpenArray, tySequence, tyString, tyCstring, tyVarargs:
+  of tyArray, tyOpenArray, tySequence, tyString, tyCString, tyVarargs:
     genArrayAddr(p, n, r)
   of tyTuple:
     genFieldAddr(p, n, r)
   else: internalError(p.config, n.info, "expr(nkBracketExpr, " & $ty.kind & ')')
   r.typ = mapType(n.typ)
   if r.res == nil: internalError(p.config, n.info, "genArrayAccess")
-  if ty.kind == tyCstring:
+  if ty.kind == tyCString:
     r.res = "$1.charCodeAt($2)" % [r.address, r.res]
   elif r.typ == etyBaseIndex:
     if needsTemp(p, n[0]):
@@ -1296,89 +1286,75 @@ template isIndirect(x: PSym): bool =
     v.kind notin {skProc, skFunc, skConverter, skMethod, skIterator,
                   skConst, skTemp, skLet})
 
-proc genSymAddr(p: PProc, n: PNode, typ: PType, r: var TCompRes) = 
-  let s = n.sym
-  if s.loc.r == nil: internalError(p.config, n.info, "genAddr: 3")
-  case s.kind
-  of skParam:
-    r.res = s.loc.r
-    r.address = nil
-    r.typ = etyNone
-  of skVar, skLet, skResult:
-    r.kind = resExpr
-    let jsType = mapType(p):
-      if typ.isNil:
-        n.typ
-      else:
-        typ
-    if jsType == etyObject:
-      # make addr() a no-op:
-      r.typ = etyNone
-      if isIndirect(s):
-        r.res = s.loc.r & "[0]"
-      else:
-        r.res = s.loc.r
-      r.address = nil
-    elif {sfGlobal, sfAddrTaken} * s.flags != {} or jsType == etyBaseIndex:
-      # for ease of code generation, we do not distinguish between
-      # sfAddrTaken and sfGlobal.
-      r.typ = etyBaseIndex
-      r.address = s.loc.r
-      r.res = rope("0")
-    else:
-      # 'var openArray' for instance produces an 'addr' but this is harmless:
-      gen(p, n, r)
-      #internalError(p.config, n.info, "genAddr: 4 " & renderTree(n))
-  else: internalError(p.config, n.info, $("genAddr: 2", s.kind))
-
 proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
-  if n.kind == nkSym:
-    genSymAddr(p, n, nil, r)
-  else:
-    case n[0].kind
-    of nkSym:
-      genSymAddr(p, n[0], n.typ, r)
-    of nkCheckedFieldExpr:
-      genCheckedFieldOp(p, n[0], n.typ, r)
-    of nkDotExpr:
-      if mapType(p, n.typ) == etyBaseIndex:
-        genFieldAddr(p, n[0], r)
+  case n[0].kind
+  of nkSym:
+    let s = n[0].sym
+    if s.loc.r == nil: internalError(p.config, n.info, "genAddr: 3")
+    case s.kind
+    of skParam:
+      r.res = s.loc.r
+      r.address = nil
+      r.typ = etyNone
+    of skVar, skLet, skResult:
+      r.kind = resExpr
+      let jsType = mapType(p, n.typ)
+      if jsType == etyObject:
+        # make addr() a no-op:
+        r.typ = etyNone
+        if isIndirect(s):
+          r.res = s.loc.r & "[0]"
+        else:
+          r.res = s.loc.r
+        r.address = nil
+      elif {sfGlobal, sfAddrTaken} * s.flags != {} or jsType == etyBaseIndex:
+        # for ease of code generation, we do not distinguish between
+        # sfAddrTaken and sfGlobal.
+        r.typ = etyBaseIndex
+        r.address = s.loc.r
+        r.res = rope("0")
       else:
-        genFieldAccess(p, n[0], r)
-    of nkBracketExpr:
-      var ty = skipTypes(n[0].typ, abstractVarRange)
-      if ty.kind in MappedToObject:
-        gen(p, n[0], r)
-      else:
-        let kindOfIndexedExpr = skipTypes(n[0][0].typ, abstractVarRange).kind
-        case kindOfIndexedExpr
-        of tyArray, tyOpenArray, tySequence, tyString, tyCstring, tyVarargs:
-          genArrayAddr(p, n[0], r)
-        of tyTuple:
-          genFieldAddr(p, n[0], r)
-        of tyGenericBody:
-          genAddr(p, n[^1], r)
-        else: internalError(p.config, n[0].info, "expr(nkBracketExpr, " & $kindOfIndexedExpr & ')')
-    of nkObjDownConv:
-      gen(p, n[0], r)
-    of nkHiddenDeref:
-      gen(p, n[0], r)
-    of nkHiddenAddr:
-      gen(p, n[0], r)
-    of nkConv:
-      genAddr(p, n[0], r)
-    of nkStmtListExpr:
-      if n.len == 1: gen(p, n[0], r)
-      else: internalError(p.config, n[0].info, "genAddr for complex nkStmtListExpr")
-    of nkCallKinds:
-      if n[0].typ.kind == tyOpenArray:
         # 'var openArray' for instance produces an 'addr' but this is harmless:
-        # namely toOpenArray(a, 1, 3)
         gen(p, n[0], r)
-      else:
-        internalError(p.config, n[0].info, "genAddr: " & $n[0].kind)
+        #internalError(p.config, n.info, "genAddr: 4 " & renderTree(n))
+    else: internalError(p.config, n.info, $("genAddr: 2", s.kind))
+  of nkCheckedFieldExpr:
+    genCheckedFieldOp(p, n[0], n.typ, r)
+  of nkDotExpr:
+    if mapType(p, n.typ) == etyBaseIndex:
+      genFieldAddr(p, n[0], r)
+    else:
+      genFieldAccess(p, n[0], r)
+  of nkBracketExpr:
+    var ty = skipTypes(n[0].typ, abstractVarRange)
+    if ty.kind in MappedToObject:
+      gen(p, n[0], r)
+    else:
+      let kindOfIndexedExpr = skipTypes(n[0][0].typ, abstractVarRange).kind
+      case kindOfIndexedExpr
+      of tyArray, tyOpenArray, tySequence, tyString, tyCString, tyVarargs:
+        genArrayAddr(p, n[0], r)
+      of tyTuple:
+        genFieldAddr(p, n[0], r)
+      else: internalError(p.config, n[0].info, "expr(nkBracketExpr, " & $kindOfIndexedExpr & ')')
+  of nkObjDownConv:
+    gen(p, n[0], r)
+  of nkHiddenDeref:
+    gen(p, n[0], r)
+  of nkHiddenAddr:
+    gen(p, n[0], r)
+  of nkStmtListExpr:
+    if n.len == 1: gen(p, n[0], r)
+    else: internalError(p.config, n[0].info, "genAddr for complex nkStmtListExpr")
+  of nkCallKinds:
+    if n[0].typ.kind == tyOpenArray:
+      # 'var openArray' for instance produces an 'addr' but this is harmless:
+      # namely toOpenArray(a, 1, 3)
+      gen(p, n[0], r)
     else:
       internalError(p.config, n[0].info, "genAddr: " & $n[0].kind)
+  else:
+    internalError(p.config, n[0].info, "genAddr: " & $n[0].kind)
 
 proc attachProc(p: PProc; content: Rope; s: PSym) =
   p.g.code.add(content)
@@ -1451,10 +1427,10 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
           s.name.s)
     discard mangleName(p.module, s)
     r.res = s.loc.r
-    if lfNoDecl in s.loc.flags or s.magic notin generatedMagics or
+    if lfNoDecl in s.loc.flags or s.magic notin {mNone, mIsolate} or
        {sfImportc, sfInfixCall} * s.flags != {}:
       discard
-    elif s.kind == skMethod and getBody(p.module.graph, s).kind == nkEmpty:
+    elif s.kind == skMethod and s.getBody.kind == nkEmpty:
       # we cannot produce code for the dispatcher yet:
       discard
     elif sfForward in s.flags:
@@ -1464,17 +1440,13 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
   else:
     if s.loc.r == nil:
       internalError(p.config, n.info, "symbol has no generated name: " & s.name.s)
-    if mapType(p, s.typ) == etyBaseIndex:
-      r.address = s.loc.r
-      r.res = s.loc.r & "_Idx"
-    else:
-      r.res = s.loc.r
+    r.res = s.loc.r
   r.kind = resVal
 
 proc genDeref(p: PProc, n: PNode, r: var TCompRes) =
   let it = n[0]
   let t = mapType(p, it.typ)
-  if t == etyObject or it.typ.kind == tyLent:
+  if t == etyObject:
     gen(p, it, r)
   else:
     var a: TCompRes
@@ -1715,7 +1687,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
       result = putToSeq("0", indirect)
   of tyFloat..tyFloat128:
     result = putToSeq("0.0", indirect)
-  of tyRange, tyGenericInst, tyAlias, tySink, tyOwned, tyLent:
+  of tyRange, tyGenericInst, tyAlias, tySink, tyOwned:
     result = createVar(p, lastSon(typ), indirect)
   of tySet:
     result = putToSeq("{}", indirect)
@@ -1757,14 +1729,14 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
     createObjInitList(p, t, initIntSet(), initList)
     result = ("({$1})") % [initList]
     if indirect: result = "[$1]" % [result]
-  of tyVar, tyPtr, tyRef, tyPointer:
+  of tyVar, tyPtr, tyLent, tyRef, tyPointer:
     if mapType(p, t) == etyBaseIndex:
       result = putToSeq("[null, 0]", indirect)
     else:
       result = putToSeq("null", indirect)
   of tySequence, tyString:
     result = putToSeq("[]", indirect)
-  of tyCstring, tyProc:
+  of tyCString, tyProc:
     result = putToSeq("null", indirect)
   of tyStatic:
     if t.n != nil:
@@ -1795,7 +1767,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
       inc p.extraIndent
     elif useGlobalPragmas:
       lineF(p, "if (globalThis.$1 === undefined) {$n", varName)
-      varCode = "globalThis." & $varName
+      varCode = $varName
       inc p.extraIndent
     else:
       varCode = "var $2"
@@ -1858,7 +1830,7 @@ proc genVarStmt(p: PProc, n: PNode) =
     var a = n[i]
     if a.kind != nkCommentStmt:
       if a.kind == nkVarTuple:
-        let unpacked = lowerTupleUnpacking(p.module.graph, a, p.module.idgen, p.prc)
+        let unpacked = lowerTupleUnpacking(p.module.graph, a, p.prc)
         genStmt(p, unpacked)
       else:
         assert(a.kind == nkIdentDefs)
@@ -1897,13 +1869,13 @@ proc genNewSeq(p: PProc, n: PNode) =
   gen(p, n[1], x)
   gen(p, n[2], y)
   let t = skipTypes(n[1].typ, abstractVar)[0]
-  lineF(p, "$1 = new Array($2); for (var i = 0 ; i < $2 ; ++i) { $1[i] = $3; }", [
+  lineF(p, "$1 = new Array($2); for (var i=0;i<$2;++i) {$1[i]=$3;}", [
     x.rdLoc, y.rdLoc, createVar(p, t, false)])
 
 proc genOrd(p: PProc, n: PNode, r: var TCompRes) =
   case skipTypes(n[1].typ, abstractVar + abstractRange).kind
   of tyEnum, tyInt..tyUInt64, tyChar: gen(p, n[1], r)
-  of tyBool: unaryExpr(p, n, r, "", "($1 ? 1 : 0)")
+  of tyBool: unaryExpr(p, n, r, "", "($1 ? 1:0)")
   else: internalError(p.config, n.info, "genOrd")
 
 proc genConStrStr(p: PProc, n: PNode, r: var TCompRes) =
@@ -2014,23 +1986,6 @@ proc genMove(p: PProc; n: PNode; r: var TCompRes) =
   genReset(p, n)
   #lineF(p, "$1 = $2;$n", [dest.rdLoc, src.rdLoc])
 
-proc genJSArrayConstr(p: PProc, n: PNode, r: var TCompRes) =
-  var a: TCompRes
-  r.res = rope("[")
-  r.kind = resExpr
-  for i in 0 ..< n.len:
-    if i > 0: r.res.add(", ")
-    gen(p, n[i], a)
-    if a.typ == etyBaseIndex:
-      r.res.addf("[$1, $2]", [a.address, a.res])
-    else:
-      if not needsNoCopy(p, n[i]):
-        let typ = n[i].typ.skipTypes(abstractInst)
-        useMagic(p, "nimCopy")
-        a.res = "nimCopy(null, $1, $2)" % [a.rdLoc, genTypeInfo(p, typ)]
-      r.res.add(a.res)
-  r.res.add("]")
-
 proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
   var
     a: TCompRes
@@ -2050,7 +2005,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
     gen(p, n[1], lhs)
     gen(p, n[2], rhs)
 
-    if skipTypes(n[1].typ, abstractVarRange).kind == tyCstring:
+    if skipTypes(n[1].typ, abstractVarRange).kind == tyCString:
       let (b, tmp) = maybeMakeTemp(p, n[2], rhs)
       r.res = "if (null != $1) { if (null == $2) $2 = $3; else $2 += $3; }" %
         [b, lhs.rdLoc, tmp]
@@ -2094,22 +2049,19 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
   of mNew, mNewFinalize: genNew(p, n)
   of mChr: gen(p, n[1], r)
   of mArrToSeq:
-    # only array literals doesn't need copy
-    if n[1].kind == nkBracket:
-      genJSArrayConstr(p, n[1], r)
+    if needsNoCopy(p, n[1]):
+      gen(p, n[1], r)
     else:
       var x: TCompRes
       gen(p, n[1], x)
       useMagic(p, "nimCopy")
       r.res = "nimCopy(null, $1, $2)" % [x.rdLoc, genTypeInfo(p, n.typ)]
-  of mOpenArrayToSeq:
-    genCall(p, n, r)
-  of mDestroy, mTrace: discard "ignore calls to the default destructor"
+  of mDestroy: discard "ignore calls to the default destructor"
   of mOrd: genOrd(p, n, r)
   of mLengthStr, mLengthSeq, mLengthOpenArray, mLengthArray:
     var x: TCompRes
     gen(p, n[1], x)
-    if skipTypes(n[1].typ, abstractInst).kind == tyCstring:
+    if skipTypes(n[1].typ, abstractInst).kind == tyCString:
       let (a, tmp) = maybeMakeTemp(p, n[1], x)
       r.res = "(($1) == null ? 0 : ($2).length)" % [a, tmp]
     else:
@@ -2118,7 +2070,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
   of mHigh:
     var x: TCompRes
     gen(p, n[1], x)
-    if skipTypes(n[1].typ, abstractInst).kind == tyCstring:
+    if skipTypes(n[1].typ, abstractInst).kind == tyCString:
       let (a, tmp) = maybeMakeTemp(p, n[1], x)
       r.res = "(($1) == null ? -1 : ($2).length - 1)" % [a, tmp]
     else:
@@ -2145,7 +2097,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
     let t = skipTypes(n[1].typ, abstractVar)[0]
     let (a, tmp) = maybeMakeTemp(p, n[1], x)
     let (b, tmp2) = maybeMakeTemp(p, n[2], y)
-    r.res = """if ($1.length < $2) { for (var i = $4.length ; i < $5 ; ++i) $4.push($3); }
+    r.res = """if ($1.length < $2) { for (var i=$4.length;i<$5;++i) $4.push($3); }
                else { $4.length = $5; }""" % [a, b, createVar(p, t, false), tmp, tmp2]
     r.kind = resExpr
   of mCard: unaryExpr(p, n, r, "SetCard", "SetCard($1)")
@@ -2182,7 +2134,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
     gen(p, n[1], x)
     gen(p, n[2], y)
     gen(p, n[3], z)
-    r.res = "($1.slice($2, $3 + 1))" % [x.rdLoc, y.rdLoc, z.rdLoc]
+    r.res = "($1.slice($2, $3+1))" % [x.rdLoc, y.rdLoc, z.rdLoc]
     r.kind = resExpr
   of mMove:
     genMove(p, n, r)
@@ -2202,11 +2154,7 @@ proc genSetConstr(p: PProc, n: PNode, r: var TCompRes) =
     if it.kind == nkRange:
       gen(p, it[0], a)
       gen(p, it[1], b)
-
-      if it[0].typ.kind == tyBool:
-        r.res.addf("$1, $2", [a.res, b.res])
-      else:
-        r.res.addf("[$1, $2]", [a.res, b.res])
+      r.res.addf("[$1, $2]", [a.res, b.res])
     else:
       gen(p, it, a)
       r.res.add(a.res)
@@ -2219,26 +2167,21 @@ proc genSetConstr(p: PProc, n: PNode, r: var TCompRes) =
     r.res = tmp
 
 proc genArrayConstr(p: PProc, n: PNode, r: var TCompRes) =
-  ## Constructs array or sequence.
-  ## Nim array of uint8..uint32, int8..int32 maps to JS typed arrays.
-  ## Nim sequence maps to JS array.
-  var t = skipTypes(n.typ, abstractInst)
-  let e = elemType(t)
-  let jsTyp = arrayTypeForElemType(e)
-  if skipTypes(n.typ, abstractVarRange).kind != tySequence and jsTyp.len > 0:
-    # generate typed array
-    # for example Nim generates `new Uint8Array([1, 2, 3])` for `[byte(1), 2, 3]`
-    # TODO use `set` or loop to initialize typed array which improves performances in some situations
-    var a: TCompRes
-    r.res = "new $1([" % [rope(jsTyp)]
-    r.kind = resExpr
-    for i in 0 ..< n.len:
-      if i > 0: r.res.add(", ")
-      gen(p, n[i], a)
+  var a: TCompRes
+  r.res = rope("[")
+  r.kind = resExpr
+  for i in 0..<n.len:
+    if i > 0: r.res.add(", ")
+    gen(p, n[i], a)
+    if a.typ == etyBaseIndex:
+      r.res.addf("[$1, $2]", [a.address, a.res])
+    else:
+      if not needsNoCopy(p, n[i]):
+        let typ = n[i].typ.skipTypes(abstractInst)
+        useMagic(p, "nimCopy")
+        a.res = "nimCopy(null, $1, $2)" % [a.rdLoc, genTypeInfo(p, typ)]
       r.res.add(a.res)
-    r.res.add("])")
-  else:
-    genJSArrayConstr(p, n, r)
+  r.res.add("]")
 
 proc genTupleConstr(p: PProc, n: PNode, r: var TCompRes) =
   var a: TCompRes
@@ -2272,7 +2215,7 @@ proc genObjConstr(p: PProc, n: PNode, r: var TCompRes) =
     gen(p, val, a)
     var f = it[0].sym
     if f.loc.r == nil: f.loc.r = mangleName(p.module, f)
-    fieldIDs.incl(lookupFieldAgain(n.typ, f).id)
+    fieldIDs.incl(f.id)
 
     let typ = val.typ.skipTypes(abstractInst)
     if a.typ == etyBaseIndex:
@@ -2304,7 +2247,7 @@ proc genConv(p: PProc, n: PNode, r: var TCompRes) =
     r.res = "(!!($1))" % [r.res]
     r.kind = resExpr
   elif toInt:
-    r.res = "(($1) | 0)" % [r.res]
+    r.res = "(($1)|0)" % [r.res]
   else:
     # TODO: What types must we handle here?
     discard
@@ -2360,7 +2303,7 @@ proc genReturnStmt(p: PProc, n: PNode) =
 
 proc frameCreate(p: PProc; procname, filename: Rope): Rope =
   const frameFmt =
-    "var F = {procname: $1, prev: framePtr, filename: $2, line: 0};$n"
+    "var F={procname:$1,prev:framePtr,filename:$2,line:0};$n"
 
   result = p.indentLine(frameFmt % [procname, filename])
   result.add p.indentLine(ropes.`%`("framePtr = F;$n", []))
@@ -2408,11 +2351,9 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
   if prc.typ[0] != nil and sfPure notin prc.flags:
     resultSym = prc.ast[resultPos].sym
     let mname = mangleName(p.module, resultSym)
-    # otherwise uses "fat pointers"
-    let useRawPointer = not isIndirect(resultSym) and
+    if not isIndirect(resultSym) and
       resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef, tyOwned} and
-        mapType(p, resultSym.typ) == etyBaseIndex
-    if useRawPointer:
+        mapType(p, resultSym.typ) == etyBaseIndex:
       resultAsgn = p.indentLine(("var $# = null;$n") % [mname])
       resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname])
     else:
@@ -2424,9 +2365,9 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
     else:
       returnStmt = "return $#;$n" % [a.res]
 
-  var transformedBody = transformBody(p.module.graph, p.module.idgen, prc, cache = false)
+  var transformedBody = transformBody(oldProc.module.graph, prc, cache = false)
   if sfInjectDestructors in prc.flags:
-    transformedBody = injectDestructorCalls(p.module.graph, p.module.idgen, prc, transformedBody)
+    transformedBody = injectDestructorCalls(oldProc.module.graph, prc, transformedBody)
 
   p.nested: genStmt(p, transformedBody)
 
@@ -2456,10 +2397,10 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
       # references will end up calling the reloaded code.
       var thunkName = name
       name = name & "IMLP"
-      result.add("\Lfunction $#() { return $#.apply(this, arguments); }$n" %
+      result.add("function $#() { return $#.apply(this, arguments); }$n" %
                  [thunkName, name])
 
-    def = "\Lfunction $#($#) {$n$#$#$#$#$#" %
+    def = "function $#($#) {$n$#$#$#$#$#" %
             [ name,
               header,
               optionalLine(p.globals),
@@ -2507,7 +2448,7 @@ proc genCast(p: PProc, n: PNode, r: var TCompRes) =
     elif fromUint:
       if src.size == 4 and dest.size == 4:
         # XXX prevent multi evaluations
-        r.res = "($1 | 0)" % [r.res]
+        r.res = "($1|0)" % [r.res]
       else:
         let trimmer = unsignedTrimmer(dest.size)
         let minuend = case dest.size
@@ -2564,10 +2505,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     let f = n.floatVal
     case classify(f)
     of fcNan:
-      if signbit(f):
-        r.res = rope"-NaN"
-      else:
-        r.res = rope"NaN"
+      r.res = rope"NaN"
     of fcNegZero:
       r.res = rope"-0.0"
     of fcZero:
@@ -2595,15 +2533,8 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkObjConstr: genObjConstr(p, n, r)
   of nkHiddenStdConv, nkHiddenSubConv, nkConv: genConv(p, n, r)
   of nkAddr, nkHiddenAddr:
-    if n.typ.kind in {tyLent}:
-      gen(p, n[0], r)
-    else:
-      genAddr(p, n, r)
-  of nkDerefExpr, nkHiddenDeref:
-    if n.typ.kind in {tyLent}:
-      gen(p, n[0], r)
-    else:
-      genDeref(p, n, r)
+    genAddr(p, n, r)
+  of nkDerefExpr, nkHiddenDeref: genDeref(p, n, r)
   of nkBracketExpr: genArrayAccess(p, n, r)
   of nkDotExpr: genFieldAccess(p, n, r)
   of nkCheckedFieldExpr: genCheckedFieldOp(p, n, nil, r)
@@ -2620,7 +2551,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     let s = n[namePos].sym
     discard mangleName(p.module, s)
     r.res = s.loc.r
-    if lfNoDecl in s.loc.flags or s.magic notin generatedMagics: discard
+    if lfNoDecl in s.loc.flags or s.magic notin {mNone, mIsolate}: discard
     elif not p.g.generatedSyms.containsOrIncl(s.id):
       p.locals.add(genProc(p, s))
   of nkType: r.res = genTypeInfo(p, n.typ)
@@ -2721,13 +2652,13 @@ proc genModule(p: PProc, n: PNode) =
     p.body.add(frameCreate(p,
         makeJSString("module " & p.module.module.name.s),
         makeJSString(toFilenameOption(p.config, p.module.module.info.fileIndex, foStacktrace))))
-  var transformedN = transformStmt(p.module.graph, p.module.idgen, p.module.module, n)
+  var transformedN = transformStmt(p.module.graph, p.module.module, n)
   if sfInjectDestructors in p.module.module.flags:
-    transformedN = injectDestructorCalls(p.module.graph, p.module.idgen, p.module.module, transformedN)
+    transformedN = injectDestructorCalls(p.module.graph, p.module.module, transformedN)
   if p.config.hcrOn and n.kind == nkStmtList:
     let moduleSym = p.module.module
     var moduleLoadedVar = rope(moduleSym.name.s) & "_loaded" &
-                          idOrSig(moduleSym, moduleSym.name.s, p.module.sigConflicts, p.config)
+                          idOrSig(moduleSym, moduleSym.name.s, p.module.sigConflicts)
     lineF(p, "var $1;$n", [moduleLoadedVar])
     var inGuardedBlock = false
 
@@ -2800,9 +2731,7 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
       writeFile(outFile.string & ".map", $(%map))
     discard writeRopeIfNotEqual(code, outFile)
 
-
-proc myOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext =
+proc myOpen(graph: ModuleGraph; s: PSym): PPassContext =
   result = newModule(graph, s)
-  result.idgen = idgen
 
 const JSgenPass* = makePass(myOpen, myProcess, myClose)

compiler/jstypes.nim

@@ -128,21 +128,21 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
   case t.kind
   of tyDistinct:
     result = genTypeInfo(p, t[0])
-  of tyPointer, tyProc, tyBool, tyChar, tyCstring, tyString, tyInt..tyUInt64:
+  of tyPointer, tyProc, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
       [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
   of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet:
     var s =
-      "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" %
+      "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
     p.g.typeInfo.addf("$1.base = $2;$n",
          [result, genTypeInfo(p, t.lastSon)])
   of tyArray:
     var s =
-      "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" %
+      "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
     p.g.typeInfo.addf("$1.base = $2;$n",

compiler/lambdalifting.nim

@@ -12,7 +12,7 @@
 import
   intsets, strutils, options, ast, astalgo, msgs,
   idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos,
-  transf, liftdestructors, typeallowed
+  transf, liftdestructors
 
 discard """
   The basic approach is that captured vars need to be put on the heap and
@@ -126,32 +126,32 @@ proc newCall(a: PSym, b: PNode): PNode =
   result.add newSymNode(a)
   result.add b
 
-proc createClosureIterStateType*(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PType =
+proc createClosureIterStateType*(g: ModuleGraph; iter: PSym): PType =
   var n = newNodeI(nkRange, iter.info)
   n.add newIntNode(nkIntLit, -1)
   n.add newIntNode(nkIntLit, 0)
-  result = newType(tyRange, nextTypeId(idgen), iter)
+  result = newType(tyRange, iter)
   result.n = n
   var intType = nilOrSysInt(g)
-  if intType.isNil: intType = newType(tyInt, nextTypeId(idgen), iter)
+  if intType.isNil: intType = newType(tyInt, iter)
   rawAddSon(result, intType)
 
-proc createStateField(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym =
-  result = newSym(skField, getIdent(g.cache, ":state"), nextSymId(idgen), iter, iter.info)
-  result.typ = createClosureIterStateType(g, iter, idgen)
+proc createStateField(g: ModuleGraph; iter: PSym): PSym =
+  result = newSym(skField, getIdent(g.cache, ":state"), iter, iter.info)
+  result.typ = createClosureIterStateType(g, iter)
 
-proc createEnvObj(g: ModuleGraph; idgen: IdGenerator; owner: PSym; info: TLineInfo): PType =
+proc createEnvObj(g: ModuleGraph; owner: PSym; info: TLineInfo): PType =
   # YYY meh, just add the state field for every closure for now, it's too
   # hard to figure out if it comes from a closure iterator:
-  result = createObj(g, idgen, owner, info, final=false)
-  rawAddField(result, createStateField(g, owner, idgen))
+  result = createObj(g, owner, info, final=false)
+  rawAddField(result, createStateField(g, owner))
 
-proc getClosureIterResult*(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym =
+proc getClosureIterResult*(g: ModuleGraph; iter: PSym): PSym =
   if resultPos < iter.ast.len:
     result = iter.ast[resultPos].sym
   else:
     # XXX a bit hacky:
-    result = newSym(skResult, getIdent(g.cache, ":result"), nextSymId(idgen), iter, iter.info, {})
+    result = newSym(skResult, getIdent(g.cache, ":result"), iter, iter.info, {})
     result.typ = iter.typ[0]
     incl(result.flags, sfUsed)
     iter.ast.add newSymNode(result)
@@ -191,7 +191,9 @@ proc interestingVar(s: PSym): bool {.inline.} =
     s.typ.kind notin {tyStatic, tyTypeDesc}
 
 proc illegalCapture(s: PSym): bool {.inline.} =
-  result = classifyViewType(s.typ) != noView or s.kind == skResult
+  result = skipTypes(s.typ, abstractInst).kind in
+                   {tyVar, tyOpenArray, tyVarargs, tyLent} or
+      s.kind == skResult
 
 proc isInnerProc(s: PSym): bool =
   if s.kind in {skProc, skFunc, skMethod, skConverter, skIterator} and s.magic == mNone:
@@ -207,7 +209,7 @@ proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode =
   result[0] = le
   result[1] = ri
 
-proc makeClosure*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; env: PNode; info: TLineInfo): PNode =
+proc makeClosure*(g: ModuleGraph; prc: PSym; env: PNode; info: TLineInfo): PNode =
   result = newNodeIT(nkClosure, info, prc.typ)
   result.add(newSymNode(prc))
   if env == nil:
@@ -217,7 +219,7 @@ proc makeClosure*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; env: PNode; inf
       localError(g.config, info, "internal error: taking closure of closure")
     result.add(env)
   #if isClosureIterator(result.typ):
-  createTypeBoundOps(g, nil, result.typ, info, idgen)
+  createTypeBoundOps(g, nil, result.typ, info)
   if tfHasAsgn in result.typ.flags or optSeqDestructors in g.config.globalOptions:
     prc.flags.incl sfInjectDestructors
 
@@ -235,14 +237,14 @@ proc liftingHarmful(conf: ConfigRef; owner: PSym): bool {.inline.} =
   let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
   result = conf.backend == backendJs and not isCompileTime
 
-proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; idgen: IdGenerator; owner: PSym) =
+proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; owner: PSym) =
   if owner.kind != skMacro:
-    createTypeBoundOps(g, nil, refType.lastSon, info, idgen)
-    createTypeBoundOps(g, nil, refType, info, idgen)
+    createTypeBoundOps(g, nil, refType.lastSon, info)
+    createTypeBoundOps(g, nil, refType, info)
     if tfHasAsgn in refType.flags or optSeqDestructors in g.config.globalOptions:
       owner.flags.incl sfInjectDestructors
 
-proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNode =
+proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env))
   if liftingHarmful(g.config, owner): return n
   let iter = n.sym
@@ -254,10 +256,10 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PN
   var env: PNode
   if owner.isIterator:
     let it = getHiddenParam(g, owner)
-    addUniqueField(it.typ.skipTypes({tyOwned})[0], hp, g.cache, idgen)
+    addUniqueField(it.typ.skipTypes({tyOwned})[0], hp, g.cache)
     env = indirectAccess(newSymNode(it), hp, hp.info)
   else:
-    let e = newSym(skLet, iter.name, nextSymId(idgen), owner, n.info)
+    let e = newSym(skLet, iter.name, owner, n.info)
     e.typ = hp.typ
     e.flags = hp.flags
     env = newSymNode(e)
@@ -266,17 +268,22 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PN
     result.add(v)
   # add 'new' statement:
   result.add newCall(getSysSym(g, n.info, "internalNew"), env)
-  createTypeBoundOpsLL(g, env.typ, n.info, idgen, owner)
-  result.add makeClosure(g, idgen, iter, env, n.info)
+  createTypeBoundOpsLL(g, env.typ, n.info, owner)
+  result.add makeClosure(g, iter, env, n.info)
 
-proc freshVarForClosureIter*(g: ModuleGraph; s: PSym; idgen: IdGenerator; owner: PSym): PNode =
+proc freshVarForClosureIter*(g: ModuleGraph; s, owner: PSym): PNode =
   let envParam = getHiddenParam(g, owner)
   let obj = envParam.typ.skipTypes({tyOwned, tyRef, tyPtr})
-  let field = addField(obj, s, g.cache, idgen)
+  addField(obj, s, g.cache)
 
   var access = newSymNode(envParam)
   assert obj.kind == tyObject
-  result = rawIndirectAccess(access, field, s.info)
+  let field = getFieldFromObj(obj, s)
+  if field != nil:
+    result = rawIndirectAccess(access, field, s.info)
+  else:
+    localError(g.config, s.info, "internal error: cannot generate fresh variable")
+    result = access
 
 # ------------------ new stuff -------------------------------------------
 
@@ -285,7 +292,7 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
   if illegalCapture(s):
     localError(g.config, n.info,
       ("'$1' is of type <$2> which cannot be captured as it would violate memory" &
-       " safety, declared here: $3; using '-d:nimNoLentIterators' helps in some cases") %
+       " safety, declared here: $3; using '-d:nimWorkaround14447' helps in some cases") %
       [s.name.s, typeToString(s.typ), g.config$s.info])
   elif not (owner.typ.callConv == ccClosure or owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags):
     localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
@@ -298,17 +305,14 @@ type
     processed, capturedVars: IntSet
     ownerToType: Table[int, PType]
     somethingToDo: bool
-    inTypeOf: bool
     graph: ModuleGraph
-    idgen: IdGenerator
 
-proc initDetectionPass(g: ModuleGraph; fn: PSym; idgen: IdGenerator): DetectionPass =
+proc initDetectionPass(g: ModuleGraph; fn: PSym): DetectionPass =
   result.processed = initIntSet()
   result.capturedVars = initIntSet()
   result.ownerToType = initTable[int, PType]()
   result.processed.incl(fn.id)
   result.graph = g
-  result.idgen = idgen
 
 discard """
 proc outer =
@@ -324,15 +328,15 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
                         info: TLineInfo): PType =
   result = c.ownerToType.getOrDefault(owner.id)
   if result.isNil:
-    result = newType(tyRef, nextTypeId(c.idgen), owner)
-    let obj = createEnvObj(c.graph, c.idgen, owner, info)
+    result = newType(tyRef, owner)
+    let obj = createEnvObj(c.graph, owner, info)
     rawAddSon(result, obj)
     c.ownerToType[owner.id] = result
 
-proc asOwnedRef(c: var DetectionPass; t: PType): PType =
+proc asOwnedRef(c: DetectionPass; t: PType): PType =
   if optOwnedRefs in c.graph.config.globalOptions:
     assert t.kind == tyRef
-    result = newType(tyOwned, nextTypeId(c.idgen), t.owner)
+    result = newType(tyOwned, t.owner)
     result.flags.incl tfHasOwned
     result.rawAddSon t
   else:
@@ -341,7 +345,7 @@ proc asOwnedRef(c: var DetectionPass; t: PType): PType =
 proc getEnvTypeForOwnerUp(c: var DetectionPass; owner: PSym;
                           info: TLineInfo): PType =
   var r = c.getEnvTypeForOwner(owner, info)
-  result = newType(tyPtr, nextTypeId(c.idgen), owner)
+  result = newType(tyPtr, owner)
   rawAddSon(result, r.skipTypes({tyOwned, tyRef, tyPtr}))
 
 proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
@@ -369,7 +373,7 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
       if c.graph.config.selectedGC == gcDestructors and sfCursor notin upField.flags:
         localError(c.graph.config, dep.info, "internal error: up reference is not a .cursor")
   else:
-    let result = newSym(skField, upIdent, nextSymId(c.idgen), obj.owner, obj.owner.info)
+    let result = newSym(skField, upIdent, obj.owner, obj.owner.info)
     result.typ = fieldType
     when false:
       if c.graph.config.selectedGC == gcDestructors:
@@ -402,15 +406,12 @@ Consider:
 
 """
 
-proc isTypeOf(n: PNode): bool =
-  n.kind == nkSym and n.sym.magic in {mTypeOf, mType}
-
 proc addClosureParam(c: var DetectionPass; fn: PSym; info: TLineInfo) =
   var cp = getEnvParam(fn)
   let owner = if fn.kind == skIterator: fn else: fn.skipGenericOwner
   let t = c.getEnvTypeForOwner(owner, info)
   if cp == nil:
-    cp = newSym(skParam, getIdent(c.graph.cache, paramName), nextSymId(c.idgen), fn, fn.info)
+    cp = newSym(skParam, getIdent(c.graph.cache, paramName), fn, fn.info)
     incl(cp.flags, sfFromGeneric)
     cp.typ = t
     addHiddenParam(fn, cp)
@@ -433,7 +434,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     if innerProc:
       if s.isIterator: c.somethingToDo = true
       if not c.processed.containsOrIncl(s.id):
-        let body = transformBody(c.graph, c.idgen, s, cache = true)
+        let body = transformBody(c.graph, s, cache = true)
         detectCapturedVars(body, s, c)
     let ow = s.skipGenericOwner
     if ow == owner:
@@ -441,17 +442,16 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
         c.somethingToDo = true
         addClosureParam(c, owner, n.info)
         if interestingIterVar(s):
-          if not c.capturedVars.contains(s.id):
-            if not c.inTypeOf: c.capturedVars.incl(s.id)
+          if not c.capturedVars.containsOrIncl(s.id):
             let obj = getHiddenParam(c.graph, owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
             #let obj = c.getEnvTypeForOwner(s.owner).skipTypes({tyOwned, tyRef, tyPtr})
 
             if s.name.id == getIdent(c.graph.cache, ":state").id:
-              obj.n[0].sym.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
+              obj.n[0].sym.id = -s.id
             else:
-              discard addField(obj, s, c.graph.cache, c.idgen)
+              addField(obj, s, c.graph.cache)
     # direct or indirect dependency:
-    elif (innerProc and not s.isIterator and s.typ.callConv == ccClosure) or interestingVar(s):
+    elif (innerProc and s.typ.callConv == ccClosure) or interestingVar(s):
       discard """
         proc outer() =
           var x: int
@@ -468,12 +468,10 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       addClosureParam(c, owner, n.info)
       #echo "capturing ", n.info
       # variable 's' is actually captured:
-      if interestingVar(s):
-        if not c.capturedVars.contains(s.id):
-          if not c.inTypeOf: c.capturedVars.incl(s.id)
-          let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef, tyPtr})
-          #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
-          discard addField(obj, s, c.graph.cache, c.idgen)
+      if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id):
+        let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef, tyPtr})
+        #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
+        addField(obj, s, c.graph.cache)
       # create required upFields:
       var w = owner.skipGenericOwner
       if isInnerProc(w) or owner.isIterator:
@@ -505,14 +503,9 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       detectCapturedVars(n[namePos], owner, c)
   of nkReturnStmt:
     detectCapturedVars(n[0], owner, c)
-  of nkIdentDefs:
-    detectCapturedVars(n[^1], owner, c)
   else:
-    if n.isCallExpr and n[0].isTypeOf:
-      c.inTypeOf = true
     for i in 0..<n.len:
       detectCapturedVars(n[i], owner, c)
-    c.inTypeOf = false
 
 type
   LiftingPass = object
@@ -544,8 +537,8 @@ proc accessViaEnvParam(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   localError(g.config, n.info, "internal error: environment misses: " & s.name.s)
   result = n
 
-proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo; idgen: IdGenerator): PNode =
-  var v = newSym(skVar, getIdent(cache, envName), nextSymId(idgen), owner, info)
+proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo): PNode =
+  var v = newSym(skVar, getIdent(cache, envName), owner, info)
   v.flags = {sfShadowed, sfGeneratedOp}
   v.typ = typ
   result = newSymNode(v)
@@ -557,7 +550,7 @@ proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo; idge
     else:
       result = newSymNode(v)
 
-proc setupEnvVar(owner: PSym; d: var DetectionPass;
+proc setupEnvVar(owner: PSym; d: DetectionPass;
                  c: var LiftingPass; info: TLineInfo): PNode =
   if owner.isIterator:
     return getHiddenParam(d.graph, owner).newSymNode
@@ -566,10 +559,10 @@ proc setupEnvVar(owner: PSym; d: var DetectionPass;
     let envVarType = d.ownerToType.getOrDefault(owner.id)
     if envVarType.isNil:
       localError d.graph.config, owner.info, "internal error: could not determine closure type"
-    result = newEnvVar(d.graph.cache, owner, asOwnedRef(d, envVarType), info, d.idgen)
+    result = newEnvVar(d.graph.cache, owner, asOwnedRef(d, envVarType), info)
     c.envVars[owner.id] = result
     if optOwnedRefs in d.graph.config.globalOptions:
-      var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), nextSymId d.idgen, owner, info)
+      var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), owner, info)
       v.flags = {sfShadowed, sfGeneratedOp}
       v.typ = envVarType
       c.unownedEnvVars[owner.id] = newSymNode(v)
@@ -585,7 +578,7 @@ proc getUpViaParam(g: ModuleGraph; owner: PSym): PNode =
       result = rawIndirectAccess(result, upField, p.info)
 
 proc rawClosureCreation(owner: PSym;
-                        d: var DetectionPass; c: var LiftingPass;
+                        d: DetectionPass; c: var LiftingPass;
                         info: TLineInfo): PNode =
   result = newNodeI(nkStmtList, owner.info)
 
@@ -611,7 +604,7 @@ proc rawClosureCreation(owner: PSym;
       let env2 = copyTree(env)
       env2.typ = unowned.typ
       result.add newAsgnStmt(unowned, env2, env.info)
-      createTypeBoundOpsLL(d.graph, unowned.typ, env.info, d.idgen, owner)
+      createTypeBoundOpsLL(d.graph, unowned.typ, env.info, owner)
 
     # add assignment statements for captured parameters:
     for i in 1..<owner.typ.n.len:
@@ -621,7 +614,7 @@ proc rawClosureCreation(owner: PSym;
         # add ``env.param = param``
         result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info))
         if owner.kind != skMacro:
-          createTypeBoundOps(d.graph, nil, fieldAccess.typ, env.info, d.idgen)
+          createTypeBoundOps(d.graph, nil, fieldAccess.typ, env.info)
         if tfHasAsgn in fieldAccess.typ.flags or optSeqDestructors in d.graph.config.globalOptions:
           owner.flags.incl sfInjectDestructors
 
@@ -638,28 +631,28 @@ proc rawClosureCreation(owner: PSym;
       localError(d.graph.config, env.info, "internal error: cannot create up reference")
   # we are not in the sem'check phase anymore! so pass 'nil' for the PContext
   # and hope for the best:
-  createTypeBoundOpsLL(d.graph, env.typ, owner.info, d.idgen, owner)
+  createTypeBoundOpsLL(d.graph, env.typ, owner.info, owner)
 
-proc finishClosureCreation(owner: PSym; d: var DetectionPass; c: LiftingPass;
+proc finishClosureCreation(owner: PSym; d: DetectionPass; c: LiftingPass;
                            info: TLineInfo; res: PNode) =
   if optOwnedRefs in d.graph.config.globalOptions:
     let unowned = c.unownedEnvVars[owner.id]
     assert unowned != nil
     let nilLit = newNodeIT(nkNilLit, info, unowned.typ)
     res.add newAsgnStmt(unowned, nilLit, info)
-    createTypeBoundOpsLL(d.graph, unowned.typ, info, d.idgen, owner)
+    createTypeBoundOpsLL(d.graph, unowned.typ, info, owner)
 
 proc closureCreationForIter(iter: PNode;
-                            d: var DetectionPass; c: var LiftingPass): PNode =
+                            d: DetectionPass; c: var LiftingPass): PNode =
   result = newNodeIT(nkStmtListExpr, iter.info, iter.sym.typ)
   let owner = iter.sym.skipGenericOwner
-  var v = newSym(skVar, getIdent(d.graph.cache, envName), nextSymId(d.idgen), owner, iter.info)
+  var v = newSym(skVar, getIdent(d.graph.cache, envName), owner, iter.info)
   incl(v.flags, sfShadowed)
   v.typ = asOwnedRef(d, getHiddenParam(d.graph, iter.sym).typ)
   var vnode: PNode
   if owner.isIterator:
     let it = getHiddenParam(d.graph, owner)
-    addUniqueField(it.typ.skipTypes({tyOwned, tyRef, tyPtr}), v, d.graph.cache, d.idgen)
+    addUniqueField(it.typ.skipTypes({tyOwned, tyRef, tyPtr}), v, d.graph.cache)
     vnode = indirectAccess(newSymNode(it), v, v.info)
   else:
     vnode = v.newSymNode
@@ -667,7 +660,7 @@ proc closureCreationForIter(iter: PNode;
     addVar(vs, vnode)
     result.add(vs)
   result.add(newCall(getSysSym(d.graph, iter.info, "internalNew"), vnode))
-  createTypeBoundOpsLL(d.graph, vnode.typ, iter.info, d.idgen, owner)
+  createTypeBoundOpsLL(d.graph, vnode.typ, iter.info, owner)
 
   let upField = lookupInRecord(v.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
@@ -677,9 +670,9 @@ proc closureCreationForIter(iter: PNode;
                  u, iter.info))
     else:
       localError(d.graph.config, iter.info, "internal error: cannot create up reference for iter")
-  result.add makeClosure(d.graph, d.idgen, iter.sym, vnode, iter.info)
+  result.add makeClosure(d.graph, iter.sym, vnode, iter.info)
 
-proc accessViaEnvVar(n: PNode; owner: PSym; d: var DetectionPass;
+proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
                      c: var LiftingPass): PNode =
   var access = setupEnvVar(owner, d, c, n.info)
   if optOwnedRefs in d.graph.config.globalOptions:
@@ -695,21 +688,21 @@ proc accessViaEnvVar(n: PNode; owner: PSym; d: var DetectionPass;
 proc getStateField*(g: ModuleGraph; owner: PSym): PSym =
   getHiddenParam(g, owner).typ.skipTypes({tyOwned, tyRef, tyPtr}).n[0].sym
 
-proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
+proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
                       c: var LiftingPass): PNode
 
-proc symToClosure(n: PNode; owner: PSym; d: var DetectionPass;
+proc symToClosure(n: PNode; owner: PSym; d: DetectionPass;
                   c: var LiftingPass): PNode =
   let s = n.sym
   if s == owner:
     # recursive calls go through (lambda, hiddenParam):
     let available = getHiddenParam(d.graph, owner)
-    result = makeClosure(d.graph, d.idgen, s, available.newSymNode, n.info)
+    result = makeClosure(d.graph, s, available.newSymNode, n.info)
   elif s.isIterator:
     result = closureCreationForIter(n, d, c)
   elif s.skipGenericOwner == owner:
     # direct dependency, so use the outer's env variable:
-    result = makeClosure(d.graph, d.idgen, s, setupEnvVar(owner, d, c, n.info), n.info)
+    result = makeClosure(d.graph, s, setupEnvVar(owner, d, c, n.info), n.info)
   else:
     let available = getHiddenParam(d.graph, owner)
     let wanted = getHiddenParam(d.graph, s).typ
@@ -717,7 +710,7 @@ proc symToClosure(n: PNode; owner: PSym; d: var DetectionPass;
     var access = newSymNode(available)
     while true:
       if access.typ == wanted:
-        return makeClosure(d.graph, d.idgen, s, access, n.info)
+        return makeClosure(d.graph, s, access, n.info)
       let obj = access.typ.skipTypes({tyOwned, tyRef, tyPtr})
       let upField = lookupInRecord(obj.n, getIdent(d.graph.cache, upName))
       if upField == nil:
@@ -725,7 +718,7 @@ proc symToClosure(n: PNode; owner: PSym; d: var DetectionPass;
         return n
       access = rawIndirectAccess(access, upField, n.info)
 
-proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
+proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
                       c: var LiftingPass): PNode =
   result = n
   case n.kind
@@ -737,7 +730,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
         #  echo renderTree(s.getBody, {renderIds})
         let oldInContainer = c.inContainer
         c.inContainer = 0
-        var body = transformBody(d.graph, d.idgen, s, cache = false)
+        var body = transformBody(d.graph, s, cache = false)
         body = liftCapturedVars(body, s, d, c)
         if c.envVars.getOrDefault(s.id).isNil:
           s.transformedBody = body
@@ -792,8 +785,6 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
   of nkTypeOfExpr:
     result = n
   else:
-    if n.isCallExpr and n[0].isTypeOf:
-      return
     if owner.isIterator:
       if nfLL in n.flags:
         # special case 'when nimVm' due to bug #3636:
@@ -845,9 +836,8 @@ proc semCaptureSym*(s, owner: PSym) =
     # since the analysis is not entirely correct, we don't set 'tfCapturesEnv'
     # here
 
-proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType;
-                     idgen: IdGenerator): PNode =
-  var d = initDetectionPass(g, fn, idgen)
+proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType): PNode =
+  var d = initDetectionPass(g, fn)
   var c = initLiftingPass(fn)
   # pretend 'fn' is a closure iterator for the analysis:
   let oldKind = fn.kind
@@ -860,8 +850,7 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType;
   fn.transitionRoutineSymKind(oldKind)
   fn.typ.callConv = oldCC
 
-proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
-                  idgen: IdGenerator): PNode =
+proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PNode =
   # XXX backend == backendJs does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
 
@@ -876,7 +865,7 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
     result = body
     tooEarly = true
   else:
-    var d = initDetectionPass(g, fn, idgen)
+    var d = initDetectionPass(g, fn)
     detectCapturedVars(body, fn, d)
     if not d.somethingToDo and fn.isIterator:
       addClosureParam(d, fn, body.info)
@@ -900,7 +889,7 @@ proc liftLambdasForTopLevel*(module: PSym, body: PNode): PNode =
 
 # ------------------- iterator transformation --------------------------------
 
-proc liftForLoop*(g: ModuleGraph; body: PNode; idgen: IdGenerator; owner: PSym): PNode =
+proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
   # problem ahead: the iterator could be invoked indirectly, but then
   # we don't know what environment to create here:
   #
@@ -945,7 +934,7 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; idgen: IdGenerator; owner: PSym):
     let iter = op.sym
 
     let hp = getHiddenParam(g, iter)
-    env = newSym(skLet, iter.name, nextSymId(idgen), owner, body.info)
+    env = newSym(skLet, iter.name, owner, body.info)
     env.typ = hp.typ
     env.flags = hp.flags
 
@@ -954,7 +943,7 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; idgen: IdGenerator; owner: PSym):
     result.add(v)
     # add 'new' statement:
     result.add(newCall(getSysSym(g, env.info, "internalNew"), env.newSymNode))
-    createTypeBoundOpsLL(g, env.typ, body.info, idgen, owner)
+    createTypeBoundOpsLL(g, env.typ, body.info, owner)
 
   elif op.kind == nkStmtListExpr:
     let closure = op.lastSon
@@ -980,7 +969,7 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; idgen: IdGenerator; owner: PSym):
 
   vpart.add newNodeI(nkEmpty, body.info) # no explicit type
   if not env.isNil:
-    call[0] = makeClosure(g, idgen, call[0].sym, env.newSymNode, body.info)
+    call[0] = makeClosure(g, call[0].sym, env.newSymNode, body.info)
   vpart.add call
   v2.add vpart
 

compiler/layouter.nim

@@ -514,8 +514,7 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
         (not em.wasExportMarker or tok.tokType == tkCurlyDotLe):
       wrSpace em
     wr(em, $tok.tokType, ltSomeParLe)
-    if tok.tokType != tkCurlyDotLe:
-      rememberSplit(splitParLe)
+    rememberSplit(splitParLe)
   of closedPars:
     wr(em, $tok.tokType, ltSomeParRi)
   of tkColonColon:
@@ -551,15 +550,11 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
     if not preventComment:
       emitComment(em, tok, dontIndent = false)
   of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, tkGTripleStrLit, tkCharLit:
-    if not em.inquote:
-      let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
-      if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
-      em.lineSpan = countNewlines(lit)
-      if em.lineSpan > 0: calcCol(em, lit)
-      wr em, lit, ltLit
-    else:
-      if endsInAlpha(em): wrSpace(em)
-      wr em, tok.literal, ltLit
+    let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
+    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
+    em.lineSpan = countNewlines(lit)
+    if em.lineSpan > 0: calcCol(em, lit)
+    wr em, lit, ltLit
   of tkEof: discard
   else:
     let lit = if tok.ident != nil: tok.ident.s else: tok.literal

compiler/lexer.nim

@@ -7,12 +7,12 @@
 #    distribution, for details about the copyright.
 #
 
-# This lexer is handwritten for efficiency. I used an elegant buffering
+# This scanner is handwritten for efficiency. I used an elegant buffering
 # scheme which I have not seen anywhere else:
 # We guarantee that a whole line is in the buffer. Thus only when scanning
-# the \n or \r character we have to check whether we need to read in the next
+# the \n or \r character we have to check wether we need to read in the next
 # chunk. (\n or \r already need special handling for incrementing the line
-# counter; choosing both \n and \r allows the lexer to properly read Unix,
+# counter; choosing both \n and \r allows the scanner to properly read Unix,
 # DOS or Macintosh text files, even when it is not the native format.
 
 import
@@ -26,7 +26,6 @@ const
   SymStartChars*: set[char] = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'}
   OpChars*: set[char] = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.',
     '|', '=', '%', '&', '$', '@', '~', ':'}
-  UnaryMinusWhitelist = {' ', '\t', '\n', '\r', ',', ';', '(', '[', '{'}
 
 # don't forget to update the 'highlite' module if these charsets should change
 
@@ -52,27 +51,26 @@ type
     tkVar = "var", tkWhen = "when", tkWhile = "while", tkXor = "xor",
     tkYield = "yield", # end of keywords
 
-    tkIntLit = "tkIntLit", tkInt8Lit = "tkInt8Lit", tkInt16Lit = "tkInt16Lit",
+    tkIntLit = "tkIntLit", tkInt8Lit = "tkInt8Lit", tkInt16Lit = "tkInt16Lit", 
     tkInt32Lit = "tkInt32Lit", tkInt64Lit = "tkInt64Lit",
-    tkUIntLit = "tkUIntLit", tkUInt8Lit = "tkUInt8Lit", tkUInt16Lit = "tkUInt16Lit",
+    tkUIntLit = "tkUIntLit", tkUInt8Lit = "tkUInt8Lit", tkUInt16Lit = "tkUInt16Lit", 
     tkUInt32Lit = "tkUInt32Lit", tkUInt64Lit = "tkUInt64Lit",
     tkFloatLit = "tkFloatLit", tkFloat32Lit = "tkFloat32Lit",
     tkFloat64Lit = "tkFloat64Lit", tkFloat128Lit = "tkFloat128Lit",
     tkStrLit = "tkStrLit", tkRStrLit = "tkRStrLit", tkTripleStrLit = "tkTripleStrLit",
-    tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit",
-    tkCustomLit = "tkCustomLit",
-
+    tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit", 
+    
     tkParLe = "(", tkParRi = ")", tkBracketLe = "[",
     tkBracketRi = "]", tkCurlyLe = "{", tkCurlyRi = "}",
     tkBracketDotLe = "[.", tkBracketDotRi = ".]",
     tkCurlyDotLe = "{.", tkCurlyDotRi = ".}",
     tkParDotLe = "(.", tkParDotRi = ".)",
     tkComma = ",", tkSemiColon = ";",
-    tkColon = ":", tkColonColon = "::", tkEquals = "=",
+    tkColon = ":", tkColonColon = "::", tkEquals = "=", 
     tkDot = ".", tkDotDot = "..", tkBracketLeColon = "[:",
     tkOpr, tkComment, tkAccent = "`",
     # these are fake tokens used by renderer.nim
-    tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr, tkHideableStart, tkHideableEnd
+    tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
 
   TokTypes* = set[TokType]
 
@@ -121,8 +119,6 @@ type
     cache*: IdentCache
     when defined(nimsuggest):
       previousToken: TLineInfo
-      tokenEnd*: TLineInfo
-      previousTokenEnd*: TLineInfo
     config*: ConfigRef
 
 proc getLineInfo*(L: Lexer, tok: Token): TLineInfo {.inline.} =
@@ -165,7 +161,6 @@ proc prettyTok*(tok: Token): string =
   else: $tok
 
 proc printTok*(conf: ConfigRef; tok: Token) =
-  # xxx factor with toLocation
   msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" & $tok.tokType & " " & $tok)
 
 proc initToken*(L: var Token) =
@@ -317,7 +312,8 @@ proc getNumber(L: var Lexer, result: var Token) =
 
   proc lexMessageLitNum(L: var Lexer, msg: string, startpos: int, msgKind = errGenerated) =
     # Used to get slightly human friendlier err messages.
-    const literalishChars = {'A'..'Z', 'a'..'z', '0'..'9', '_', '.', '\''}
+    const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O',
+      'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
     var msgPos = L.bufpos
     var t: Token
     t.literal = ""
@@ -329,14 +325,15 @@ proc getNumber(L: var Lexer, result: var Token) =
       t.literal.add(L.buf[L.bufpos])
       inc(L.bufpos)
       matchChars(L, t, literalishChars)
-    if L.buf[L.bufpos] in literalishChars:
-      t.literal.add(L.buf[L.bufpos])
+    if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
       inc(L.bufpos)
+      t.literal.add(L.buf[L.bufpos])
       matchChars(L, t, {'0'..'9'})
     L.bufpos = msgPos
     lexMessage(L, msgKind, msg % t.literal)
 
   var
+    startpos, endpos: int
     xi: BiggestInt
     isBase10 = true
     numDigits = 0
@@ -348,17 +345,8 @@ proc getNumber(L: var Lexer, result: var Token) =
   result.tokType = tkIntLit   # int literal until we know better
   result.literal = ""
   result.base = base10
-  tokenBegin(result, L.bufpos)
-
-  var isPositive = true
-  if L.buf[L.bufpos] == '-':
-    eatChar(L, result)
-    isPositive = false
-
-  let startpos = L.bufpos
-
-  template setNumber(field, value) =
-    field = (if isPositive: value else: -value)
+  startpos = L.bufpos
+  tokenBegin(result, startpos)
 
   # First stage: find out base, make verifications, build token literal string
   # {'c', 'C'} is added for deprecation reasons to provide a clear error message
@@ -398,184 +386,200 @@ proc getNumber(L: var Lexer, result: var Token) =
       discard matchUnderscoreChars(L, result, {'0'..'9'})
     if L.buf[L.bufpos] in {'e', 'E'}:
       result.tokType = tkFloatLit
-      eatChar(L, result)
+      eatChar(L, result, 'e')
       if L.buf[L.bufpos] in {'+', '-'}:
         eatChar(L, result)
       discard matchUnderscoreChars(L, result, {'0'..'9'})
-  let endpos = L.bufpos
+  endpos = L.bufpos
 
   # Second stage, find out if there's a datatype suffix and handle it
   var postPos = endpos
-
   if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
-    let errPos = postPos
-    var customLitPossible = false
     if L.buf[postPos] == '\'':
       inc(postPos)
-      customLitPossible = true
 
-    if L.buf[postPos] in SymChars:
-      var suffix = newStringOfCap(10)
-      while true:
-        suffix.add L.buf[postPos]
-        inc postPos
-        if L.buf[postPos] notin SymChars+{'_'}: break
-      let suffixAsLower = suffix.toLowerAscii
-      case suffixAsLower
-      of "f", "f32": result.tokType = tkFloat32Lit
-      of "d", "f64": result.tokType = tkFloat64Lit
-      of "f128": result.tokType = tkFloat128Lit
-      of "i8": result.tokType = tkInt8Lit
-      of "i16": result.tokType = tkInt16Lit
-      of "i32": result.tokType = tkInt32Lit
-      of "i64": result.tokType = tkInt64Lit
-      of "u": result.tokType = tkUIntLit
-      of "u8": result.tokType = tkUInt8Lit
-      of "u16": result.tokType = tkUInt16Lit
-      of "u32": result.tokType = tkUInt32Lit
-      of "u64": result.tokType = tkUInt64Lit
-      elif customLitPossible:
-        # remember the position of the `'` so that the parser doesn't
-        # have to reparse the custom literal:
-        result.iNumber = len(result.literal)
-        result.literal.add '\''
-        result.literal.add suffix
-        result.tokType = tkCustomLit
+    case L.buf[postPos]
+    of 'f', 'F':
+      inc(postPos)
+      if (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
+        result.tokType = tkFloat32Lit
+        inc(postPos, 2)
+      elif (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
+        result.tokType = tkFloat64Lit
+        inc(postPos, 2)
+      elif (L.buf[postPos] == '1') and
+           (L.buf[postPos + 1] == '2') and
+           (L.buf[postPos + 2] == '8'):
+        result.tokType = tkFloat128Lit
+        inc(postPos, 3)
+      else:   # "f" alone defaults to float32
+        result.tokType = tkFloat32Lit
+    of 'd', 'D':  # ad hoc convenience shortcut for f64
+      inc(postPos)
+      result.tokType = tkFloat64Lit
+    of 'i', 'I':
+      inc(postPos)
+      if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
+        result.tokType = tkInt64Lit
+        inc(postPos, 2)
+      elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
+        result.tokType = tkInt32Lit
+        inc(postPos, 2)
+      elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
+        result.tokType = tkInt16Lit
+        inc(postPos, 2)
+      elif (L.buf[postPos] == '8'):
+        result.tokType = tkInt8Lit
+        inc(postPos)
       else:
-        lexMessageLitNum(L, "invalid number suffix: '$1'", errPos)
+        lexMessageLitNum(L, "invalid number: '$1'", startpos)
+    of 'u', 'U':
+      inc(postPos)
+      if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
+        result.tokType = tkUInt64Lit
+        inc(postPos, 2)
+      elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
+        result.tokType = tkUInt32Lit
+        inc(postPos, 2)
+      elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
+        result.tokType = tkUInt16Lit
+        inc(postPos, 2)
+      elif (L.buf[postPos] == '8'):
+        result.tokType = tkUInt8Lit
+        inc(postPos)
+      else:
+        result.tokType = tkUIntLit
     else:
-      lexMessageLitNum(L, "invalid number suffix: '$1'", errPos)
+      lexMessageLitNum(L, "invalid number: '$1'", startpos)
 
   # Is there still a literalish char awaiting? Then it's an error!
   if  L.buf[postPos] in literalishChars or
      (L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}):
     lexMessageLitNum(L, "invalid number: '$1'", startpos)
 
-  if result.tokType != tkCustomLit:
-    # Third stage, extract actual number
-    L.bufpos = startpos            # restore position
-    var pos = startpos
-    try:
-      if (L.buf[pos] == '0') and (L.buf[pos + 1] in baseCodeChars):
-        inc(pos, 2)
-        xi = 0                  # it is a base prefix
+  # Third stage, extract actual number
+  L.bufpos = startpos            # restore position
+  var pos: int = startpos
+  try:
+    if (L.buf[pos] == '0') and (L.buf[pos + 1] in baseCodeChars):
+      inc(pos, 2)
+      xi = 0                  # it is a base prefix
 
-        case L.buf[pos - 1]
-        of 'b', 'B':
-          result.base = base2
-          while pos < endpos:
-            if L.buf[pos] != '_':
-              xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
+      case L.buf[pos - 1]
+      of 'b', 'B':
+        result.base = base2
+        while pos < endpos:
+          if L.buf[pos] != '_':
+            xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
+          inc(pos)
+      # 'c', 'C' is deprecated
+      of 'o', 'c', 'C':
+        result.base = base8
+        while pos < endpos:
+          if L.buf[pos] != '_':
+            xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
+          inc(pos)
+      of 'x', 'X':
+        result.base = base16
+        while pos < endpos:
+          case L.buf[pos]
+          of '_':
             inc(pos)
-        # 'c', 'C' is deprecated (a warning is issued elsewhere)
-        of 'o', 'c', 'C':
-          result.base = base8
-          while pos < endpos:
-            if L.buf[pos] != '_':
-              xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
+          of '0'..'9':
+            xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
             inc(pos)
-        of 'x', 'X':
-          result.base = base16
-          while pos < endpos:
-            case L.buf[pos]
-            of '_':
-              inc(pos)
-            of '0'..'9':
-              xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
-              inc(pos)
-            of 'a'..'f':
-              xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10)
-              inc(pos)
-            of 'A'..'F':
-              xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
-              inc(pos)
-            else:
-              break
-        else:
-          internalError(L.config, getLineInfo(L), "getNumber")
-
-        case result.tokType
-        of tkIntLit, tkInt64Lit: setNumber result.iNumber, xi
-        of tkInt8Lit: setNumber result.iNumber, ashr(xi shl 56, 56)
-        of tkInt16Lit: setNumber result.iNumber, ashr(xi shl 48, 48)
-        of tkInt32Lit: setNumber result.iNumber, ashr(xi shl 32, 32)
-        of tkUIntLit, tkUInt64Lit: setNumber result.iNumber, xi
-        of tkUInt8Lit: setNumber result.iNumber, xi and 0xff
-        of tkUInt16Lit: setNumber result.iNumber, xi and 0xffff
-        of tkUInt32Lit: setNumber result.iNumber, xi and 0xffffffff
-        of tkFloat32Lit:
-          setNumber result.fNumber, (cast[PFloat32](addr(xi)))[]
-          # note: this code is endian neutral!
-          # XXX: Test this on big endian machine!
-        of tkFloat64Lit, tkFloatLit:
-          setNumber result.fNumber, (cast[PFloat64](addr(xi)))[]
-        else: internalError(L.config, getLineInfo(L), "getNumber")
-
-        # Bounds checks. Non decimal literals are allowed to overflow the range of
-        # the datatype as long as their pattern don't overflow _bitwise_, hence
-        # below checks of signed sizes against uint*.high is deliberate:
-        # (0x80'u8 = 128, 0x80'i8 = -128, etc == OK)
-        if result.tokType notin floatTypes:
-          let outOfRange =
-            case result.tokType
-            of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi
-            of tkInt8Lit:  (xi > BiggestInt(uint8.high))
-            of tkInt16Lit: (xi > BiggestInt(uint16.high))
-            of tkInt32Lit: (xi > BiggestInt(uint32.high))
-            else: false
-
-          if outOfRange:
-            #echo "out of range num: ", result.iNumber, " vs ", xi
-            lexMessageLitNum(L, "number out of range: '$1'", startpos)
-
+          of 'a'..'f':
+            xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10)
+            inc(pos)
+          of 'A'..'F':
+            xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
+            inc(pos)
+          else:
+            break
       else:
-        case result.tokType
-        of floatTypes:
-          result.fNumber = parseFloat(result.literal)
-        of tkUInt64Lit, tkUIntLit:
-          var iNumber: uint64
-          var len: int
-          try:
-            len = parseBiggestUInt(result.literal, iNumber)
-          except ValueError:
-            raise newException(OverflowDefect, "number out of range: " & result.literal)
-          if len != result.literal.len:
-            raise newException(ValueError, "invalid integer: " & result.literal)
-          result.iNumber = cast[int64](iNumber)
-        else:
-          var iNumber: int64
-          var len: int
-          try:
-            len = parseBiggestInt(result.literal, iNumber)
-          except ValueError:
-            raise newException(OverflowDefect, "number out of range: " & result.literal)
-          if len != result.literal.len:
-            raise newException(ValueError, "invalid integer: " & result.literal)
-          result.iNumber = iNumber
+        internalError(L.config, getLineInfo(L), "getNumber")
 
-        # Explicit bounds checks.
-        let outOfRange =
-          case result.tokType
-          of tkInt8Lit: result.iNumber > int8.high or result.iNumber < int8.low
-          of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high) or result.iNumber < 0
-          of tkInt16Lit: result.iNumber > int16.high or result.iNumber < int16.low
-          of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high) or result.iNumber < 0
-          of tkInt32Lit: result.iNumber > int32.high or result.iNumber < int32.low
-          of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high) or result.iNumber < 0
-          else: false
+      case result.tokType
+      of tkIntLit, tkInt64Lit: result.iNumber = xi
+      of tkInt8Lit: result.iNumber = ashr(xi shl 56, 56)
+      of tkInt16Lit: result.iNumber = ashr(xi shl 48, 48)
+      of tkInt32Lit: result.iNumber = ashr(xi shl 32, 32)
+      of tkUIntLit, tkUInt64Lit: result.iNumber = xi
+      of tkUInt8Lit: result.iNumber = xi and 0xff
+      of tkUInt16Lit: result.iNumber = xi and 0xffff
+      of tkUInt32Lit: result.iNumber = xi and 0xffffffff
+      of tkFloat32Lit:
+        result.fNumber = (cast[PFloat32](addr(xi)))[]
+        # note: this code is endian neutral!
+        # XXX: Test this on big endian machine!
+      of tkFloat64Lit, tkFloatLit:
+        result.fNumber = (cast[PFloat64](addr(xi)))[]
+      else: internalError(L.config, getLineInfo(L), "getNumber")
+
+      # Bounds checks. Non decimal literals are allowed to overflow the range of
+      # the datatype as long as their pattern don't overflow _bitwise_, hence
+      # below checks of signed sizes against uint*.high is deliberate:
+      # (0x80'u8 = 128, 0x80'i8 = -128, etc == OK)
+      if result.tokType notin floatTypes:
+        let outOfRange = case result.tokType:
+        of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi
+        of tkInt8Lit: (xi > BiggestInt(uint8.high))
+        of tkInt16Lit: (xi > BiggestInt(uint16.high))
+        of tkInt32Lit: (xi > BiggestInt(uint32.high))
+        else: false
 
         if outOfRange:
+          #echo "out of range num: ", result.iNumber, " vs ", xi
           lexMessageLitNum(L, "number out of range: '$1'", startpos)
 
-      # Promote int literal to int64? Not always necessary, but more consistent
-      if result.tokType == tkIntLit:
-        if result.iNumber > high(int32) or result.iNumber < low(int32):
-          result.tokType = tkInt64Lit
+    else:
+      case result.tokType
+      of floatTypes:
+        result.fNumber = parseFloat(result.literal)
+      of tkUInt64Lit, tkUIntLit:
+        var iNumber: uint64
+        var len: int
+        try:
+          len = parseBiggestUInt(result.literal, iNumber)
+        except ValueError:
+          raise newException(OverflowDefect, "number out of range: " & $result.literal)
+        if len != result.literal.len:
+          raise newException(ValueError, "invalid integer: " & $result.literal)
+        result.iNumber = cast[int64](iNumber)
+      else:
+        var iNumber: int64
+        var len: int
+        try:
+          len = parseBiggestInt(result.literal, iNumber)
+        except ValueError:
+          raise newException(OverflowDefect, "number out of range: " & $result.literal)
+        if len != result.literal.len:
+          raise newException(ValueError, "invalid integer: " & $result.literal)
+        result.iNumber = iNumber
 
-    except ValueError:
-      lexMessageLitNum(L, "invalid number: '$1'", startpos)
-    except OverflowDefect, RangeDefect:
-      lexMessageLitNum(L, "number out of range: '$1'", startpos)
+      # Explicit bounds checks. Only T.high needs to be considered
+      # since result.iNumber can't be negative.
+      let outOfRange =
+        case result.tokType
+        of tkInt8Lit: result.iNumber > int8.high
+        of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high)
+        of tkInt16Lit: result.iNumber > int16.high
+        of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high)
+        of tkInt32Lit: result.iNumber > int32.high
+        of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high)
+        else: false
+
+      if outOfRange: lexMessageLitNum(L, "number out of range: '$1'", startpos)
+
+    # Promote int literal to int64? Not always necessary, but more consistent
+    if result.tokType == tkIntLit:
+      if result.iNumber > high(int32):
+        result.tokType = tkInt64Lit
+
+  except ValueError:
+    lexMessageLitNum(L, "invalid number: '$1'", startpos)
+  except OverflowDefect, RangeDefect:
+    lexMessageLitNum(L, "number out of range: '$1'", startpos)
   tokenEnd(result, postPos-1)
   L.bufpos = postPos
 
@@ -816,9 +820,8 @@ proc getString(L: var Lexer, tok: var Token, mode: StringMode) =
         inc(pos)
     L.bufpos = pos
 
-proc getCharacter(L: var Lexer; tok: var Token) =
+proc getCharacter(L: var Lexer, tok: var Token) =
   tokenBegin(tok, L.bufpos)
-  let startPos = L.bufpos
   inc(L.bufpos)               # skip '
   var c = L.buf[L.bufpos]
   case c
@@ -829,59 +832,10 @@ proc getCharacter(L: var Lexer; tok: var Token) =
   else:
     tok.literal = $c
     inc(L.bufpos)
-  if L.buf[L.bufpos] == '\'':
-    tokenEndIgnore(tok, L.bufpos)
-    inc(L.bufpos)               # skip '
-  else:
-    if startPos > 0 and L.buf[startPos-1] == '`':
-      tok.literal = "'"
-      L.bufpos = startPos+1
-    else:
-      lexMessage(L, errGenerated, "missing closing ' for character literal")
-    tokenEndIgnore(tok, L.bufpos)
-
-const
-  UnicodeOperatorStartChars = {'\226', '\194', '\195'}
-    # the allowed unicode characters ("∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔")
-    # all start with one of these.
-
-type
-  UnicodeOprPred = enum
-    Mul, Add
-
-proc unicodeOprLen(buf: cstring; pos: int): (int8, UnicodeOprPred) =
-  template m(len): untyped = (int8(len), Mul)
-  template a(len): untyped = (int8(len), Add)
-  result = 0.m
-  case buf[pos]
-  of '\226':
-    if buf[pos+1] == '\136':
-      if buf[pos+2] == '\152': result = 3.m # ∘
-      elif buf[pos+2] == '\153': result = 3.m # ∙
-      elif buf[pos+2] == '\167': result = 3.m # ∧
-      elif buf[pos+2] == '\168': result = 3.a # ∨
-      elif buf[pos+2] == '\169': result = 3.m # ∩
-      elif buf[pos+2] == '\170': result = 3.a # ∪
-    elif buf[pos+1] == '\138':
-      if buf[pos+2] == '\147': result = 3.m # ⊓
-      elif buf[pos+2] == '\148': result = 3.a # ⊔
-      elif buf[pos+2] == '\149': result = 3.a # ⊕
-      elif buf[pos+2] == '\150': result = 3.a # ⊖
-      elif buf[pos+2] == '\151': result = 3.m # ⊗
-      elif buf[pos+2] == '\152': result = 3.m # ⊘
-      elif buf[pos+2] == '\153': result = 3.m # ⊙
-      elif buf[pos+2] == '\155': result = 3.m # ⊛
-      elif buf[pos+2] == '\158': result = 3.a # ⊞
-      elif buf[pos+2] == '\159': result = 3.a # ⊟
-      elif buf[pos+2] == '\160': result = 3.m # ⊠
-      elif buf[pos+2] == '\161': result = 3.m # ⊡
-    elif buf[pos+1] == '\152' and buf[pos+2] == '\133': result = 3.m # ★
-  of '\194':
-    if buf[pos+1] == '\177': result = 2.a # ±
-  of '\195':
-    if buf[pos+1] == '\151': result = 2.m # ×
-  else:
-    discard
+  if L.buf[L.bufpos] != '\'':
+    lexMessage(L, errGenerated, "missing closing ' for character literal")
+  tokenEndIgnore(tok, L.bufpos)
+  inc(L.bufpos)               # skip '
 
 proc getSymbol(L: var Lexer, tok: var Token) =
   var h: Hash = 0
@@ -891,7 +845,7 @@ proc getSymbol(L: var Lexer, tok: var Token) =
   while true:
     var c = L.buf[pos]
     case c
-    of 'a'..'z', '0'..'9':
+    of 'a'..'z', '0'..'9', '\x80'..'\xFF':
       h = h !& ord(c)
       inc(pos)
     of 'A'..'Z':
@@ -905,16 +859,10 @@ proc getSymbol(L: var Lexer, tok: var Token) =
         break
       inc(pos)
       suspicious = true
-    of '\x80'..'\xFF':
-      if c in UnicodeOperatorStartChars and unicodeOperators in L.config.features and unicodeOprLen(L.buf, pos)[0] != 0:
-        break
-      else:
-        h = h !& ord(c)
-        inc(pos)
     else: break
   tokenEnd(tok, pos-1)
   h = !$h
-  tok.ident = L.cache.getIdent(cast[cstring](addr(L.buf[L.bufpos])), pos - L.bufpos, h)
+  tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
   if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
       (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
     tok.tokType = tkSymbol
@@ -928,7 +876,7 @@ proc getSymbol(L: var Lexer, tok: var Token) =
 proc endOperator(L: var Lexer, tok: var Token, pos: int,
                  hash: Hash) {.inline.} =
   var h = !$hash
-  tok.ident = L.cache.getIdent(cast[cstring](addr(L.buf[L.bufpos])), pos - L.bufpos, h)
+  tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
   if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr
   else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon))
   L.bufpos = pos
@@ -939,17 +887,9 @@ proc getOperator(L: var Lexer, tok: var Token) =
   var h: Hash = 0
   while true:
     var c = L.buf[pos]
-    if c in OpChars:
-      h = h !& ord(c)
-      inc(pos)
-    elif c in UnicodeOperatorStartChars and unicodeOperators in L.config.features:
-      let oprLen = unicodeOprLen(L.buf, pos)[0]
-      if oprLen == 0: break
-      for i in 0..<oprLen:
-        h = h !& ord(L.buf[pos])
-        inc pos
-    else:
-      break
+    if c notin OpChars: break
+    h = h !& ord(c)
+    inc(pos)
   endOperator(L, tok, pos, h)
   tokenEnd(tok, pos-1)
   # advance pos but don't store it in L.bufpos so the next token (which might
@@ -963,9 +903,6 @@ proc getOperator(L: var Lexer, tok: var Token) =
 
 proc getPrecedence*(tok: Token): int =
   ## Calculates the precedence of the given token.
-  const
-    MulPred = 9
-    PlusPred = 8
   case tok.tokType
   of tkOpr:
     let relevantChar = tok.ident.s[0]
@@ -979,22 +916,13 @@ proc getPrecedence*(tok: Token): int =
 
     case relevantChar
     of '$', '^': considerAsgn(10)
-    of '*', '%', '/', '\\': considerAsgn(MulPred)
+    of '*', '%', '/', '\\': considerAsgn(9)
     of '~': result = 8
-    of '+', '-', '|': considerAsgn(PlusPred)
+    of '+', '-', '|': considerAsgn(8)
     of '&': considerAsgn(7)
     of '=', '<', '>', '!': result = 5
     of '.': considerAsgn(6)
     of '?': result = 2
-    of UnicodeOperatorStartChars:
-      if tok.ident.s[^1] == '=':
-        result = 1
-      else:
-        let (len, pred) = unicodeOprLen(cstring(tok.ident.s), 0)
-        if len != 0:
-          result = if pred == Mul: MulPred else: PlusPred
-        else:
-          result = 2
     else: considerAsgn(2)
   of tkDiv, tkMod, tkShl, tkShr: result = 9
   of tkDotDot: result = 6
@@ -1027,10 +955,8 @@ proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
   # detect the amount of indentation:
   if isDoc:
     toStrip = getColNumber(L, pos)
-    while L.buf[pos] == ' ':
-      inc pos
-      inc toStrip
-    while L.buf[pos] in {CR, LF}:  # skip blank lines
+    while L.buf[pos] == ' ': inc pos
+    if L.buf[pos] in {CR, LF}:
       pos = handleCRLF(L, pos)
       toStrip = 0
       while L.buf[pos] == ' ':
@@ -1102,17 +1028,11 @@ proc scanComment(L: var Lexer, tok: var Token) =
   inc(pos, 2)
 
   var toStrip = 0
-  var stripInit = false
+  while L.buf[pos] == ' ':
+    inc pos
+    inc toStrip
 
   while true:
-    if not stripInit:  # find baseline indentation inside comment
-      while L.buf[pos] == ' ':
-        inc pos
-        inc toStrip
-      if L.buf[pos] in {CR, LF}:  # don't set toStrip in blank comment lines
-        toStrip = 0
-      else:  # found first non-whitespace character
-        stripInit = true
     var lastBackslash = -1
     while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
       if L.buf[pos] == '\\': lastBackslash = pos+1
@@ -1128,12 +1048,11 @@ proc scanComment(L: var Lexer, tok: var Token) =
     if L.buf[pos] == '#' and L.buf[pos+1] == '#':
       tok.literal.add "\n"
       inc(pos, 2)
-      if stripInit:
-        var c = toStrip
-        while L.buf[pos] == ' ' and c > 0:
-          inc pos
-          dec c
-        inc tok.iNumber
+      var c = toStrip
+      while L.buf[pos] == ' ' and c > 0:
+        inc pos
+        dec c
+      inc tok.iNumber
     else:
       if L.buf[pos] > ' ':
         L.indentAhead = indent
@@ -1218,10 +1137,6 @@ proc skip(L: var Lexer, tok: var Token) =
 proc rawGetTok*(L: var Lexer, tok: var Token) =
   template atTokenEnd() {.dirty.} =
     when defined(nimsuggest):
-      L.previousTokenEnd.line = L.tokenEnd.line
-      L.previousTokenEnd.col = L.tokenEnd.col
-      L.tokenEnd.line = tok.line.uint16
-      L.tokenEnd.col = getColNumber(L, L.bufpos).int16
       # we attach the cursor to the last *strong* token
       if tok.tokType notin weakTokens:
         L.previousToken.line = tok.line.uint16
@@ -1242,15 +1157,10 @@ proc rawGetTok*(L: var Lexer, tok: var Token) =
   var c = L.buf[L.bufpos]
   tok.line = L.lineNumber
   tok.col = getColNumber(L, L.bufpos)
-  if c in SymStartChars - {'r', 'R'} - UnicodeOperatorStartChars:
+  if c in SymStartChars - {'r', 'R'}:
     getSymbol(L, tok)
   else:
     case c
-    of UnicodeOperatorStartChars:
-      if unicodeOperators in L.config.features and unicodeOprLen(L.buf, L.bufpos)[0] != 0:
-        getOperator(L, tok)
-      else:
-        getSymbol(L, tok)
     of '#':
       scanComment(L, tok)
     of '*':
@@ -1358,28 +1268,7 @@ proc rawGetTok*(L: var Lexer, tok: var Token) =
       getNumber(L, tok)
       let c = L.buf[L.bufpos]
       if c in SymChars+{'_'}:
-        if c in UnicodeOperatorStartChars and unicodeOperators in L.config.features and
-            unicodeOprLen(L.buf, L.bufpos)[0] != 0:
-          discard
-        else:
-          lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier")
-    of '-':
-      if L.buf[L.bufpos+1] in {'0'..'9'} and
-          (L.bufpos-1 == 0 or L.buf[L.bufpos-1] in UnaryMinusWhitelist):
-        # x)-23 # binary minus
-        # ,-23  # unary minus
-        # \n-78 # unary minus? Yes.
-        # =-3   # parsed as `=-` anyway
-        getNumber(L, tok)
-        let c = L.buf[L.bufpos]
-        if c in SymChars+{'_'}:
-          if c in UnicodeOperatorStartChars and unicodeOperators in L.config.features and
-              unicodeOprLen(L.buf, L.bufpos)[0] != 0:
-            discard
-          else:
-            lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier")
-      else:
-        getOperator(L, tok)
+        lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier")
     else:
       if c in OpChars:
         getOperator(L, tok)

compiler/liftdestructors.nim

@@ -11,7 +11,7 @@
 ## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``).
 
 import modulegraphs, lineinfos, idents, ast, renderer, semdata,
-  sighashes, lowerings, options, types, msgs, magicsys, tables, ccgutils
+  sighashes, lowerings, options, types, msgs, magicsys, tables
 
 from trees import isCaseObj
 
@@ -27,18 +27,12 @@ type
     canRaise: bool
     filterDiscriminator: PSym  # we generating destructor for case branch
     c: PContext # c can be nil, then we are called from lambdalifting!
-    idgen: IdGenerator
-
-template destructor*(t: PType): PSym = getAttachedOp(c.g, t, attachedDestructor)
-template assignment*(t: PType): PSym = getAttachedOp(c.g, t, attachedAsgn)
-template asink*(t: PType): PSym = getAttachedOp(c.g, t, attachedSink)
 
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode)
 proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
-              info: TLineInfo; idgen: IdGenerator): PSym
+              info: TLineInfo): PSym
 
-proc createTypeBoundOps*(g: ModuleGraph; c: PContext; orig: PType; info: TLineInfo;
-                         idgen: IdGenerator)
+proc createTypeBoundOps*(g: ModuleGraph; c: PContext; orig: PType; info: TLineInfo)
 
 proc at(a, i: PNode, elemType: PType): PNode =
   result = newNodeI(nkBracketExpr, a.info, 2)
@@ -46,9 +40,8 @@ proc at(a, i: PNode, elemType: PType): PNode =
   result[1] = i
   result.typ = elemType
 
-proc destructorOverriden(g: ModuleGraph; t: PType): bool =
-  let op = getAttachedOp(g, t, attachedDestructor)
-  op != nil and sfOverriden in op.flags
+proc destructorOverriden(t: PType): bool =
+  t.attachedOps[attachedDestructor] != nil and sfOverriden in t.attachedOps[attachedDestructor].flags
 
 proc fillBodyTup(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   for i in 0..<t.len:
@@ -70,33 +63,30 @@ proc newAsgnStmt(le, ri: PNode): PNode =
   result[0] = le
   result[1] = ri
 
-proc genBuiltin*(g: ModuleGraph; idgen: IdGenerator; magic: TMagic; name: string; i: PNode): PNode =
+proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode =
   result = newNodeI(nkCall, i.info)
-  result.add createMagic(g, idgen, name, magic).newSymNode
+  result.add createMagic(g, name, magic).newSymNode
   result.add i
 
-proc genBuiltin(c: var TLiftCtx; magic: TMagic; name: string; i: PNode): PNode =
-  result = genBuiltin(c.g, c.idgen, magic, name, i)
-
 proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   if c.kind in {attachedAsgn, attachedDeepCopy, attachedSink}:
     body.add newAsgnStmt(x, y)
   elif c.kind == attachedDestructor and c.addMemReset:
-    let call = genBuiltin(c, mDefault, "default", x)
+    let call = genBuiltin(c.g, mDefault, "default", x)
     call.typ = t
     body.add newAsgnStmt(x, call)
 
-proc genAddr(c: var TLiftCtx; x: PNode): PNode =
+proc genAddr(g: ModuleGraph; x: PNode): PNode =
   if x.kind == nkHiddenDeref:
-    checkSonsLen(x, 1, c.g.config)
+    checkSonsLen(x, 1, g.config)
     result = x[0]
   else:
-    result = newNodeIT(nkHiddenAddr, x.info, makeVarType(x.typ.owner, x.typ, c.idgen))
+    result = newNodeIT(nkHiddenAddr, x.info, makeVarType(x.typ.owner, x.typ))
     result.add x
 
 proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
   result = newNodeI(nkWhileStmt, c.info, 2)
-  let cmp = genBuiltin(c, mLtI, "<", i)
+  let cmp = genBuiltin(c.g, mLtI, "<", i)
   cmp.add genLen(c.g, dest)
   cmp.typ = getSysType(c.g, c.info, tyBool)
   result[0] = cmp
@@ -105,11 +95,11 @@ proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
 proc genIf(c: var TLiftCtx; cond, action: PNode): PNode =
   result = newTree(nkIfStmt, newTree(nkElifBranch, cond, action))
 
-proc genContainerOf(c: var TLiftCtx; objType: PType, field, x: PSym): PNode =
+proc genContainerOf(c: TLiftCtx; objType: PType, field, x: PSym): PNode =
   # generate: cast[ptr ObjType](cast[int](addr(x)) - offsetOf(objType.field))
   let intType = getSysType(c.g, unknownLineInfo, tyInt)
 
-  let addrOf = newNodeIT(nkAddr, c.info, makePtrType(x.owner, x.typ, c.idgen))
+  let addrOf = newNodeIT(nkAddr, c.info, makePtrType(x.owner, x.typ))
   addrOf.add newDeref(newSymNode(x))
   let castExpr1 = newNodeIT(nkCast, c.info, intType)
   castExpr1.add newNodeIT(nkType, c.info, intType)
@@ -119,14 +109,14 @@ proc genContainerOf(c: var TLiftCtx; objType: PType, field, x: PSym): PNode =
   dotExpr.add newNodeIT(nkType, c.info, objType)
   dotExpr.add newSymNode(field)
 
-  let offsetOf = genBuiltin(c, mOffsetOf, "offsetof", dotExpr)
+  let offsetOf = genBuiltin(c.g, mOffsetOf, "offsetof", dotExpr)
   offsetOf.typ = intType
 
-  let minusExpr = genBuiltin(c, mSubI, "-", castExpr1)
+  let minusExpr = genBuiltin(c.g, mSubI, "-", castExpr1)
   minusExpr.typ = intType
   minusExpr.add offsetOf
 
-  let objPtr = makePtrType(objType.owner, objType, c.idgen)
+  let objPtr = makePtrType(objType.owner, objType)
   result = newNodeIT(nkCast, c.info, objPtr)
   result.add newNodeIT(nkType, c.info, objPtr)
   result.add minusExpr
@@ -134,11 +124,11 @@ proc genContainerOf(c: var TLiftCtx; objType: PType, field, x: PSym): PNode =
 proc destructorCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
   var destroy = newNodeIT(nkCall, x.info, op.typ[0])
   destroy.add(newSymNode(op))
-  destroy.add genAddr(c, x)
+  destroy.add genAddr(c.g, x)
   if sfNeverRaises notin op.flags:
     c.canRaise = true
   if c.addMemReset:
-    result = newTree(nkStmtList, destroy, genBuiltin(c, mWasMoved,  "wasMoved", x))
+    result = newTree(nkStmtList, destroy, genBuiltin(c.g, mWasMoved,  "wasMoved", x))
   else:
     result = destroy
 
@@ -162,12 +152,9 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       # the value needs to be destroyed before we assign the selector
       # or the value is lost
       let prevKind = c.kind
-      let prevAddMemReset = c.addMemReset
       c.kind = attachedDestructor
-      c.addMemReset = true
       fillBodyObj(c, n, body, x, y, enforceDefaultOp = false)
       c.kind = prevKind
-      c.addMemReset = prevAddMemReset
       localEnforceDefaultOp = true
 
     if c.kind != attachedDestructor:
@@ -233,16 +220,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
     # for every field (dependent on dest.kind):
     #   `=` dest.field, src.field
     # =destroy(blob)
-    var dummy = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId c.idgen, c.fn, c.info)
-    dummy.typ = y.typ
-    if ccgIntroducedPtr(c.g.config, dummy, y.typ):
-      # Because of potential aliasing when the src param is passed by ref, we need to check for equality here,
-      # because the wasMoved(dest) call would zero out src, if dest aliases src.
-      var cond = newTree(nkCall, newSymNode(c.g.getSysMagic(c.info, "==", mEqRef)),
-        newTreeIT(nkAddr, c.info, makePtrType(c.fn, x.typ, c.idgen), x), newTreeIT(nkAddr, c.info, makePtrType(c.fn, y.typ, c.idgen), y))
-      cond.typ = getSysType(c.g, x.info, tyBool)
-      body.add genIf(c, cond, newTreeI(nkReturnStmt, c.info, newNodeI(nkEmpty, c.info)))
-    var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId c.idgen, c.fn, c.info)
+    var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.fn, c.info)
     temp.typ = x.typ
     incl(temp.flags, sfFromGeneric)
     var v = newNodeI(nkVarSection, c.info)
@@ -252,7 +230,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
     #body.add newAsgnStmt(blob, x)
 
     var wasMovedCall = newNodeI(nkCall, c.info)
-    wasMovedCall.add(newSymNode(createMagic(c.g, c.idgen, "wasMoved", mWasMoved)))
+    wasMovedCall.add(newSymNode(createMagic(c.g, "wasMoved", mWasMoved)))
     wasMovedCall.add x # mWasMoved does not take the address
     body.add wasMovedCall
 
@@ -268,19 +246,6 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
   else:
     fillBodyObjTImpl(c, t, body, x, y)
 
-proc boolLit*(g: ModuleGraph; info: TLineInfo; value: bool): PNode =
-  result = newIntLit(g, info, ord value)
-  result.typ = getSysType(g, info, tyBool)
-
-proc getCycleParam(c: TLiftCtx): PNode =
-  assert c.kind == attachedAsgn
-  if c.fn.typ.len == 4:
-    result = c.fn.typ.n.lastSon
-    assert result.kind == nkSym
-    assert result.sym.name.s == "cyclic"
-  else:
-    result = boolLit(c.g, c.info, true)
-
 proc newHookCall(c: var TLiftCtx; op: PSym; x, y: PNode): PNode =
   #if sfError in op.flags:
   #  localError(c.config, x.info, "usage of '$1' is a user-defined error" % op.name.s)
@@ -289,18 +254,11 @@ proc newHookCall(c: var TLiftCtx; op: PSym; x, y: PNode): PNode =
   if sfNeverRaises notin op.flags:
     c.canRaise = true
   if op.typ.sons[1].kind == tyVar:
-    result.add genAddr(c, x)
+    result.add genAddr(c.g, x)
   else:
     result.add x
   if y != nil:
     result.add y
-  if op.typ.len == 4:
-    assert y != nil
-    if c.fn.typ.len == 4:
-      result.add getCycleParam(c)
-    else:
-      # assume the worst: A cycle is created:
-      result.add boolLit(c.g, y.info, true)
 
 proc newOpCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
   result = newNodeIT(nkCall, x.info, op.typ[0])
@@ -336,8 +294,8 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
                         field: var PSym): bool =
   if optSeqDestructors in c.g.config.globalOptions:
     var op = field
-    let destructorOverriden = destructorOverriden(c.g, t)
-    if op != nil and op != c.fn and
+    let destructorOverriden = destructorOverriden(t)
+    if op != nil and op != c.fn and 
         (sfOverriden in op.flags or destructorOverriden):
       if sfError in op.flags:
         incl c.fn.flags, sfError
@@ -347,7 +305,7 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
       body.add newHookCall(c, op, x, y)
       result = true
     elif op == nil and destructorOverriden:
-      op = produceSym(c.g, c.c, t, c.kind, c.info, c.idgen)
+      op = produceSym(c.g, c.c, t, c.kind, c.info)
       body.add newHookCall(c, op, x, y)
       result = true
   elif tfHasAsgn in t.flags:
@@ -362,14 +320,14 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
     else:
       op = field
       if op == nil:
-        op = produceSym(c.g, c.c, t, c.kind, c.info, c.idgen)
+        op = produceSym(c.g, c.c, t, c.kind, c.info)
     if sfError in op.flags:
       incl c.fn.flags, sfError
     #else:
     #  markUsed(c.g.config, c.info, op, c.g.usageSym)
     onUse(c.info, op)
     # We also now do generic instantiations in the destructor lifting pass:
-    if op.ast.isGenericRoutine:
+    if op.ast[genericParamsPos].kind != nkEmpty:
       op = instantiateGeneric(c, op, t, t.typeInst)
       field = op
       #echo "trying to use ", op.ast
@@ -381,17 +339,17 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
     result = true
 
 proc addDestructorCall(c: var TLiftCtx; orig: PType; body, x: PNode) =
-  let t = orig.skipTypes(abstractInst - {tyDistinct})
+  let t = orig.skipTypes(abstractInst)
   var op = t.destructor
 
   if op != nil and sfOverriden in op.flags:
-    if op.ast.isGenericRoutine:
+    if op.ast[genericParamsPos].kind != nkEmpty:
       # patch generic destructor:
       op = instantiateGeneric(c, op, t, t.typeInst)
-      setAttachedOp(c.g, c.idgen.module, t, attachedDestructor, op)
+      t.attachedOps[attachedDestructor] = op
 
   if op == nil and (useNoGc(c, t) or requiresDestructor(c, t)):
-    op = produceSym(c.g, c.c, t, attachedDestructor, c.info, c.idgen)
+    op = produceSym(c.g, c.c, t, attachedDestructor, c.info)
     doAssert op != nil
     doAssert op == t.destructor
 
@@ -409,10 +367,10 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
     var op = t.destructor
     if op != nil and sfOverriden in op.flags:
 
-      if op.ast.isGenericRoutine:
+      if op.ast[genericParamsPos].kind != nkEmpty:
         # patch generic destructor:
         op = instantiateGeneric(c, op, t, t.typeInst)
-        setAttachedOp(c.g, c.idgen.module, t, attachedDestructor, op)
+        t.attachedOps[attachedDestructor] = op
 
       #markUsed(c.g.config, c.info, op, c.g.usageSym)
       onUse(c.info, op)
@@ -420,19 +378,11 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       result = true
     #result = addDestructorCall(c, t, body, x)
   of attachedAsgn, attachedSink, attachedTrace:
-    var op = getAttachedOp(c.g, t, c.kind)
-    if op != nil and sfOverriden in op.flags:
-      if op.ast.isGenericRoutine:
-        # patch generic =trace:
-        op = instantiateGeneric(c, op, t, t.typeInst)
-        setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
-
-    result = considerAsgnOrSink(c, t, body, x, y, op)
-    if op != nil:
-      setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
-
+    result = considerAsgnOrSink(c, t, body, x, y, t.attachedOps[c.kind])
+  of attachedDispose:
+    result = considerAsgnOrSink(c, t, body, x, nil, t.attachedOps[c.kind])
   of attachedDeepCopy:
-    let op = getAttachedOp(c.g, t, attachedDeepCopy)
+    let op = t.attachedOps[attachedDeepCopy]
     if op != nil:
       #markUsed(c.g.config, c.info, op, c.g.usageSym)
       onUse(c.info, op)
@@ -440,7 +390,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       result = true
 
 proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
-  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info)
+  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.fn, c.info)
   temp.typ = getSysType(c.g, body.info, tyInt)
   incl(temp.flags, sfFromGeneric)
 
@@ -450,7 +400,7 @@ proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
   body.add v
 
 proc declareTempOf(c: var TLiftCtx; body: PNode; value: PNode): PNode =
-  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info)
+  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.fn, c.info)
   temp.typ = value.typ
   incl(temp.flags, sfFromGeneric)
 
@@ -460,25 +410,25 @@ proc declareTempOf(c: var TLiftCtx; body: PNode; value: PNode): PNode =
   body.add v
 
 proc addIncStmt(c: var TLiftCtx; body, i: PNode) =
-  let incCall = genBuiltin(c, mInc, "inc", i)
+  let incCall = genBuiltin(c.g, mInc, "inc", i)
   incCall.add lowerings.newIntLit(c.g, c.info, 1)
   body.add incCall
 
-proc newSeqCall(c: var TLiftCtx; x, y: PNode): PNode =
+proc newSeqCall(g: ModuleGraph; x, y: PNode): PNode =
   # don't call genAddr(c, x) here:
-  result = genBuiltin(c, mNewSeq, "newSeq", x)
-  let lenCall = genBuiltin(c, mLengthSeq, "len", y)
-  lenCall.typ = getSysType(c.g, x.info, tyInt)
+  result = genBuiltin(g, mNewSeq, "newSeq", x)
+  let lenCall = genBuiltin(g, mLengthSeq, "len", y)
+  lenCall.typ = getSysType(g, x.info, tyInt)
   result.add lenCall
 
-proc setLenStrCall(c: var TLiftCtx; x, y: PNode): PNode =
-  let lenCall = genBuiltin(c, mLengthStr, "len", y)
-  lenCall.typ = getSysType(c.g, x.info, tyInt)
-  result = genBuiltin(c, mSetLengthStr, "setLen", x) # genAddr(g, x))
+proc setLenStrCall(g: ModuleGraph; x, y: PNode): PNode =
+  let lenCall = genBuiltin(g, mLengthStr, "len", y)
+  lenCall.typ = getSysType(g, x.info, tyInt)
+  result = genBuiltin(g, mSetLengthStr, "setLen", x) # genAddr(g, x))
   result.add lenCall
 
 proc setLenSeqCall(c: var TLiftCtx; t: PType; x, y: PNode): PNode =
-  let lenCall = genBuiltin(c, mLengthSeq, "len", y)
+  let lenCall = genBuiltin(c.g, mLengthSeq, "len", y)
   lenCall.typ = getSysType(c.g, x.info, tyInt)
   var op = getSysMagic(c.g, x.info, "setLen", mSetLengthSeq)
   op = instantiateGeneric(c, op, t, t)
@@ -508,7 +458,7 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add setLenSeqCall(c, t, x, y)
     forallElements(c, t, body, x, y)
   of attachedSink:
-    let moveCall = genBuiltin(c, mMove, "move", x)
+    let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
     moveCall.add destructorCall(c, t.destructor, x)
@@ -516,19 +466,22 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of attachedDestructor:
     # destroy all elements:
     forallElements(c, t, body, x, y)
-    body.add genBuiltin(c, mDestroy, "destroy", x)
+    body.add genBuiltin(c.g, mDestroy, "destroy", x)
   of attachedTrace:
-    if canFormAcycle(c.g, t.elemType):
+    if canFormAcycle(t.elemType):
       # follow all elements:
       forallElements(c, t, body, x, y)
+  of attachedDispose:
+    forallElements(c, t, body, x, y)
+    body.add genBuiltin(c.g, mDestroy, "destroy", x)
 
 proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  createTypeBoundOps(c.g, c.c, t, body.info, c.idgen)
+  createTypeBoundOps(c.g, c.c, t, body.info)
   # recursions are tricky, so we might need to forward the generated
   # operation here:
   var t = t
   if t.assignment == nil or t.destructor == nil:
-    let h = sighashes.hashType(t,c.g.config, {CoType, CoConsiderOwned, CoDistinct})
+    let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
     let canon = c.g.canonTypes.getOrDefault(h)
     if canon != nil: t = canon
 
@@ -540,7 +493,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add newHookCall(c, t.assignment, x, y)
   of attachedSink:
     # we always inline the move for better performance:
-    let moveCall = genBuiltin(c, mMove, "move", x)
+    let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
     moveCall.add destructorCall(c, t.destructor, x)
@@ -553,33 +506,29 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     doAssert t.destructor != nil
     body.add destructorCall(c, t.destructor, x)
   of attachedTrace:
-    if t.kind != tyString and canFormAcycle(c.g, t.elemType):
-      let op = getAttachedOp(c.g, t, c.kind)
-      if op == nil:
-        return # protect from recursion
-      body.add newHookCall(c, op, x, y)
+    if t.attachedOps[c.kind] == nil:
+      return # protect from recursion
+    body.add newHookCall(c, t.attachedOps[c.kind], x, y)
+  of attachedDispose:
+    if t.attachedOps[c.kind] == nil:
+      return # protect from recursion
+    body.add newHookCall(c, t.attachedOps[c.kind], x, nil)
 
 proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
   of attachedAsgn, attachedDeepCopy:
-    body.add callCodegenProc(c.g, "nimAsgnStrV2", c.info, genAddr(c, x), y)
+    body.add callCodegenProc(c.g, "nimAsgnStrV2", c.info, genAddr(c.g, x), y)
   of attachedSink:
-    let moveCall = genBuiltin(c, mMove, "move", x)
+    let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
     moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
-  of attachedDestructor:
-    body.add genBuiltin(c, mDestroy, "destroy", x)
+  of attachedDestructor, attachedDispose:
+    body.add genBuiltin(c.g, mDestroy, "destroy", x)
   of attachedTrace:
     discard "strings are atomic and have no inner elements that are to trace"
 
-proc cyclicType*(g: ModuleGraph, t: PType): bool =
-  case t.kind
-  of tyRef: result = types.canFormAcycle(g, t.lastSon)
-  of tyProc: result = t.callConv == ccClosure
-  else: result = false
-
 proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   #[ bug #15753 is really subtle. Usually the classical write barrier for reference
   counting looks like this::
@@ -603,8 +552,8 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   var actions = newNodeI(nkStmtList, c.info)
   let elemType = t.lastSon
 
-  createTypeBoundOps(c.g, c.c, elemType, c.info, c.idgen)
-  let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(c.g, elemType)
+  createTypeBoundOps(c.g, c.c, elemType, c.info)
+  let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(elemType)
 
   let tmp =
     if isCyclic and c.kind in {attachedAsgn, attachedSink}:
@@ -614,7 +563,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
 
   if isFinal(elemType):
     addDestructorCall(c, elemType, actions, genDeref(tmp, nkDerefExpr))
-    var alignOf = genBuiltin(c, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType))
+    var alignOf = genBuiltin(c.g, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType))
     alignOf.typ = getSysType(c.g, c.info, tyInt)
     actions.add callCodegenProc(c.g, "nimRawDispose", c.info, tmp, alignOf)
   else:
@@ -624,7 +573,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   var cond: PNode
   if isCyclic:
     if isFinal(elemType):
-      let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
+      let typInfo = genBuiltin(c.g, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
       typInfo.typ = getSysType(c.g, c.info, tyPointer)
       cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo)
     else:
@@ -642,13 +591,12 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.add genIf(c, cond, actions)
       body.add newAsgnStmt(x, y)
   of attachedAsgn:
+    body.add genIf(c, y, callCodegenProc(c.g,
+        if isCyclic: "nimIncRefCyclic" else: "nimIncRef", c.info, y))
     if isCyclic:
-      body.add genIf(c, y, callCodegenProc(c.g,
-          "nimIncRefCyclic", c.info, y, getCycleParam(c)))
       body.add newAsgnStmt(x, y)
       body.add genIf(c, cond, actions)
     else:
-      body.add genIf(c, y, callCodegenProc(c.g, "nimIncRef", c.info, y))
       body.add genIf(c, cond, actions)
       body.add newAsgnStmt(x, y)
   of attachedDestructor:
@@ -657,18 +605,28 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of attachedTrace:
     if isCyclic:
       if isFinal(elemType):
-        let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
+        let typInfo = genBuiltin(c.g, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
         typInfo.typ = getSysType(c.g, c.info, tyPointer)
-        body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x, c.idgen), typInfo, y)
+        body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x), typInfo, y)
       else:
         # If the ref is polymorphic we have to account for this
-        body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(x, c.idgen), y)
+        body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(x), y)
       #echo "can follow ", elemType, " static ", isFinal(elemType)
+  of attachedDispose:
+    # this is crucial! dispose is like =destroy but we don't follow refs
+    # as that is dealt within the cycle collector.
+    if not isCyclic:
+      body.add genIf(c, cond, actions)
+    when false:
+      let cond = copyTree(x)
+      cond.typ = getSysType(c.g, x.info, tyBool)
+      actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x)
+      body.add genIf(c, cond, actions)
 
 proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   ## Closures are really like refs except they always use a virtual destructor
   ## and we need to do the refcounting only on the ref field which we call 'xenv':
-  let xenv = genBuiltin(c, mAccessEnv, "accessEnv", x)
+  let xenv = genBuiltin(c.g, mAccessEnv, "accessEnv", x)
   xenv.typ = getSysType(c.g, c.info, tyPointer)
 
   let isCyclic = c.g.config.selectedGC == gcOrc
@@ -696,22 +654,31 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.add genIf(c, cond, actions)
       body.add newAsgnStmt(x, y)
   of attachedAsgn:
-    let yenv = genBuiltin(c, mAccessEnv, "accessEnv", y)
+    let yenv = genBuiltin(c.g, mAccessEnv, "accessEnv", y)
     yenv.typ = getSysType(c.g, c.info, tyPointer)
+    let incRefProc =
+      if c.g.config.selectedGC == gcOrc: "nimIncRefCyclic"
+      else: "nimIncRef"
+    body.add genIf(c, yenv, callCodegenProc(c.g, incRefProc, c.info, yenv))
     if isCyclic:
-      body.add genIf(c, yenv, callCodegenProc(c.g, "nimIncRefCyclic", c.info, yenv, getCycleParam(c)))
       body.add newAsgnStmt(x, y)
       body.add genIf(c, cond, actions)
     else:
-      body.add genIf(c, yenv, callCodegenProc(c.g, "nimIncRef", c.info, yenv))
-
       body.add genIf(c, cond, actions)
       body.add newAsgnStmt(x, y)
   of attachedDestructor:
     body.add genIf(c, cond, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace:
-    body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv, c.idgen), y)
+    body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv), y)
+  of attachedDispose:
+    # this is crucial! dispose is like =destroy but we don't follow refs
+    # as that is dealt within the cycle collector.
+    when false:
+      let cond = copyTree(xenv)
+      cond.typ = getSysType(c.g, xenv.info, tyBool)
+      actions.add callCodegenProc(c.g, "nimRawDispose", c.info, xenv)
+      body.add genIf(c, cond, actions)
 
 proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
@@ -735,18 +702,18 @@ proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     else:
       body.sons.insert(des, 0)
   of attachedDeepCopy: assert(false, "cannot happen")
-  of attachedTrace: discard
+  of attachedTrace, attachedDispose: discard
 
 proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   var actions = newNodeI(nkStmtList, c.info)
 
   let elemType = t.lastSon
   #fillBody(c, elemType, actions, genDeref(x), genDeref(y))
-  #var disposeCall = genBuiltin(c, mDispose, "dispose", x)
+  #var disposeCall = genBuiltin(c.g, mDispose, "dispose", x)
 
   if isFinal(elemType):
     addDestructorCall(c, elemType, actions, genDeref(x, nkDerefExpr))
-    var alignOf = genBuiltin(c, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType))
+    var alignOf = genBuiltin(c.g, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType))
     alignOf.typ = getSysType(c.g, c.info, tyInt)
     actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x, alignOf)
   else:
@@ -760,7 +727,7 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of attachedDestructor:
     body.add genIf(c, x, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
-  of attachedTrace: discard
+  of attachedTrace, attachedDispose: discard
 
 proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   if c.kind == attachedDeepCopy:
@@ -768,12 +735,12 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     # have to go through some indirection; we delegate this to the codegen:
     let call = newNodeI(nkCall, c.info, 2)
     call.typ = t
-    call[0] = newSymNode(createMagic(c.g, c.idgen, "deepCopy", mDeepCopy))
+    call[0] = newSymNode(createMagic(c.g, "deepCopy", mDeepCopy))
     call[1] = y
     body.add newAsgnStmt(x, call)
   elif (optOwnedRefs in c.g.config.globalOptions and
       optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcOrc}:
-    let xx = genBuiltin(c, mAccessEnv, "accessEnv", x)
+    let xx = genBuiltin(c.g, mAccessEnv, "accessEnv", x)
     xx.typ = getSysType(c.g, c.info, tyPointer)
     case c.kind
     of attachedSink:
@@ -782,7 +749,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
       body.add newAsgnStmt(x, y)
     of attachedAsgn:
-      let yy = genBuiltin(c, mAccessEnv, "accessEnv", y)
+      let yy = genBuiltin(c.g, mAccessEnv, "accessEnv", y)
       yy.typ = getSysType(c.g, c.info, tyPointer)
       body.add genIf(c, yy, callCodegenProc(c.g, "nimIncRef", c.info, yy))
       body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
@@ -794,10 +761,10 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       else:
         body.sons.insert(des, 0)
     of attachedDeepCopy: assert(false, "cannot happen")
-    of attachedTrace: discard
+    of attachedTrace, attachedDispose: discard
 
 proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  let xx = genBuiltin(c, mAccessEnv, "accessEnv", x)
+  let xx = genBuiltin(c.g, mAccessEnv, "accessEnv", x)
   xx.typ = getSysType(c.g, c.info, tyPointer)
   var actions = newNodeI(nkStmtList, c.info)
   #discard addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(xx))
@@ -809,12 +776,12 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of attachedDestructor:
     body.add genIf(c, xx, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
-  of attachedTrace: discard
+  of attachedTrace, attachedDispose: discard
 
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case t.kind
   of tyNone, tyEmpty, tyVoid: discard
-  of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCstring,
+  of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString,
       tyPtr, tyUncheckedArray, tyVar, tyLent:
     defaultOp(c, t, body, x, y)
   of tyRef:
@@ -860,7 +827,7 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       discard considerUserDefinedOp(c, t, body, x, y)
     elif tfHasAsgn in t.flags:
       if c.kind in {attachedAsgn, attachedSink, attachedDeepCopy}:
-        body.add newSeqCall(c, x, y)
+        body.add newSeqCall(c.g, x, y)
       forallElements(c, t, body, x, y)
     else:
       defaultOp(c, t, body, x, y)
@@ -897,44 +864,35 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of tyOrdinal, tyRange, tyInferred,
      tyGenericInst, tyAlias, tySink:
     fillBody(c, lastSon(t), body, x, y)
-  of tyConcept, tyIterable: doAssert false
+  of tyOptDeprecated: doAssert false
 
 proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType;
-                            kind: TTypeAttachedOp; info: TLineInfo;
-                            idgen: IdGenerator): PSym =
+                            kind: TTypeAttachedOp; info: TLineInfo): PSym =
   assert typ.kind == tyDistinct
   let baseType = typ[0]
-  if getAttachedOp(g, baseType, kind) == nil:
-    discard produceSym(g, c, baseType, kind, info, idgen)
-  result = getAttachedOp(g, baseType, kind)
-  setAttachedOp(g, idgen.module, typ, kind, result)
+  if baseType.attachedOps[kind] == nil:
+    discard produceSym(g, c, baseType, kind, info)
+  typ.attachedOps[kind] = baseType.attachedOps[kind]
+  result = typ.attachedOps[kind]
 
 proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp;
-              info: TLineInfo; idgen: IdGenerator): PSym =
+              info: TLineInfo): PSym =
 
   let procname = getIdent(g.cache, AttachedOpToStr[kind])
-  result = newSym(skProc, procname, nextSymId(idgen), owner, info)
-  let dest = newSym(skParam, getIdent(g.cache, "dest"), nextSymId(idgen), result, info)
-  let src = newSym(skParam, getIdent(g.cache, if kind == attachedTrace: "env" else: "src"),
-                   nextSymId(idgen), result, info)
-  dest.typ = makeVarType(typ.owner, typ, idgen)
+  result = newSym(skProc, procname, owner, info)
+  let dest = newSym(skParam, getIdent(g.cache, "dest"), result, info)
+  let src = newSym(skParam, getIdent(g.cache, if kind == attachedTrace: "env" else: "src"), result, info)
+  dest.typ = makeVarType(typ.owner, typ)
   if kind == attachedTrace:
     src.typ = getSysType(g, info, tyPointer)
   else:
     src.typ = typ
 
-  result.typ = newProcType(info, nextTypeId(idgen), owner)
+  result.typ = newProcType(info, owner)
   result.typ.addParam dest
-  if kind != attachedDestructor:
+  if kind notin {attachedDestructor, attachedDispose}:
     result.typ.addParam src
 
-  if kind == attachedAsgn and g.config.selectedGC == gcOrc and
-      cyclicType(g, typ.skipTypes(abstractInst)):
-    let cycleParam = newSym(skParam, getIdent(g.cache, "cyclic"),
-                            nextSymId(idgen), result, info)
-    cycleParam.typ = getSysType(g, info, tyBool)
-    result.typ.addParam cycleParam
-
   var n = newNodeI(nkProcDef, info, bodyPos+1)
   for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
   n[namePos] = newSymNode(result)
@@ -944,37 +902,31 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
   incl result.flags, sfFromGeneric
   incl result.flags, sfGeneratedOp
 
-proc genTypeFieldCopy(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  let xx = genBuiltin(c, mAccessTypeField, "accessTypeField", x)
-  let yy = genBuiltin(c, mAccessTypeField, "accessTypeField", y)
-  xx.typ = getSysType(c.g, c.info, tyPointer)
-  yy.typ = xx.typ
-  body.add newAsgnStmt(xx, yy)
 
 proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
-              info: TLineInfo; idgen: IdGenerator): PSym =
+              info: TLineInfo): PSym =
   if typ.kind == tyDistinct:
-    return produceSymDistinctType(g, c, typ, kind, info, idgen)
+    return produceSymDistinctType(g, c, typ, kind, info)
 
-  result = getAttachedOp(g, typ, kind)
+  result = typ.attachedOps[kind]
   if result == nil:
-    result = symPrototype(g, typ, typ.owner, kind, info, idgen)
+    result = symPrototype(g, typ, typ.owner, kind, info)
 
-  var a = TLiftCtx(info: info, g: g, kind: kind, c: c, asgnForType: typ, idgen: idgen,
-                   fn: result)
+  var a = TLiftCtx(info: info, g: g, kind: kind, c: c, asgnForType:typ)
+  a.fn = result
 
   let dest = result.typ.n[1].sym
   let d = newDeref(newSymNode(dest))
-  let src = if kind == attachedDestructor: newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
+  let src = if kind in {attachedDestructor, attachedDispose}: newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
             else: newSymNode(result.typ.n[2].sym)
 
   # register this operation already:
-  setAttachedOpPartial(g, idgen.module, typ, kind, result)
+  typ.attachedOps[kind] = result
 
-  if kind == attachedSink and destructorOverriden(g, typ):
+  if kind == attachedSink and destructorOverriden(typ):
     ## compiler can use a combination of `=destroy` and memCopy for sink op
     dest.flags.incl sfCursor
-    result.ast[bodyPos].add newOpCall(a, getAttachedOp(g, typ, attachedDestructor), d[0])
+    result.ast[bodyPos].add newOpCall(a, typ.attachedOps[attachedDestructor], d[0])
     result.ast[bodyPos].add newAsgnStmt(d, src)
   else:
     var tk: TTypeKind
@@ -989,27 +941,21 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
       fillStrOp(a, typ, result.ast[bodyPos], d, src)
     else:
       fillBody(a, typ, result.ast[bodyPos], d, src)
-      if tk == tyObject and a.kind in {attachedAsgn, attachedSink, attachedDeepCopy} and not lacksMTypeField(typ):
-        # bug #19205: Do not forget to also copy the hidden type field:
-        genTypeFieldCopy(a, typ, result.ast[bodyPos], d, src)
-
   if not a.canRaise: incl result.flags, sfNeverRaises
-  completePartialOp(g, idgen.module, typ, kind, result)
 
 
-proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym,
-                                        info: TLineInfo; idgen: IdGenerator): PSym =
+proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym, info: TLineInfo): PSym =
   assert(typ.skipTypes({tyAlias, tyGenericInst}).kind == tyObject)
-  result = symPrototype(g, field.typ, typ.owner, attachedDestructor, info, idgen)
-  var a = TLiftCtx(info: info, g: g, kind: attachedDestructor, asgnForType: typ, idgen: idgen,
-                   fn: result)
+  result = symPrototype(g, field.typ, typ.owner, attachedDestructor, info)
+  var a = TLiftCtx(info: info, g: g, kind: attachedDestructor, asgnForType: typ)
+  a.fn = result
   a.asgnForType = typ
   a.filterDiscriminator = field
   a.addMemReset = true
   let discrimantDest = result.typ.n[1].sym
 
-  let dst = newSym(skVar, getIdent(g.cache, "dest"), nextSymId(idgen), result, info)
-  dst.typ = makePtrType(typ.owner, typ, idgen)
+  let dst = newSym(skVar, getIdent(g.cache, "dest"), result, info)
+  dst.typ = makePtrType(typ.owner, typ)
   let dstSym = newSymNode(dst)
   let d = newDeref(dstSym)
   let v = newNodeI(nkVarSection, info)
@@ -1023,46 +969,40 @@ proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym,
 template liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
   discard "now a nop"
 
-proc patchBody(g: ModuleGraph; c: PContext; n: PNode; info: TLineInfo; idgen: IdGenerator) =
+proc patchBody(g: ModuleGraph; c: PContext; n: PNode; info: TLineInfo) =
   if n.kind in nkCallKinds:
     if n[0].kind == nkSym and n[0].sym.magic == mDestroy:
       let t = n[1].typ.skipTypes(abstractVar)
-      if getAttachedOp(g, t, attachedDestructor) == nil:
-        discard produceSym(g, c, t, attachedDestructor, info, idgen)
+      if t.destructor == nil:
+        discard produceSym(g, c, t, attachedDestructor, info)
 
-      let op = getAttachedOp(g, t, attachedDestructor)
-      if op != nil:
-        if op.ast.isGenericRoutine:
+      if t.destructor != nil:
+        if t.destructor.ast[genericParamsPos].kind != nkEmpty:
           internalError(g.config, info, "resolved destructor is generic")
-        if op.magic == mDestroy:
+        if t.destructor.magic == mDestroy:
           internalError(g.config, info, "patching mDestroy with mDestroy?")
-        n[0] = newSymNode(op)
-  for x in n: patchBody(g, c, x, info, idgen)
+        n[0] = newSymNode(t.destructor)
+  for x in n: patchBody(g, c, x, info)
 
-proc inst(g: ModuleGraph; c: PContext; t: PType; kind: TTypeAttachedOp; idgen: IdGenerator;
-          info: TLineInfo) =
-  let op = getAttachedOp(g, t, kind)
-  if op != nil and op.ast != nil and op.ast.isGenericRoutine:
+template inst(field, t) =
+  if field.ast != nil and field.ast[genericParamsPos].kind != nkEmpty:
     if t.typeInst != nil:
       var a: TLiftCtx
       a.info = info
       a.g = g
-      a.kind = kind
+      a.kind = k
       a.c = c
-      a.idgen = idgen
 
-      let opInst = instantiateGeneric(a, op, t, t.typeInst)
-      if opInst.ast != nil:
-        patchBody(g, c, opInst.ast, info, a.idgen)
-      setAttachedOp(g, idgen.module, t, kind, opInst)
+      field = instantiateGeneric(a, field, t, t.typeInst)
+      if field.ast != nil:
+        patchBody(g, c, field.ast, info)
     else:
       localError(g.config, info, "unresolved generic parameter")
 
 proc isTrival(s: PSym): bool {.inline.} =
   s == nil or (s.ast != nil and s.ast[bodyPos].len == 0)
 
-proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInfo;
-                        idgen: IdGenerator) =
+proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInfo) =
   ## In the semantic pass this is called in strategic places
   ## to ensure we lift assignment, destructors and moves properly.
   ## The later 'injectdestructors' pass depends on it.
@@ -1072,7 +1012,7 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
   let skipped = orig.skipTypes({tyGenericInst, tyAlias, tySink})
   if isEmptyContainer(skipped) or skipped.kind == tyStatic: return
 
-  let h = sighashes.hashType(skipped, g.config, {CoType, CoConsiderOwned, CoDistinct})
+  let h = sighashes.hashType(skipped, {CoType, CoConsiderOwned, CoDistinct})
   var canon = g.canonTypes.getOrDefault(h)
   if canon == nil:
     g.canonTypes[h] = skipped
@@ -1085,32 +1025,30 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
   # 4. We have a custom destructor.
   # 5. We have a (custom) generic destructor.
 
-  # we do not generate '=trace' procs if we
+  # we do not generate '=trace' nor '=dispose' procs if we
   # have the cycle detection disabled, saves code size.
-  let lastAttached = if g.config.selectedGC == gcOrc: attachedTrace
+  let lastAttached = if g.config.selectedGC == gcOrc: attachedDispose
                      else: attachedSink
 
   # bug #15122: We need to produce all prototypes before entering the
-  # mind boggling recursion. Hacks like these imply we should rewrite
+  # mind boggling recursion. Hacks like these imply we shoule rewrite
   # this module.
-  var generics: array[attachedDestructor..attachedTrace, bool]
+  var generics: array[attachedDestructor..attachedDispose, bool]
   for k in attachedDestructor..lastAttached:
-    generics[k] = getAttachedOp(g, canon, k) != nil
+    generics[k] = canon.attachedOps[k] != nil
     if not generics[k]:
-      setAttachedOp(g, idgen.module, canon, k,
-          symPrototype(g, canon, canon.owner, k, info, idgen))
+      canon.attachedOps[k] = symPrototype(g, canon, canon.owner, k, info)
 
   # we generate the destructor first so that other operators can depend on it:
   for k in attachedDestructor..lastAttached:
     if not generics[k]:
-      discard produceSym(g, c, canon, k, info, idgen)
+      discard produceSym(g, c, canon, k, info)
     else:
-      inst(g, c, canon, k, idgen, info)
+      inst(canon.attachedOps[k], canon)
     if canon != orig:
-      setAttachedOp(g, idgen.module, orig, k, getAttachedOp(g, canon, k))
+      orig.attachedOps[k] = canon.attachedOps[k]
 
-  if not isTrival(getAttachedOp(g, orig, attachedDestructor)):
+  if not isTrival(orig.destructor):
     #or not isTrival(orig.assignment) or
     # not isTrival(orig.sink):
     orig.flags.incl tfHasAsgn
-    # ^ XXX Breaks IC!

compiler/liftlocals.nim

@@ -21,14 +21,13 @@ type
     partialParam: PSym
     objType: PType
     cache: IdentCache
-    idgen: IdGenerator
 
 proc interestingVar(s: PSym): bool {.inline.} =
   result = s.kind in {skVar, skLet, skTemp, skForVar, skResult} and
     sfGlobal notin s.flags
 
 proc lookupOrAdd(c: var Ctx; s: PSym; info: TLineInfo): PNode =
-  let field = addUniqueField(c.objType, s, c.cache, c.idgen)
+  let field = addUniqueField(c.objType, s, c.cache)
   var deref = newNodeI(nkHiddenDeref, info)
   deref.typ = c.objType
   deref.add(newSymNode(c.partialParam, info))
@@ -54,8 +53,7 @@ proc lookupParam(params, dest: PNode): PSym =
     if params[i].kind == nkSym and params[i].sym.name.id == dest.ident.id:
       return params[i].sym
 
-proc liftLocalsIfRequested*(prc: PSym; n: PNode; cache: IdentCache; conf: ConfigRef;
-                            idgen: IdGenerator): PNode =
+proc liftLocalsIfRequested*(prc: PSym; n: PNode; cache: IdentCache; conf: ConfigRef): PNode =
   let liftDest = getPragmaVal(prc.ast, wLiftLocals)
   if liftDest == nil: return n
   let partialParam = lookupParam(prc.typ.n, liftDest)
@@ -67,7 +65,7 @@ proc liftLocalsIfRequested*(prc: PSym; n: PNode; cache: IdentCache; conf: Config
   if objType.kind != tyObject or tfPartial notin objType.flags:
     localError(conf, liftDest.info, "parameter '$1' is not a pointer to a partial object" % $liftDest)
     return n
-  var c = Ctx(partialParam: partialParam, objType: objType, cache: cache, idgen: idgen)
+  var c = Ctx(partialParam: partialParam, objType: objType, cache: cache)
   let w = newTree(nkStmtList, n)
   liftLocals(w, 0, c)
   result = w[0]

compiler/lineinfos.nim

@@ -27,36 +27,23 @@ proc createDocLink*(urlSuffix: string): string =
 
 type
   TMsgKind* = enum
-    # fatal errors
-    errUnknown, errFatal, errInternal,
-    # non-fatal errors
-    errIllFormedAstX, errCannotOpenFile,
+    errUnknown, errInternal, errIllFormedAstX, errCannotOpenFile,
     errXExpected,
-    errRstGridTableNotImplemented,
-    errRstMarkdownIllformedTable,
-    errRstNewSectionExpected,
-    errRstGeneralParseError,
-    errRstInvalidDirectiveX,
-    errRstInvalidField,
-    errRstFootnoteMismatch,
-    errRstSandboxedDirective,
+    errGridTableNotImplemented,
+    errGeneralParseError,
+    errNewSectionExpected,
+    errInvalidDirectiveX,
     errProveInit, # deadcode
     errGenerated,
     errUser,
-    # warnings
+
     warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape",
     warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit",
     warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated",
-    warnDotLikeOps = "DotLikeOps",
     warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic",
-    warnRstRedefinitionOfLabel = "RedefinitionOfLabel",
-    warnRstUnknownSubstitutionX = "UnknownSubstitutionX",
-    warnRstBrokenLink = "BrokenLink",
-    warnRstLanguageXNotSupported = "LanguageXNotSupported",
-    warnRstFieldXNotSupported = "FieldXNotSupported",
-    warnRstStyle = "warnRstStyle",
-    warnCommentXIgnored = "CommentXIgnored",
-    warnTypelessParam = "TypelessParam",
+    warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX",
+    warnLanguageXNotSupported = "LanguageXNotSupported", warnFieldXNotSupported = "FieldXNotSupported",
+    warnCommentXIgnored = "CommentXIgnored", warnTypelessParam = "TypelessParam",
     warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap",
     warnUnsafeCode = "UnsafeCode", warnUnusedImportX = "UnusedImport",
     warnInheritFromException = "InheritFromException", warnEachIdentIsTuple = "EachIdentIsTuple",
@@ -68,27 +55,14 @@ type
     warnLockLevel = "LockLevel", warnResultShadowed = "ResultShadowed",
     warnInconsistentSpacing = "Spacing",  warnCaseTransition = "CaseTransition",
     warnCycleCreated = "CycleCreated", warnObservableStores = "ObservableStores",
-    warnStrictNotNil = "StrictNotNil",
     warnResultUsed = "ResultUsed",
-    warnCannotOpen = "CannotOpen",
-    warnFileChanged = "FileChanged",
-    warnSuspiciousEnumConv = "EnumConv",
-    warnAnyEnumConv = "AnyEnumConv",
-    warnHoleEnumConv = "HoleEnumConv",
-    warnCstringConv = "CStringConv",
-    warnPtrToCstringConv = "PtrToCstringConv",
-    warnEffect = "Effect",
-    warnBareExcept = "BareExcept",
-    warnCastSizes = "CastSizes"
     warnUser = "User",
-    # hints
-    hintSuccess = "Success", hintSuccessX = "SuccessX",
-    hintCC = "CC",
-    hintLineTooLong = "LineTooLong",
-    hintXDeclaredButNotUsed = "XDeclaredButNotUsed", hintDuplicateModuleImport = "DuplicateModuleImport",
+
+    hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC",
+    hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed",
     hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded",
     hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX",
-    hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintProcessingStmt = "ProcessingStmt", hintCodeBegin = "CodeBegin",
+    hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintCodeBegin = "CodeBegin",
     hintCodeEnd = "CodeEnd", hintConf = "Conf", hintPath = "Path",
     hintConditionAlwaysTrue = "CondTrue", hintConditionAlwaysFalse = "CondFalse", hintName = "Name",
     hintPattern = "Pattern", hintExecuting = "Exec", hintLinking = "Link", hintDependency = "Dependency",
@@ -96,24 +70,18 @@ type
     hintGCStats = "GCStats", hintGlobalVar = "GlobalVar", hintExpandMacro = "ExpandMacro",
     hintUser = "User", hintUserRaw = "UserRaw", hintExtendedContext = "ExtendedContext",
     hintMsgOrigin = "MsgOrigin", # since 1.3.5
-    hintDeclaredLoc = "DeclaredLoc", # since 1.5.1
 
 const
   MsgKindToStr*: array[TMsgKind, string] = [
     errUnknown: "unknown error",
-    errFatal: "fatal error: $1",
     errInternal: "internal error: $1",
     errIllFormedAstX: "illformed AST: $1",
     errCannotOpenFile: "cannot open '$1'",
     errXExpected: "'$1' expected",
-    errRstGridTableNotImplemented: "grid table is not implemented",
-    errRstMarkdownIllformedTable: "illformed delimiter row of a markdown table",
-    errRstNewSectionExpected: "new section expected $1",
-    errRstGeneralParseError: "general parse error",
-    errRstInvalidDirectiveX: "invalid directive: '$1'",
-    errRstInvalidField: "invalid field: $1",
-    errRstFootnoteMismatch: "number of footnotes and their references don't match: $1",
-    errRstSandboxedDirective: "disabled directive: '$1'",
+    errGridTableNotImplemented: "grid table is not implemented",
+    errGeneralParseError: "general parse error",
+    errNewSectionExpected: "new section expected",
+    errInvalidDirectiveX: "invalid directive: '$1'",
     errProveInit: "Cannot prove that '$1' is initialized.",  # deadcode
     errGenerated: "$1",
     errUser: "$1",
@@ -123,17 +91,14 @@ const
     warnXmightNotBeenInit: "'$1' might not have been initialized",
     warnDeprecated: "$1",
     warnConfigDeprecated: "config file '$1' is deprecated",
-    warnDotLikeOps: "$1",
     warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)",
     warnUnknownMagic: "unknown magic '$1' might crash the compiler",
-    warnRstRedefinitionOfLabel: "redefinition of label '$1'",
-    warnRstUnknownSubstitutionX: "unknown substitution '$1'",
-    warnRstBrokenLink: "broken link '$1'",
-    warnRstLanguageXNotSupported: "language '$1' not supported",
-    warnRstFieldXNotSupported: "field '$1' not supported",
-    warnRstStyle: "RST style: $1",
+    warnRedefinitionOfLabel: "redefinition of label '$1'",
+    warnUnknownSubstitutionX: "unknown substitution '$1'",
+    warnLanguageXNotSupported: "language '$1' not supported",
+    warnFieldXNotSupported: "field '$1' not supported",
     warnCommentXIgnored: "comment '$1' ignored",
-    warnTypelessParam: "", # deadcode
+    warnTypelessParam: "'$1' has no type. Typeless parameters are deprecated; only allowed for 'template'",
     warnUseBase: "use {.base.} for base methods; baseless methods are deprecated",
     warnWriteToForeignHeap: "write to foreign heap",
     warnUnsafeCode: "unsafe code: '$1'",
@@ -161,33 +126,20 @@ const
     warnCaseTransition: "Potential object case transition, instantiate new object instead",
     warnCycleCreated: "$1",
     warnObservableStores: "observable stores to '$1'",
-    warnStrictNotNil: "$1",
     warnResultUsed: "used 'result' variable",
-    warnCannotOpen: "cannot open: $1",
-    warnFileChanged: "file changed: $1",
-    warnSuspiciousEnumConv: "$1",
-    warnAnyEnumConv: "$1",
-    warnHoleEnumConv: "$1",
-    warnCstringConv: "$1",
-    warnPtrToCstringConv: "unsafe conversion to 'cstring' from '$1'; this will become a compile time error in the future",
-    warnEffect: "$1",
-    warnBareExcept: "$1",
-    warnCastSizes: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
-    hintSuccessX: "$build\n$loc lines; ${sec}s; $mem; proj: $project; out: $output",
+    hintSuccessX: "${loc} lines; ${sec}s; $mem; $build build; proj: $project; out: $output",
     hintCC: "CC: $1",
     hintLineTooLong: "line too long",
     hintXDeclaredButNotUsed: "'$1' is declared but not used",
-    hintDuplicateModuleImport: "$1",
     hintXCannotRaiseY: "$1",
     hintConvToBaseNotNeeded: "conversion to base object is not needed",
     hintConvFromXtoItselfNotNeeded: "conversion from $1 to itself is pointless",
     hintExprAlwaysX: "expression evaluates always to '$1'",
     hintQuitCalled: "quit() called",
     hintProcessing: "$1",
-    hintProcessingStmt: "$1",
     hintCodeBegin: "generated code listing:",
     hintCodeEnd: "end of listing",
     hintConf: "used config file '$1'",
@@ -209,29 +161,28 @@ const
     hintUserRaw: "$1",
     hintExtendedContext: "$1",
     hintMsgOrigin: "$1",
-    hintDeclaredLoc: "$1",
   ]
 
 const
-  fatalMsgs* = {errUnknown..errInternal}
+  fatalMin* = errUnknown
+  fatalMax* = errInternal
   errMin* = errUnknown
   errMax* = errUser
   warnMin* = warnCannotOpenFile
   warnMax* = pred(hintSuccess)
   hintMin* = hintSuccess
   hintMax* = high(TMsgKind)
-  rstWarnings* = {warnRstRedefinitionOfLabel..warnRstStyle}
 
 type
   TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints
   TNoteKinds* = set[TNoteKind]
 
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
-  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed, warnAnyEnumConv, warnBareExcept}
-  result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext, hintDeclaredLoc, hintProcessingStmt}
+  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed}
+  result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext}
   result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
-    hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin, hintPerformance}
+    hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin}
   result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf,
     hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC}
 
@@ -325,4 +276,3 @@ proc initMsgConfig*(): MsgConfig =
   result.filenameToIndexTbl = initTable[string, FileIndex]()
   result.fileInfos = @[]
   result.errorOutputs = {eStdOut, eStdErr}
-  result.filenameToIndexTbl["???"] = FileIndex(-1)

compiler/linter.nim

@@ -9,11 +9,9 @@
 
 ## This module implements the style checker.
 
-import std/strutils
-from std/sugar import dup
+import strutils
 
-import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs, semdata, packages
-export packages
+import options, ast, msgs, idents, lineinfos, wordrecg
 
 const
   Letters* = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF', '_'}
@@ -42,7 +40,7 @@ proc beautifyName(s: string, k: TSymKind): string =
              "pointer", "float", "csize", "csize_t", "cdouble", "cchar", "cschar",
              "cshort", "cu", "nil", "typedesc", "auto", "any",
              "range", "openarray", "varargs", "set", "cfloat", "ref", "ptr",
-             "untyped", "typed", "static", "sink", "lent", "type", "owned", "iterable"]:
+             "untyped", "typed", "static", "sink", "lent", "type", "owned"]:
       result.add s[i]
     else:
       result.add toUpperAscii(s[i])
@@ -57,9 +55,7 @@ proc beautifyName(s: string, k: TSymKind): string =
   inc i
   while i < s.len:
     if s[i] == '_':
-      if i+1 >= s.len:
-        discard "trailing underscores should be stripped off"
-      elif i > 0 and s[i-1] in {'A'..'Z'}:
+      if i > 0 and s[i-1] in {'A'..'Z'}:
         # don't skip '_' as it's essential for e.g. 'GC_disable'
         result.add('_')
         inc i
@@ -86,33 +82,24 @@ proc differ*(line: string, a, b: int, x: string): string =
       result = y
 
 proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
+  # operators stay as they are:
+  if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
+  if k in {skType, skGenericParam} and sfAnon in s.flags: return
+  if s.typ != nil and s.typ.kind == tyTypeDesc: return
+  if {sfImportc, sfExportc} * s.flags != {}: return
+  if optStyleCheck notin s.options: return
   let beau = beautifyName(s.name.s, k)
   if s.name.s != beau:
     lintReport(conf, info, beau, s.name.s)
 
-template styleCheckDef*(ctx: PContext; info: TLineInfo; sym: PSym; k: TSymKind) =
-  ## Check symbol definitions adhere to NEP1 style rules.
-  if optStyleCheck in ctx.config.options and # ignore if styleChecks are off
-     {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # check only if hint/error is enabled
-     hintName in ctx.config.notes and # ignore if name checks are not requested
-     ctx.config.belongsToProjectPackage(sym) and # ignore foreign packages
-     optStyleUsages notin ctx.config.globalOptions and # ignore if requested to only check name usage
-     sym.kind != skResult and # ignore `result`
-     sym.kind != skTemp and # ignore temporary variables created by the compiler
-     sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols???
-     k notin {skType, skGenericParam} and # ignore types and generic params
-     (sym.typ == nil or sym.typ.kind != tyTypeDesc) and # ignore `typedesc`
-     {sfImportc, sfExportc} * sym.flags == {} and # ignore FFI
-     sfAnon notin sym.flags: # ignore if created by compiler
-    nep1CheckDefImpl(ctx.config, info, sym, k)
+template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
+  if {optStyleHint, optStyleError} * conf.globalOptions != {} and optStyleUsages notin conf.globalOptions:
+    nep1CheckDefImpl(conf, info, s, k)
 
-template styleCheckDef*(ctx: PContext; info: TLineInfo; s: PSym) =
-  ## Check symbol definitions adhere to NEP1 style rules.
-  styleCheckDef(ctx, info, s, s.kind)
-
-template styleCheckDef*(ctx: PContext; s: PSym) =
-  ## Check symbol definitions adhere to NEP1 style rules.
-  styleCheckDef(ctx, s.info, s, s.kind)
+template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) =
+  styleCheckDef(conf, info, s, s.kind)
+template styleCheckDef*(conf: ConfigRef; s: PSym) =
+  styleCheckDef(conf, s.info, s, s.kind)
 
 proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string =
   let line = sourceLine(conf, info)
@@ -126,31 +113,21 @@ proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string =
   let last = first+identLen(line, first)-1
   result = differ(line, first, last, newName)
 
-proc styleCheckUseImpl(conf: ConfigRef; info: TLineInfo; s: PSym) =
+proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) =
+  if info.fileIndex.int < 0: return
+  # we simply convert it to what it looks like in the definition
+  # for consistency
+
+  # operators stay as they are:
+  if s.kind == skTemp or s.name.s[0] notin Letters or sfAnon in s.flags:
+    return
+
   let newName = s.name.s
-  let badName = differs(conf, info, newName)
-  if badName.len > 0:
-    lintReport(conf, info, newName, badName, "".dup(addDeclaredLoc(conf, s)))
+  let oldName = differs(conf, info, newName)
+  if oldName.len > 0:
+    lintReport(conf, info, newName, oldName)
 
-template styleCheckUse*(ctx: PContext; info: TLineInfo; sym: PSym) =
-  ## Check symbol uses match their definition's style.
-  if {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # ignore if styleChecks are off
-     hintName in ctx.config.notes and # ignore if name checks are not requested
-     ctx.config.belongsToProjectPackage(sym) and # ignore foreign packages
-     sym.kind != skTemp and # ignore temporary variables created by the compiler
-     sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols???
-     sfAnon notin sym.flags: # ignore temporary variables created by the compiler
-    styleCheckUseImpl(ctx.config, info, sym)
-
-proc checkPragmaUseImpl(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
+proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
   let wanted = $w
   if pragmaName != wanted:
     lintReport(conf, info, wanted, pragmaName)
-
-template checkPragmaUse*(ctx: PContext; info: TLineInfo; w: TSpecialWord; pragmaName: string, sym: PSym) =
-  ## Check builtin pragma uses match their definition's style.
-  ## Note: This only applies to builtin pragmas, not user pragmas.
-  if {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # ignore if styleChecks are off
-     hintName in ctx.config.notes and # ignore if name checks are not requested
-     (sym != nil and ctx.config.belongsToProjectPackage(sym)): # ignore foreign packages
-    checkPragmaUseImpl(ctx.config, info, w, pragmaName)

compiler/llstream.nim

@@ -20,7 +20,6 @@ when hasRstdin: import rdstdin
 
 type
   TLLRepl* = proc (s: PLLStream, buf: pointer, bufLen: int): int
-  OnPrompt* = proc() {.closure.}
   TLLStreamKind* = enum       # enum of different stream implementations
     llsNone,                  # null stream: reading and writing has no effect
     llsString,                # stream encapsulates a string
@@ -33,7 +32,6 @@ type
     rd*, wr*: int             # for string streams
     lineOffset*: int          # for fake stdin line numbers
     repl*: TLLRepl            # gives stdin control to clients
-    onPrompt*: OnPrompt
 
   PLLStream* = ref TLLStream
 
@@ -57,13 +55,12 @@ proc llStreamOpen*(): PLLStream =
   result.kind = llsNone
 
 proc llReadFromStdin(s: PLLStream, buf: pointer, bufLen: int): int
-proc llStreamOpenStdIn*(r: TLLRepl = llReadFromStdin, onPrompt: OnPrompt = nil): PLLStream =
+proc llStreamOpenStdIn*(r: TLLRepl = llReadFromStdin): PLLStream =
   new(result)
   result.kind = llsStdIn
   result.s = ""
   result.lineOffset = -1
   result.repl = r
-  result.onPrompt = onPrompt
 
 proc llStreamClose*(s: PLLStream) =
   case s.kind
@@ -75,10 +72,10 @@ proc llStreamClose*(s: PLLStream) =
 when not declared(readLineFromStdin):
   # fallback implementation:
   proc readLineFromStdin(prompt: string, line: var string): bool =
-    stdout.write(prompt)
+    stderr.write(prompt)
     result = readLine(stdin, line)
     if not result:
-      stdout.write("\n")
+      stderr.write("\n")
       quit(0)
 
 proc endsWith*(x: string, s: set[char]): bool =
@@ -136,7 +133,6 @@ proc llStreamRead*(s: PLLStream, buf: pointer, bufLen: int): int =
   of llsFile:
     result = readBuffer(s.f, buf, bufLen)
   of llsStdIn:
-    if s.onPrompt!=nil: s.onPrompt()
     result = s.repl(s, buf, bufLen)
 
 proc llStreamReadLine*(s: PLLStream, line: var string): bool =
@@ -147,11 +143,11 @@ proc llStreamReadLine*(s: PLLStream, line: var string): bool =
   of llsString:
     while s.rd < s.s.len:
       case s.s[s.rd]
-      of '\r':
+      of '\x0D':
         inc(s.rd)
-        if s.s[s.rd] == '\n': inc(s.rd)
+        if s.s[s.rd] == '\x0A': inc(s.rd)
         break
-      of '\n':
+      of '\x0A':
         inc(s.rd)
         break
       else:

compiler/lookups.nim

@@ -8,10 +8,10 @@
 #
 
 # This module implements lookup helpers.
-import std/[algorithm, strutils, tables]
+
 import
   intsets, ast, astalgo, idents, semdata, types, msgs, options,
-  renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs, wordrecg
+  renderer, nimfix/prettybase, lineinfos, strutils
 
 proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope)
 
@@ -44,11 +44,6 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
         case x.kind
         of nkIdent: id.add(x.ident.s)
         of nkSym: id.add(x.sym.name.s)
-        of nkSymChoices:
-          if x[0].kind == nkSym:
-            id.add(x[0].sym.name.s)
-          else:
-            handleError(n, origin)
         of nkLiterals - nkFloatLiterals: id.add(x.renderTree)
         else: handleError(n, origin)
       result = getIdent(c.cache, id)
@@ -79,30 +74,24 @@ proc closeScope*(c: PContext) =
   ensureNoMissingOrUnusedSymbols(c, c.currentScope)
   rawCloseScope(c)
 
-iterator allScopes*(scope: PScope): PScope =
+iterator walkScopes*(scope: PScope): PScope =
   var current = scope
   while current != nil:
     yield current
     current = current.parent
 
-iterator localScopesFrom*(c: PContext; scope: PScope): PScope =
-  for s in allScopes(scope):
-    if s == c.topLevelScope: break
-    yield s
-
 proc skipAlias*(s: PSym; n: PNode; conf: ConfigRef): PSym =
   if s == nil or s.kind != skAlias:
     result = s
   else:
     result = s.owner
-    if conf.cmd == cmdNimfix:
+    if conf.cmd == cmdPretty:
       prettybase.replaceDeprecated(conf, n.info, s, result)
     else:
       message(conf, n.info, warnDeprecated, "use " & result.name.s & " instead; " &
               s.name.s & " is deprecated")
 
-proc isShadowScope*(s: PScope): bool {.inline.} =
-  s.parent != nil and s.parent.depthLevel == s.depthLevel
+proc isShadowScope*(s: PScope): bool {.inline.} = s.parent != nil and s.parent.depthLevel == s.depthLevel
 
 proc localSearchInScope*(c: PContext, s: PIdent): PSym =
   var scope = c.currentScope
@@ -112,127 +101,30 @@ proc localSearchInScope*(c: PContext, s: PIdent): PSym =
     scope = scope.parent
     result = strTableGet(scope.symbols, s)
 
-proc initIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule; name: PIdent;
-                   g: ModuleGraph): PSym =
-  result = initModuleIter(ti, g, im.m, name)
-  while result != nil:
-    let b =
-      case im.mode
-      of importAll: true
-      of importSet: result.id in im.imported
-      of importExcept: name.id notin im.exceptSet
-    if b and not containsOrIncl(marked, result.id):
-      return result
-    result = nextModuleIter(ti, g)
-
-proc nextIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule;
-                   g: ModuleGraph): PSym =
-  while true:
-    result = nextModuleIter(ti, g)
-    if result == nil: return nil
-    case im.mode
-    of importAll:
-      if not containsOrIncl(marked, result.id):
-        return result
-    of importSet:
-      if result.id in im.imported and not containsOrIncl(marked, result.id):
-        return result
-    of importExcept:
-      if result.name.id notin im.exceptSet and not containsOrIncl(marked, result.id):
-        return result
-
-iterator symbols(im: ImportedModule; marked: var IntSet; name: PIdent; g: ModuleGraph): PSym =
-  var ti: ModuleIter
-  var candidate = initIdentIter(ti, marked, im, name, g)
-  while candidate != nil:
-    yield candidate
-    candidate = nextIdentIter(ti, marked, im, g)
-
-iterator importedItems*(c: PContext; name: PIdent): PSym =
-  var marked = initIntSet()
-  for im in c.imports.mitems:
-    for s in symbols(im, marked, name, c.graph):
-      yield s
-
-proc allPureEnumFields(c: PContext; name: PIdent): seq[PSym] =
-  var ti: TIdentIter
-  result = @[]
-  var res = initIdentIter(ti, c.pureEnumFields, name)
-  while res != nil:
-    result.add res
-    res = nextIdentIter(ti, c.pureEnumFields)
-
-iterator allSyms*(c: PContext): (PSym, int, bool) =
-  # really iterate over all symbols in all the scopes. This is expensive
-  # and only used by suggest.nim.
-  var isLocal = true
-  var scopeN = 0
-  for scope in allScopes(c.currentScope):
-    if scope == c.topLevelScope: isLocal = false
-    dec scopeN
-    for item in scope.symbols:
-      yield (item, scopeN, isLocal)
-
-  dec scopeN
-  isLocal = false
-  for im in c.imports.mitems:
-    for s in modulegraphs.allSyms(c.graph, im.m):
-      assert s != nil
-      yield (s, scopeN, isLocal)
-
-proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PSym =
-  var marked = initIntSet()
-  var symSet = OverloadableSyms
-  if overloadableEnums notin c.features:
-    symSet.excl skEnumField
-  result = nil
-  block outer:
-    for im in c.imports.mitems:
-      for s in symbols(im, marked, name, c.graph):
-        if result == nil:
-          result = s
-        elif s.kind notin symSet or result.kind notin symSet:
-          ambiguous = true
-          break outer
-
-proc searchInScopes*(c: PContext, s: PIdent; ambiguous: var bool): PSym =
-  for scope in allScopes(c.currentScope):
+proc searchInScopes*(c: PContext, s: PIdent): PSym =
+  for scope in walkScopes(c.currentScope):
     result = strTableGet(scope.symbols, s)
-    if result != nil: return result
-  result = someSymFromImportTable(c, s, ambiguous)
+    if result != nil: return
+  result = nil
 
-proc debugScopes*(c: PContext; limit=0, max = int.high) {.deprecated.} =
+proc debugScopes*(c: PContext; limit=0) {.deprecated.} =
   var i = 0
-  var count = 0
-  for scope in allScopes(c.currentScope):
+  for scope in walkScopes(c.currentScope):
     echo "scope ", i
     for h in 0..high(scope.symbols.data):
       if scope.symbols.data[h] != nil:
-        if count >= max: return
-        echo count, ": ", scope.symbols.data[h].name.s
-        count.inc
-    if i == limit: return
+        echo scope.symbols.data[h].name.s
+    if i == limit: break
     inc i
 
-proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] =
-  result = @[]
-  block outer:
-    for scope in allScopes(c.currentScope):
-      var ti: TIdentIter
-      var candidate = initIdentIter(ti, scope.symbols, s)
-      while candidate != nil:
-        if candidate.kind in filter:
-          result.add candidate
-          # Break here, because further symbols encountered would be shadowed
-          break outer
-        candidate = nextIdentIter(ti, scope.symbols)
-
-  if result.len == 0:
-    var marked = initIntSet()
-    for im in c.imports.mitems:
-      for s in symbols(im, marked, s, c.graph):
-        if s.kind in filter:
-          result.add s
+proc searchInScopes*(c: PContext, s: PIdent, filter: TSymKinds): PSym =
+  for scope in walkScopes(c.currentScope):
+    var ti: TIdentIter
+    var candidate = initIdentIter(ti, scope.symbols, s)
+    while candidate != nil:
+      if candidate.kind in filter: return candidate
+      candidate = nextIdentIter(ti, scope.symbols)
+  result = nil
 
 proc errorSym*(c: PContext, n: PNode): PSym =
   ## creates an error symbol to avoid cascading errors (for IDE support)
@@ -243,12 +135,12 @@ proc errorSym*(c: PContext, n: PNode): PSym =
       considerQuotedIdent(c, m)
     else:
       getIdent(c.cache, "err:" & renderTree(m))
-  result = newSym(skError, ident, nextSymId(c.idgen), getCurrOwner(c), n.info, {})
+  result = newSym(skError, ident, getCurrOwner(c), n.info, {})
   result.typ = errorType(c)
   incl(result.flags, sfDiscardable)
-  # pretend it's from the top level scope to prevent cascading errors:
+  # pretend it's imported from some unknown module to prevent cascading errors:
   if c.config.cmd != cmdInteractive and c.compilesContextId == 0:
-    c.moduleScope.addSym(result)
+    c.importTable.addSym(result)
 
 type
   TOverloadIterMode* = enum
@@ -256,36 +148,31 @@ type
     oimSymChoiceLocalLookup
   TOverloadIter* = object
     it*: TIdentIter
-    mit*: ModuleIter
     m*: PSym
     mode*: TOverloadIterMode
     symChoiceIndex*: int
-    currentScope: PScope
-    importIdx: int
-    marked: IntSet
+    scope*: PScope
+    inSymChoice: IntSet
 
-proc getSymRepr*(conf: ConfigRef; s: PSym, getDeclarationPath = true): string =
+proc getSymRepr*(conf: ConfigRef; s: PSym): string =
   case s.kind
   of routineKinds, skType:
-    result = getProcHeader(conf, s, getDeclarationPath = getDeclarationPath)
+    result = getProcHeader(conf, s)
   else:
-    result = "'$1'" % s.name.s
-    if getDeclarationPath:
-      result.addDeclaredLoc(conf, s)
+    result = s.name.s
 
 proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) =
   # check if all symbols have been used and defined:
   var it: TTabIter
   var s = initTabIter(it, scope.symbols)
   var missingImpls = 0
-  var unusedSyms: seq[tuple[sym: PSym, key: string]]
   while s != nil:
     if sfForward in s.flags and s.kind notin {skType, skModule}:
       # too many 'implementation of X' errors are annoying
       # and slow 'suggest' down:
       if missingImpls == 0:
         localError(c.config, s.info, "implementation of '$1' expected" %
-            getSymRepr(c.config, s, getDeclarationPath=false))
+            getSymRepr(c.config, s))
       inc missingImpls
     elif {sfUsed, sfExported} * s.flags == {}:
       if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam, skEnumField}:
@@ -293,75 +180,45 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) =
         # maybe they can be made skGenericParam as well.
         if s.typ != nil and tfImplicitTypeParam notin s.typ.flags and
            s.typ.kind != tyGenericParam:
-          unusedSyms.add (s, toFileLineCol(c.config, s.info))
+          message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s)
     s = nextIter(it, scope.symbols)
-  for (s, _) in sortedByIt(unusedSyms, it.key):
-    message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s)
 
 proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string;
-                        conflictsWith: TLineInfo, note = errGenerated) =
-  ## Emit a redefinition error if in non-interactive mode
+                        conflictsWith: TLineInfo) =
   if c.config.cmd != cmdInteractive:
-    localError(c.config, info, note,
+    localError(c.config, info,
       "redefinition of '$1'; previous declaration here: $2" %
       [s, c.config $ conflictsWith])
 
-# xxx pending bootstrap >= 1.4, replace all those overloads with a single one:
-# proc addDecl*(c: PContext, sym: PSym, info = sym.info, scope = c.currentScope) {.inline.} =
-proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) =
-  let conflict = scope.addUniqueSym(sym)
+proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) =
+  let conflict = c.currentScope.addUniqueSym(sym)
   if conflict != nil:
-    if sym.kind == skModule and conflict.kind == skModule:
-      # e.g.: import foo; import foo
-      # xxx we could refine this by issuing a different hint for the case
-      # where a duplicate import happens inside an include.
-      if c.importModuleMap[sym.id] == c.importModuleMap[conflict.id]:
-        #only hints if the conflict is the actual module not just a shared name
-        localError(c.config, info, hintDuplicateModuleImport,
-          "duplicate import of '$1'; previous import here: $2" %
-          [sym.name.s, c.config $ conflict.info])
-    else:
-      wrongRedefinition(c, info, sym.name.s, conflict.info, errGenerated)
+    wrongRedefinition(c, info, sym.name.s, conflict.info)
 
-proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) {.inline.} =
-  addDeclAt(c, scope, sym, sym.info)
-
-proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) {.inline.} =
-  addDeclAt(c, c.currentScope, sym, info)
-
-proc addDecl*(c: PContext, sym: PSym) {.inline.} =
-  addDeclAt(c, c.currentScope, sym)
+proc addDecl*(c: PContext, sym: PSym) =
+  let conflict = strTableInclReportConflict(c.currentScope.symbols, sym, true)
+  if conflict != nil:
+    wrongRedefinition(c, sym.info, sym.name.s, conflict.info)
 
 proc addPrelimDecl*(c: PContext, sym: PSym) =
   discard c.currentScope.addUniqueSym(sym)
 
-from ic / ic import addHidden
+proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) =
+  let conflict = scope.addUniqueSym(sym)
+  if conflict != nil:
+    wrongRedefinition(c, sym.info, sym.name.s, conflict.info)
 
 proc addInterfaceDeclAux(c: PContext, sym: PSym) =
-  ## adds symbol to the module for either private or public access.
   if sfExported in sym.flags:
     # add to interface:
-    if c.module != nil: exportSym(c, sym)
+    if c.module != nil: strTableAdd(c.module.tab, sym)
     else: internalError(c.config, sym.info, "addInterfaceDeclAux")
-  elif sym.kind in ExportableSymKinds and c.module != nil and isTopLevelInsideDeclaration(c, sym):
-    strTableAdd(semtabAll(c.graph, c.module), sym)
-    if c.config.symbolFiles != disabledSf:
-      addHidden(c.encoder, c.packedRepr, sym)
 
 proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) =
-  ## adds a symbol on the scope and the interface if appropriate
   addDeclAt(c, scope, sym)
-  if not scope.isShadowScope:
-    # adding into a non-shadow scope, we need to handle exports, etc
-    addInterfaceDeclAux(c, sym)
-
-proc addInterfaceDecl*(c: PContext, sym: PSym) {.inline.} =
-  ## adds a decl and the interface if appropriate
-  addInterfaceDeclAt(c, c.currentScope, sym)
+  addInterfaceDeclAux(c, sym)
 
 proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) =
-  ## adds an symbol to the given scope, will check for and raise errors if it's
-  ## a redefinition as opposed to an overload.
   if fn.kind notin OverloadableSyms:
     internalError(c.config, fn.info, "addOverloadableSymAt")
     return
@@ -371,33 +228,25 @@ proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) =
   else:
     scope.addSym(fn)
 
+proc addInterfaceDecl*(c: PContext, sym: PSym) =
+  # it adds the symbol to the interface if appropriate
+  addDecl(c, sym)
+  addInterfaceDeclAux(c, sym)
+
 proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) =
-  ## adds an overloadable symbol on the scope and the interface if appropriate
+  # it adds the symbol to the interface if appropriate
   addOverloadableSymAt(c, scope, sym)
-  if not scope.isShadowScope:
-    # adding into a non-shadow scope, we need to handle exports, etc
-    addInterfaceDeclAux(c, sym)
+  addInterfaceDeclAux(c, sym)
 
 proc openShadowScope*(c: PContext) =
-  ## opens a shadow scope, just like any other scope except the depth is the
-  ## same as the parent -- see `isShadowScope`.
   c.currentScope = PScope(parent: c.currentScope,
                           symbols: newStrTable(),
                           depthLevel: c.scopeDepth)
 
 proc closeShadowScope*(c: PContext) =
-  ## closes the shadow scope, but doesn't merge any of the symbols
-  ## Does not check for unused symbols or missing forward decls since a macro
-  ## or template consumes this AST
-  rawCloseScope(c)
+  c.closeScope
 
 proc mergeShadowScope*(c: PContext) =
-  ## close the existing scope and merge in all defined symbols, this will also
-  ## trigger any export related code if this is into a non-shadow scope.
-  ##
-  ## Merges:
-  ## shadow -> shadow: add symbols to the parent but check for redefinitions etc
-  ## shadow -> non-shadow: the above, but also handle exports and all that
   let shadowScope = c.currentScope
   c.rawCloseScope
   for sym in shadowScope.symbols:
@@ -406,113 +255,39 @@ proc mergeShadowScope*(c: PContext) =
     else:
       c.addInterfaceDecl(sym)
 
-when false:
-  # `nimfix` used to call `altSpelling` and prettybase.replaceDeprecated(n.info, ident, alt)
-  proc altSpelling(c: PContext, x: PIdent): PIdent =
+when defined(nimfix):
+  # when we cannot find the identifier, retry with a changed identifier:
+  proc altSpelling(x: PIdent): PIdent =
     case x.s[0]
-    of 'A'..'Z': result = getIdent(c.cache, toLowerAscii(x.s[0]) & x.s.substr(1))
-    of 'a'..'z': result = getIdent(c.cache, toLowerAscii(x.s[0]) & x.s.substr(1))
+    of 'A'..'Z': result = getIdent(toLowerAscii(x.s[0]) & x.s.substr(1))
+    of 'a'..'z': result = getIdent(toLowerAscii(x.s[0]) & x.s.substr(1))
     else: result = x
 
-import std/editdistance, heapqueue
+  template fixSpelling(n: PNode; ident: PIdent; op: untyped) =
+    let alt = ident.altSpelling
+    result = op(c, alt).skipAlias(n)
+    if result != nil:
+      prettybase.replaceDeprecated(n.info, ident, alt)
+      return result
+else:
+  template fixSpelling(n: PNode; ident: PIdent; op: untyped) = discard
 
-type SpellCandidate = object
-  dist: int
-  depth: int
-  msg: string
-  sym: PSym
-
-template toOrderTup(a: SpellCandidate): (int, int, string) =
-  # `dist` is first, to favor nearby matches
-  # `depth` is next, to favor nearby enclosing scopes among ties
-  # `sym.name.s` is last, to make the list ordered and deterministic among ties
-  (a.dist, a.depth, a.msg)
-
-proc `<`(a, b: SpellCandidate): bool =
-  a.toOrderTup < b.toOrderTup
-
-proc mustFixSpelling(c: PContext): bool {.inline.} =
-  result = c.config.spellSuggestMax != 0 and c.compilesContextId == 0
-    # don't slowdown inside compiles()
-
-proc fixSpelling(c: PContext, n: PNode, ident: PIdent, result: var string) =
-  ## when we cannot find the identifier, suggest nearby spellings
-  var list = initHeapQueue[SpellCandidate]()
-  let name0 = ident.s.nimIdentNormalize
-
-  for (sym, depth, isLocal) in allSyms(c):
-    let depth = -depth - 1
-    let dist = editDistance(name0, sym.name.s.nimIdentNormalize)
-    var msg: string
-    msg.add "\n ($1, $2): '$3'" % [$dist, $depth, sym.name.s]
-    addDeclaredLoc(msg, c.config, sym) # `msg` needed for deterministic ordering.
-    list.push SpellCandidate(dist: dist, depth: depth, msg: msg, sym: sym)
-
-  if list.len == 0: return
-  let e0 = list[0]
-  var count = 0
-  while true:
-    # pending https://github.com/timotheecour/Nim/issues/373 use more efficient `itemsSorted`.
-    if list.len == 0: break
-    let e = list.pop()
-    if c.config.spellSuggestMax == spellSuggestSecretSauce:
-      const
-        smallThres = 2
-        maxCountForSmall = 4
-        # avoids ton of operator matches when mis-matching short symbols such as `i`
-        # other heuristics could be devised, such as only suggesting operators if `name0`
-        # is an operator (likewise with non-operators).
-      if e.dist > e0.dist or (name0.len <= smallThres and count >= maxCountForSmall): break
-    elif count >= c.config.spellSuggestMax: break
-    if count == 0:
-      result.add "\ncandidates (edit distance, scope distance); see '--spellSuggest': "
-    result.add e.msg
-    count.inc
-
-proc errorUseQualifier(c: PContext; info: TLineInfo; s: PSym; amb: var bool): PSym =
+proc errorUseQualifier*(c: PContext; info: TLineInfo; s: PSym) =
   var err = "ambiguous identifier: '" & s.name.s & "'"
+  var ti: TIdentIter
+  var candidate = initIdentIter(ti, c.importTable.symbols, s.name)
   var i = 0
-  var ignoredModules = 0
-  for candidate in importedItems(c, s.name):
+  while candidate != nil:
     if i == 0: err.add " -- use one of the following:\n"
     else: err.add "\n"
     err.add "  " & candidate.owner.name.s & "." & candidate.name.s
     err.add ": " & typeToString(candidate.typ)
-    if candidate.kind == skModule:
-      inc ignoredModules
-    else:
-      result = candidate
-    inc i
-  if ignoredModules != i-1:
-    localError(c.config, info, errGenerated, err)
-    result = nil
-  else:
-    amb = false
-
-proc errorUseQualifier*(c: PContext; info: TLineInfo; s: PSym) =
-  var amb: bool
-  discard errorUseQualifier(c, info, s, amb)
-
-proc errorUseQualifier(c: PContext; info: TLineInfo; candidates: seq[PSym]; prefix = "use one of") =
-  var err = "ambiguous identifier: '" & candidates[0].name.s & "'"
-  var i = 0
-  for candidate in candidates:
-    if i == 0: err.add " -- $1 the following:\n" % prefix
-    else: err.add "\n"
-    err.add "  " & candidate.owner.name.s & "." & candidate.name.s
-    err.add ": " & typeToString(candidate.typ)
+    candidate = nextIdentIter(ti, c.importTable.symbols)
     inc i
   localError(c.config, info, errGenerated, err)
 
-proc errorUseQualifier*(c: PContext; info:TLineInfo; choices: PNode) =
-  var candidates = newSeq[PSym](choices.len)
-  let prefix = if choices[0].typ.kind != tyProc: "use one of" else: "you need a helper proc to disambiguate"
-  for i, n in choices:
-    candidates[i] = n.sym
-  errorUseQualifier(c, info, candidates, prefix)
-
-proc errorUndeclaredIdentifier*(c: PContext; info: TLineInfo; name: string, extra = "") =
-  var err = "undeclared identifier: '" & name & "'" & extra
+proc errorUndeclaredIdentifier*(c: PContext; info: TLineInfo; name: string) =
+  var err = "undeclared identifier: '" & name & "'"
   if c.recursiveDep.len > 0:
     err.add "\nThis might be caused by a recursive module dependency:\n"
     err.add c.recursiveDep
@@ -520,31 +295,29 @@ proc errorUndeclaredIdentifier*(c: PContext; info: TLineInfo; name: string, extr
     c.recursiveDep = ""
   localError(c.config, info, errGenerated, err)
 
-proc errorUndeclaredIdentifierHint*(c: PContext; n: PNode, ident: PIdent): PSym =
-  var extra = ""
-  if c.mustFixSpelling: fixSpelling(c, n, ident, extra)
-  errorUndeclaredIdentifier(c, n.info, ident.s, extra)
-  result = errorSym(c, n)
-
 proc lookUp*(c: PContext, n: PNode): PSym =
   # Looks up a symbol. Generates an error in case of nil.
-  var amb = false
   case n.kind
   of nkIdent:
-    result = searchInScopes(c, n.ident, amb).skipAlias(n, c.config)
-    if result == nil: result = errorUndeclaredIdentifierHint(c, n, n.ident)
+    result = searchInScopes(c, n.ident).skipAlias(n, c.config)
+    if result == nil:
+      fixSpelling(n, n.ident, searchInScopes)
+      errorUndeclaredIdentifier(c, n.info, n.ident.s)
+      result = errorSym(c, n)
   of nkSym:
     result = n.sym
   of nkAccQuoted:
     var ident = considerQuotedIdent(c, n)
-    result = searchInScopes(c, ident, amb).skipAlias(n, c.config)
-    if result == nil: result = errorUndeclaredIdentifierHint(c, n, ident)
+    result = searchInScopes(c, ident).skipAlias(n, c.config)
+    if result == nil:
+      fixSpelling(n, ident, searchInScopes)
+      errorUndeclaredIdentifier(c, n.info, ident.s)
+      result = errorSym(c, n)
   else:
     internalError(c.config, n.info, "lookUp")
     return
-  if amb:
-    #contains(c.ambiguousSymbols, result.id):
-    result = errorUseQualifier(c, n.info, result, amb)
+  if contains(c.ambiguousSymbols, result.id):
+    errorUseQualifier(c, n.info, result)
   when false:
     if result.kind == skStub: loadStub(result)
 
@@ -553,38 +326,29 @@ type
     checkAmbiguity, checkUndeclared, checkModule, checkPureEnumFields
 
 proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
-  const allExceptModule = {low(TSymKind)..high(TSymKind)} - {skModule, skPackage}
+  const allExceptModule = {low(TSymKind)..high(TSymKind)}-{skModule,skPackage}
   case n.kind
   of nkIdent, nkAccQuoted:
-    var amb = false
     var ident = considerQuotedIdent(c, n)
     if checkModule in flags:
-      result = searchInScopes(c, ident, amb).skipAlias(n, c.config)
+      result = searchInScopes(c, ident).skipAlias(n, c.config)
     else:
-      let candidates = searchInScopesFilterBy(c, ident, allExceptModule) #.skipAlias(n, c.config)
-      if candidates.len > 0:
-        result = candidates[0]
-        amb = candidates.len > 1
-        if amb and checkAmbiguity in flags:
-          errorUseQualifier(c, n.info, candidates)
-    if result == nil:
-      let candidates = allPureEnumFields(c, ident)
-      if candidates.len > 0:
-        result = candidates[0]
-        amb = candidates.len > 1
-        if amb and checkAmbiguity in flags:
-          errorUseQualifier(c, n.info, candidates)
-
+      result = searchInScopes(c, ident, allExceptModule).skipAlias(n, c.config)
+    if result == nil and checkPureEnumFields in flags:
+      result = strTableGet(c.pureEnumFields, ident)
     if result == nil and checkUndeclared in flags:
-      result = errorUndeclaredIdentifierHint(c, n, ident)
-    elif checkAmbiguity in flags and result != nil and amb:
-      result = errorUseQualifier(c, n.info, result, amb)
-    c.isAmbiguous = amb
+      fixSpelling(n, ident, searchInScopes)
+      errorUndeclaredIdentifier(c, n.info, ident.s)
+      result = errorSym(c, n)
+    elif checkAmbiguity in flags and result != nil and result.id in c.ambiguousSymbols:
+      errorUseQualifier(c, n.info, result)
   of nkSym:
     result = n.sym
+    if checkAmbiguity in flags and contains(c.ambiguousSymbols, result.id):
+      errorUseQualifier(c, n.info, n.sym)
   of nkDotExpr:
     result = nil
-    var m = qualifiedLookUp(c, n[0], (flags * {checkUndeclared}) + {checkModule})
+    var m = qualifiedLookUp(c, n[0], (flags*{checkUndeclared})+{checkModule})
     if m != nil and m.kind == skModule:
       var ident: PIdent = nil
       if n[1].kind == nkIdent:
@@ -595,18 +359,13 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
         if m == c.module:
           result = strTableGet(c.topLevelScope.symbols, ident).skipAlias(n, c.config)
         else:
-          if c.importModuleLookup.getOrDefault(m.name.id).len > 1:
-            var amb: bool
-            result = errorUseQualifier(c, n.info, m, amb)
-          else:
-            result = someSym(c.graph, m, ident).skipAlias(n, c.config)
+          result = strTableGet(m.tab, ident).skipAlias(n, c.config)
         if result == nil and checkUndeclared in flags:
-          result = errorUndeclaredIdentifierHint(c, n[1], ident)
+          fixSpelling(n[1], ident, searchInScopes)
+          errorUndeclaredIdentifier(c, n[1].info, ident.s)
+          result = errorSym(c, n[1])
       elif n[1].kind == nkSym:
         result = n[1].sym
-        if result.owner != nil and result.owner != m and checkUndeclared in flags:
-          # dotExpr in templates can end up here
-          result = errorUndeclaredIdentifierHint(c, n[1], considerQuotedIdent(c, n[1]))
       elif checkUndeclared in flags and
            n[1].kind notin {nkOpenSymChoice, nkClosedSymChoice}:
         localError(c.config, n[1].info, "identifier expected, but got: " &
@@ -618,29 +377,18 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
     if result != nil and result.kind == skStub: loadStub(result)
 
 proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
-  o.importIdx = -1
-  o.marked = initIntSet()
   case n.kind
   of nkIdent, nkAccQuoted:
     var ident = considerQuotedIdent(c, n)
-    var scope = c.currentScope
+    o.scope = c.currentScope
     o.mode = oimNoQualifier
     while true:
-      result = initIdentIter(o.it, scope.symbols, ident).skipAlias(n, c.config)
+      result = initIdentIter(o.it, o.scope.symbols, ident).skipAlias(n, c.config)
       if result != nil:
-        o.currentScope = scope
         break
       else:
-        scope = scope.parent
-        if scope == nil:
-          for i in 0..c.imports.high:
-            result = initIdentIter(o.mit, o.marked, c.imports[i], ident, c.graph).skipAlias(n, c.config)
-            if result != nil:
-              o.currentScope = nil
-              o.importIdx = i
-              return result
-          return nil
-
+        o.scope = o.scope.parent
+        if o.scope == nil: break
   of nkSym:
     result = n.sym
     o.mode = oimDone
@@ -660,7 +408,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
                                  ident).skipAlias(n, c.config)
           o.mode = oimSelfModule
         else:
-          result = initModuleIter(o.mit, c.graph, o.m, ident).skipAlias(n, c.config)
+          result = initIdentIter(o.it, o.m.tab, ident).skipAlias(n, c.config)
       else:
         noidentError(c.config, n[1], n)
         result = errorSym(c, n[1])
@@ -672,120 +420,60 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
       o.mode = oimDone
       return nil
     o.symChoiceIndex = 1
-    o.marked = initIntSet()
-    incl(o.marked, result.id)
+    o.inSymChoice = initIntSet()
+    incl(o.inSymChoice, result.id)
   else: discard
   when false:
     if result != nil and result.kind == skStub: loadStub(result)
 
 proc lastOverloadScope*(o: TOverloadIter): int =
   case o.mode
-  of oimNoQualifier:
-    result = if o.importIdx >= 0: 0
-             elif o.currentScope.isNil: -1
-             else: o.currentScope.depthLevel
+  of oimNoQualifier: result = if o.scope.isNil: -1 else: o.scope.depthLevel
   of oimSelfModule:  result = 1
   of oimOtherModule: result = 0
   else: result = -1
 
-proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym =
-  assert o.currentScope == nil
-  var idx = o.importIdx+1
-  o.importIdx = c.imports.len # assume the other imported modules lack this symbol too
-  while idx < c.imports.len:
-    result = initIdentIter(o.mit, o.marked, c.imports[idx], o.it.name, c.graph).skipAlias(n, c.config)
-    if result != nil:
-      # oh, we were wrong, some other module had the symbol, so remember that:
-      o.importIdx = idx
-      break
-    inc idx
-
-proc symChoiceExtension(o: var TOverloadIter; c: PContext; n: PNode): PSym =
-  assert o.currentScope == nil
-  while o.importIdx < c.imports.len:
-    result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph).skipAlias(n, c.config)
-    #while result != nil and result.id in o.marked:
-    #  result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx])
-    if result != nil:
-      #assert result.id notin o.marked
-      return result
-    inc o.importIdx
-
 proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
   case o.mode
   of oimDone:
     result = nil
   of oimNoQualifier:
-    if o.currentScope != nil:
-      assert o.importIdx < 0
-      result = nextIdentIter(o.it, o.currentScope.symbols).skipAlias(n, c.config)
+    if o.scope != nil:
+      result = nextIdentIter(o.it, o.scope.symbols).skipAlias(n, c.config)
       while result == nil:
-        o.currentScope = o.currentScope.parent
-        if o.currentScope != nil:
-          result = initIdentIter(o.it, o.currentScope.symbols, o.it.name).skipAlias(n, c.config)
-          # BUGFIX: o.it.name <-> n.ident
-        else:
-          o.importIdx = 0
-          if c.imports.len > 0:
-            result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph).skipAlias(n, c.config)
-            if result == nil:
-              result = nextOverloadIterImports(o, c, n)
-          break
-    elif o.importIdx < c.imports.len:
-      result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph).skipAlias(n, c.config)
-      if result == nil:
-        result = nextOverloadIterImports(o, c, n)
+        o.scope = o.scope.parent
+        if o.scope == nil: break
+        result = initIdentIter(o.it, o.scope.symbols, o.it.name).skipAlias(n, c.config)
+        # BUGFIX: o.it.name <-> n.ident
     else:
       result = nil
   of oimSelfModule:
     result = nextIdentIter(o.it, c.topLevelScope.symbols).skipAlias(n, c.config)
   of oimOtherModule:
-    result = nextModuleIter(o.mit, c.graph).skipAlias(n, c.config)
+    result = nextIdentIter(o.it, o.m.tab).skipAlias(n, c.config)
   of oimSymChoice:
     if o.symChoiceIndex < n.len:
       result = n[o.symChoiceIndex].sym
-      incl(o.marked, result.id)
+      incl(o.inSymChoice, result.id)
       inc o.symChoiceIndex
     elif n.kind == nkOpenSymChoice:
       # try 'local' symbols too for Koenig's lookup:
       o.mode = oimSymChoiceLocalLookup
-      o.currentScope = c.currentScope
-      result = firstIdentExcluding(o.it, o.currentScope.symbols,
-                                   n[0].sym.name, o.marked).skipAlias(n, c.config)
+      o.scope = c.currentScope
+      result = firstIdentExcluding(o.it, o.scope.symbols,
+                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
       while result == nil:
-        o.currentScope = o.currentScope.parent
-        if o.currentScope != nil:
-          result = firstIdentExcluding(o.it, o.currentScope.symbols,
-                                      n[0].sym.name, o.marked).skipAlias(n, c.config)
-        else:
-          o.importIdx = 0
-          result = symChoiceExtension(o, c, n)
-          break
-      if result != nil:
-        incl o.marked, result.id
+        o.scope = o.scope.parent
+        if o.scope == nil: break
+        result = firstIdentExcluding(o.it, o.scope.symbols,
+                                     n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
   of oimSymChoiceLocalLookup:
-    if o.currentScope != nil:
-      result = nextIdentExcluding(o.it, o.currentScope.symbols, o.marked).skipAlias(n, c.config)
-      while result == nil:
-        o.currentScope = o.currentScope.parent
-        if o.currentScope != nil:
-          result = firstIdentExcluding(o.it, o.currentScope.symbols,
-                                      n[0].sym.name, o.marked).skipAlias(n, c.config)
-        else:
-          o.importIdx = 0
-          result = symChoiceExtension(o, c, n)
-          break
-      if result != nil:
-        incl o.marked, result.id
-
-    elif o.importIdx < c.imports.len:
-      result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph).skipAlias(n, c.config)
-      #assert result.id notin o.marked
-      #while result != nil and result.id in o.marked:
-      #  result = nextIdentIter(o.it, c.imports[o.importIdx]).skipAlias(n, c.config)
-      if result == nil:
-        inc o.importIdx
-        result = symChoiceExtension(o, c, n)
+    result = nextIdentExcluding(o.it, o.scope.symbols, o.inSymChoice).skipAlias(n, c.config)
+    while result == nil:
+      o.scope = o.scope.parent
+      if o.scope == nil: break
+      result = firstIdentExcluding(o.it, o.scope.symbols,
+                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
 
   when false:
     if result != nil and result.kind == skStub: loadStub(result)

compiler/lowerings.nim

@@ -66,25 +66,18 @@ proc newFastMoveStmt*(g: ModuleGraph, le, ri: PNode): PNode =
   result[1].add newSymNode(getSysMagic(g, ri.info, "move", mMove))
   result[1].add ri
 
-proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNode =
+proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   assert n.kind == nkVarTuple
   let value = n.lastSon
   result = newNodeI(nkStmtList, n.info)
 
-  var tempAsNode: PNode
-  let avoidTemp = value.kind == nkSym
-  if avoidTemp:
-    tempAsNode = value
-  else:
-    var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen),
-                  owner, value.info, g.config.options)
-    temp.typ = skipTypes(value.typ, abstractInst)
-    incl(temp.flags, sfFromGeneric)
-    tempAsNode = newSymNode(temp)
+  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), owner, value.info, g.config.options)
+  temp.typ = skipTypes(value.typ, abstractInst)
+  incl(temp.flags, sfFromGeneric)
 
   var v = newNodeI(nkVarSection, value.info)
-  if not avoidTemp:
-    v.addVar(tempAsNode, value)
+  let tempAsNode = newSymNode(temp)
+  v.addVar(tempAsNode, value)
   result.add(v)
 
   for i in 0..<n.len-2:
@@ -92,13 +85,12 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: P
     if n[i].kind == nkSym: v.addVar(n[i], val)
     else: result.add newAsgnStmt(n[i], val)
 
-proc evalOnce*(g: ModuleGraph; value: PNode; idgen: IdGenerator; owner: PSym): PNode =
+proc evalOnce*(g: ModuleGraph; value: PNode; owner: PSym): PNode =
   ## Turns (value) into (let tmp = value; tmp) so that 'value' can be re-used
   ## freely, multiple times. This is frequently required and such a builtin would also be
   ## handy to have in macros.nim. The value that can be reused is 'result.lastSon'!
   result = newNodeIT(nkStmtListExpr, value.info, value.typ)
-  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen),
-                    owner, value.info, g.config.options)
+  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), owner, value.info, g.config.options)
   temp.typ = skipTypes(value.typ, abstractInst)
   incl(temp.flags, sfFromGeneric)
 
@@ -119,17 +111,17 @@ proc newTupleAccessRaw*(tup: PNode, i: int): PNode =
 proc newTryFinally*(body, final: PNode): PNode =
   result = newTree(nkHiddenTryStmt, body, newTree(nkFinally, final))
 
-proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNode =
+proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   let value = n.lastSon
   result = newNodeI(nkStmtList, n.info)
 
-  var temp = newSym(skTemp, getIdent(g.cache, "_"), nextSymId(idgen), owner, value.info, owner.options)
+  var temp = newSym(skTemp, getIdent(g.cache, "_"), owner, value.info, owner.options)
   var v = newNodeI(nkLetSection, value.info)
   let tempAsNode = newSymNode(temp) #newIdentNode(getIdent(genPrefix & $temp.id), value.info)
 
   var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
   vpart[0] = tempAsNode
-  vpart[1] = newNodeI(nkTupleClassTy, value.info)
+  vpart[1] = newNodeI(nkEmpty, value.info)
   vpart[2] = value
   v.add vpart
   result.add(v)
@@ -138,10 +130,10 @@ proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; idgen: IdGenerator; o
   for i in 0..<lhs.len:
     result.add newAsgnStmt(lhs[i], newTupleAccessRaw(tempAsNode, i))
 
-proc lowerSwap*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNode =
+proc lowerSwap*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   result = newNodeI(nkStmtList, n.info)
   # note: cannot use 'skTemp' here cause we really need the copy for the VM :-(
-  var temp = newSym(skVar, getIdent(g.cache, genPrefix), nextSymId(idgen), owner, n.info, owner.options)
+  var temp = newSym(skVar, getIdent(g.cache, genPrefix), owner, n.info, owner.options)
   temp.typ = n[1].typ
   incl(temp.flags, sfFromGeneric)
   incl(temp.flags, sfGenSym)
@@ -159,8 +151,8 @@ proc lowerSwap*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNod
   result.add newFastAsgnStmt(n[1], n[2])
   result.add newFastAsgnStmt(n[2], tempAsNode)
 
-proc createObj*(g: ModuleGraph; idgen: IdGenerator; owner: PSym, info: TLineInfo; final=true): PType =
-  result = newType(tyObject, nextTypeId(idgen), owner)
+proc createObj*(g: ModuleGraph; owner: PSym, info: TLineInfo; final=true): PType =
+  result = newType(tyObject, owner)
   if final:
     rawAddSon(result, nil)
     incl result.flags, tfFinal
@@ -168,7 +160,6 @@ proc createObj*(g: ModuleGraph; idgen: IdGenerator; owner: PSym, info: TLineInfo
     rawAddSon(result, getCompilerProc(g, "RootObj").typ)
   result.n = newNodeI(nkRecList, info)
   let s = newSym(skType, getIdent(g.cache, "Env_" & toFilename(g.config, info) & "_" & $owner.name.s),
-                  nextSymId(idgen),
                   owner, info, owner.options)
   incl s.flags, sfAnon
   s.typ = result
@@ -206,7 +197,7 @@ proc rawDirectAccess*(obj, field: PSym): PNode =
   result.add newSymNode(field)
   result.typ = field.typ
 
-proc lookupInRecord(n: PNode, id: ItemId): PSym =
+proc lookupInRecord(n: PNode, id: int): PSym =
   result = nil
   case n.kind
   of nkRecList:
@@ -224,36 +215,31 @@ proc lookupInRecord(n: PNode, id: ItemId): PSym =
         if result != nil: return
       else: discard
   of nkSym:
-    if n.sym.itemId.module == id.module and n.sym.itemId.item == -abs(id.item): result = n.sym
+    if n.sym.id == -abs(id): result = n.sym
   else: discard
 
-proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym =
+proc addField*(obj: PType; s: PSym; cache: IdentCache) =
   # because of 'gensym' support, we have to mangle the name with its ID.
   # This is hacky but the clean solution is much more complex than it looks.
-  var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len),
-                     nextSymId(idgen), s.owner, s.info, s.options)
-  field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
-  let t = skipIntLit(s.typ, idgen)
+  var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), s.owner, s.info,
+                     s.options)
+  field.id = -s.id
+  let t = skipIntLit(s.typ)
   field.typ = t
-  if s.kind in {skLet, skVar, skField, skForVar}:
-    #field.bitsize = s.bitsize
-    field.alignment = s.alignment
   assert t.kind != tyTyped
   propagateToOwner(obj, t)
   field.position = obj.n.len
-  # sfNoInit flag for skField is used in closureiterator codegen
-  field.flags = s.flags * {sfCursor, sfNoInit}
+  field.flags = s.flags * {sfCursor}
   obj.n.add newSymNode(field)
   fieldCheck()
-  result = field
 
-proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym {.discardable.} =
-  result = lookupInRecord(obj.n, s.itemId)
+proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache): PSym {.discardable.} =
+  result = lookupInRecord(obj.n, s.id)
   if result == nil:
-    var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), nextSymId(idgen),
-                       s.owner, s.info, s.options)
-    field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
-    let t = skipIntLit(s.typ, idgen)
+    var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), s.owner, s.info,
+                       s.options)
+    field.id = -s.id
+    let t = skipIntLit(s.typ)
     field.typ = t
     assert t.kind != tyTyped
     propagateToOwner(obj, t)
@@ -263,13 +249,13 @@ proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator)
 
 proc newDotExpr*(obj, b: PSym): PNode =
   result = newNodeI(nkDotExpr, obj.info)
-  let field = lookupInRecord(obj.typ.n, b.itemId)
+  let field = lookupInRecord(obj.typ.n, b.id)
   assert field != nil, b.name.s
   result.add newSymNode(obj)
   result.add newSymNode(field)
   result.typ = field.typ
 
-proc indirectAccess*(a: PNode, b: ItemId, info: TLineInfo): PNode =
+proc indirectAccess*(a: PNode, b: int, info: TLineInfo): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
   deref.typ = a.typ.skipTypes(abstractInst)[0]
@@ -321,7 +307,7 @@ proc getFieldFromObj*(t: PType; v: PSym): PSym =
   var t = t
   while true:
     assert t.kind == tyObject
-    result = lookupInRecord(t.n, v.itemId)
+    result = lookupInRecord(t.n, v.id)
     if result != nil: break
     t = t[0]
     if t == nil: break
@@ -329,15 +315,15 @@ proc getFieldFromObj*(t: PType; v: PSym): PSym =
 
 proc indirectAccess*(a: PNode, b: PSym, info: TLineInfo): PNode =
   # returns a[].b as a node
-  result = indirectAccess(a, b.itemId, info)
+  result = indirectAccess(a, b.id, info)
 
 proc indirectAccess*(a, b: PSym, info: TLineInfo): PNode =
   result = indirectAccess(newSymNode(a), b, info)
 
-proc genAddrOf*(n: PNode; idgen: IdGenerator; typeKind = tyPtr): PNode =
+proc genAddrOf*(n: PNode, typeKind = tyPtr): PNode =
   result = newNodeI(nkAddr, n.info, 1)
   result[0] = n
-  result.typ = newType(typeKind, nextTypeId(idgen), n.typ.owner)
+  result.typ = newType(typeKind, n.typ.owner)
   result.typ.rawAddSon(n.typ)
 
 proc genDeref*(n: PNode; k = nkHiddenDeref): PNode =

compiler/macrocacheimpl.nim

@@ -9,36 +9,71 @@
 
 ## This module implements helpers for the macro cache.
 
-import lineinfos, ast, vmdef
-
-proc append(c: PCtx; n: PNode) =
-  c.vmstateDiff.add((c.module, n))
+import lineinfos, ast, modulegraphs, vmdef
 
 proc recordInc*(c: PCtx; info: TLineInfo; key: string; by: BiggestInt) =
-  var recorded = newNodeI(nkReplayAction, info)
+  var recorded = newNodeI(nkCommentStmt, info)
   recorded.add newStrNode("inc", info)
   recorded.add newStrNode(key, info)
   recorded.add newIntNode(nkIntLit, by)
-  c.append(recorded)
+  c.graph.recordStmt(c.graph, c.module, recorded)
 
 proc recordPut*(c: PCtx; info: TLineInfo; key: string; k: string; val: PNode) =
-  var recorded = newNodeI(nkReplayAction, info)
+  var recorded = newNodeI(nkCommentStmt, info)
   recorded.add newStrNode("put", info)
   recorded.add newStrNode(key, info)
   recorded.add newStrNode(k, info)
   recorded.add copyTree(val)
-  c.append(recorded)
+  c.graph.recordStmt(c.graph, c.module, recorded)
 
 proc recordAdd*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
-  var recorded = newNodeI(nkReplayAction, info)
+  var recorded = newNodeI(nkCommentStmt, info)
   recorded.add newStrNode("add", info)
   recorded.add newStrNode(key, info)
   recorded.add copyTree(val)
-  c.append(recorded)
+  c.graph.recordStmt(c.graph, c.module, recorded)
 
 proc recordIncl*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
-  var recorded = newNodeI(nkReplayAction, info)
+  var recorded = newNodeI(nkCommentStmt, info)
   recorded.add newStrNode("incl", info)
   recorded.add newStrNode(key, info)
   recorded.add copyTree(val)
-  c.append(recorded)
+  c.graph.recordStmt(c.graph, c.module, recorded)
+
+when false:
+  proc genCall3(g: ModuleGraph; m: TMagic; s: string; a, b, c: PNode): PNode =
+    newTree(nkStaticStmt, newTree(nkCall, createMagic(g, s, m).newSymNode, a, b, c))
+
+  proc genCall2(g: ModuleGraph; m: TMagic; s: string; a, b: PNode): PNode =
+    newTree(nkStaticStmt, newTree(nkCall, createMagic(g, s, m).newSymNode, a, b))
+
+  template nodeFrom(s: string): PNode =
+    var res = newStrNode(s, info)
+    res.typ = getSysType(g, info, tyString)
+    res
+
+  template nodeFrom(i: BiggestInt): PNode =
+    var res = newIntNode(i, info)
+    res.typ = getSysType(g, info, tyInt)
+    res
+
+  template nodeFrom(n: PNode): PNode = copyTree(n)
+
+  template record(call) =
+    g.recordStmt(g, c.module, call)
+
+  proc recordInc*(c: PCtx; info: TLineInfo; key: string; by: BiggestInt) =
+    let g = c.graph
+    record genCall2(mNccInc, "inc", nodeFrom key, nodeFrom by)
+
+  proc recordPut*(c: PCtx; info: TLineInfo; key: string; k: string; val: PNode) =
+    let g = c.graph
+    record genCall3(mNctPut, "[]=", nodeFrom key, nodeFrom k, nodeFrom val)
+
+  proc recordAdd*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
+    let g = c.graph
+    record genCall2(mNcsAdd, "add", nodeFrom key, nodeFrom val)
+
+  proc recordIncl*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
+    let g = c.graph
+    record genCall2(mNcsIncl, "incl", nodeFrom key, nodeFrom val)

compiler/magicsys.nim

@@ -17,30 +17,36 @@ export createMagic
 
 proc nilOrSysInt*(g: ModuleGraph): PType = g.sysTypes[tyInt]
 
+proc registerSysType*(g: ModuleGraph; t: PType) =
+  if g.sysTypes[t.kind] == nil: g.sysTypes[t.kind] = t
+
 proc newSysType(g: ModuleGraph; kind: TTypeKind, size: int): PType =
-  result = newType(kind, nextTypeId(g.idgen), g.systemModule)
+  result = newType(kind, g.systemModule)
   result.size = size
   result.align = size.int16
 
 proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym =
-  result = systemModuleSym(g, getIdent(g.cache, name))
+  result = strTableGet(g.systemModule.tab, getIdent(g.cache, name))
   if result == nil:
     localError(g.config, info, "system module needs: " & name)
-    result = newSym(skError, getIdent(g.cache, name), nextSymId(g.idgen), g.systemModule, g.systemModule.info, {})
-    result.typ = newType(tyError, nextTypeId(g.idgen), g.systemModule)
+    result = newSym(skError, getIdent(g.cache, name), g.systemModule, g.systemModule.info, {})
+    result.typ = newType(tyError, g.systemModule)
   if result.kind == skAlias: result = result.owner
 
 proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSym =
+  var ti: TIdentIter
   let id = getIdent(g.cache, name)
-  for r in systemModuleSyms(g, id):
+  var r = initIdentIter(ti, g.systemModule.tab, id)
+  while r != nil:
     if r.magic == m:
       # prefer the tyInt variant:
       if r.typ[0] != nil and r.typ[0].kind == tyInt: return r
       result = r
+    r = nextIdentIter(ti, g.systemModule.tab)
   if result != nil: return result
   localError(g.config, info, "system module needs: " & name)
-  result = newSym(skError, id, nextSymId(g.idgen), g.systemModule, g.systemModule.info, {})
-  result.typ = newType(tyError, nextTypeId(g.idgen), g.systemModule)
+  result = newSym(skError, id, g.systemModule, g.systemModule.info, {})
+  result.typ = newType(tyError, g.systemModule)
 
 proc sysTypeFromName*(g: ModuleGraph; info: TLineInfo; name: string): PType =
   result = getSysSym(g, info, name).typ
@@ -50,7 +56,6 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
   result = g.sysTypes[kind]
   if result == nil:
     case kind
-    of tyVoid: result = sysTypeFromName("void")
     of tyInt: result = sysTypeFromName("int")
     of tyInt8: result = sysTypeFromName("int8")
     of tyInt16: result = sysTypeFromName("int16")
@@ -68,7 +73,7 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
     of tyBool: result = sysTypeFromName("bool")
     of tyChar: result = sysTypeFromName("char")
     of tyString: result = sysTypeFromName("string")
-    of tyCstring: result = sysTypeFromName("cstring")
+    of tyCString: result = sysTypeFromName("cstring")
     of tyPointer: result = sysTypeFromName("pointer")
     of tyNil: result = newSysType(g, tyNil, g.config.target.ptrSize)
     else: internalError(g.config, "request for typekind: " & $kind)
@@ -86,28 +91,75 @@ proc resetSysTypes*(g: ModuleGraph) =
   for i in low(g.sysTypes)..high(g.sysTypes):
     g.sysTypes[i] = nil
 
+  for i in low(g.intTypeCache)..high(g.intTypeCache):
+    g.intTypeCache[i] = nil
+
+proc getIntLitType*(g: ModuleGraph; literal: PNode): PType =
+  # we cache some common integer literal types for performance:
+  let value = literal.intVal
+  if value >= low(g.intTypeCache) and value <= high(g.intTypeCache):
+    result = g.intTypeCache[value.int]
+    if result == nil:
+      let ti = getSysType(g, literal.info, tyInt)
+      result = copyType(ti, ti.owner, false)
+      result.n = literal
+      g.intTypeCache[value.int] = result
+  else:
+    let ti = getSysType(g, literal.info, tyInt)
+    result = copyType(ti, ti.owner, false)
+    result.n = literal
+
 proc getFloatLitType*(g: ModuleGraph; literal: PNode): PType =
   # for now we do not cache these:
   result = newSysType(g, tyFloat, size=8)
   result.n = literal
 
-proc skipIntLit*(t: PType; id: IdGenerator): PType {.inline.} =
+proc skipIntLit*(t: PType): PType {.inline.} =
   if t.n != nil and t.kind in {tyInt, tyFloat}:
-    result = copyType(t, nextTypeId(id), t.owner)
+    result = copyType(t, t.owner, false)
     result.n = nil
   else:
     result = t
 
-proc addSonSkipIntLit*(father, son: PType; id: IdGenerator) =
-  let s = son.skipIntLit(id)
+proc addSonSkipIntLit*(father, son: PType) =
+  when not defined(nimNoNilSeqs):
+    if isNil(father.sons): father.sons = @[]
+  let s = son.skipIntLit
   father.sons.add(s)
   propagateToOwner(father, s)
 
+proc setIntLitType*(g: ModuleGraph; result: PNode) =
+  let i = result.intVal
+  case g.config.target.intSize
+  of 8: result.typ = getIntLitType(g, result)
+  of 4:
+    if i >= low(int32) and i <= high(int32):
+      result.typ = getIntLitType(g, result)
+    else:
+      result.typ = getSysType(g, result.info, tyInt64)
+  of 2:
+    if i >= low(int16) and i <= high(int16):
+      result.typ = getIntLitType(g, result)
+    elif i >= low(int32) and i <= high(int32):
+      result.typ = getSysType(g, result.info, tyInt32)
+    else:
+      result.typ = getSysType(g, result.info, tyInt64)
+  of 1:
+    # 8 bit CPUs are insane ...
+    if i >= low(int8) and i <= high(int8):
+      result.typ = getIntLitType(g, result)
+    elif i >= low(int16) and i <= high(int16):
+      result.typ = getSysType(g, result.info, tyInt16)
+    elif i >= low(int32) and i <= high(int32):
+      result.typ = getSysType(g, result.info, tyInt32)
+    else:
+      result.typ = getSysType(g, result.info, tyInt64)
+  else:
+    internalError(g.config, result.info, "invalid int size")
+
 proc getCompilerProc*(g: ModuleGraph; name: string): PSym =
   let ident = getIdent(g.cache, name)
   result = strTableGet(g.compilerprocs, ident)
-  if result == nil:
-    result = loadCompilerProc(g, name)
 
 proc registerCompilerProc*(g: ModuleGraph; s: PSym) =
   strTableAdd(g.compilerprocs, s)
@@ -129,13 +181,13 @@ proc resetNimScriptSymbols*(g: ModuleGraph) = initStrTable(g.exposed)
 proc getMagicEqSymForType*(g: ModuleGraph; t: PType; info: TLineInfo): PSym =
   case t.kind
   of tyInt,  tyInt8, tyInt16, tyInt32, tyInt64,
-     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64:
+     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64: 
     result = getSysMagic(g, info, "==", mEqI)
-  of tyEnum:
+  of tyEnum: 
     result = getSysMagic(g, info, "==", mEqEnum)
-  of tyBool:
+  of tyBool: 
     result = getSysMagic(g, info, "==", mEqB)
-  of tyRef, tyPtr, tyPointer:
+  of tyRef, tyPtr, tyPointer: 
     result = getSysMagic(g, info, "==", mEqRef)
   of tyString:
     result = getSysMagic(g, info, "==", mEqStr)

compiler/main.nim

@@ -13,17 +13,13 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  std/[strutils, os, times, tables, sha1, with, json],
-  llstream, ast, lexer, syntaxes, options, msgs,
-  condsyms,
+  llstream, strutils, os, ast, lexer, syntaxes, options, msgs,
+  condsyms, times,
   sem, idents, passes, extccomp,
-  cgen, nversion,
-  platform, nimconf, passaux, depends, vm,
+  cgen, json, nversion,
+  platform, nimconf, passaux, depends, vm, idgen,
   modules,
-  modulegraphs, lineinfos, pathutils, vmprofiler
-
-import ic / [cbackend, integrity, navigator]
-from ic / ic import rodViewer
+  modulegraphs, tables, rod, lineinfos, pathutils, vmprofiler
 
 when not defined(leanCompiler):
   import jsgen, docgen, docgen2
@@ -35,9 +31,9 @@ proc semanticPasses(g: ModuleGraph) =
 proc writeDepsFile(g: ModuleGraph) =
   let fname = g.config.nimcacheDir / RelativeFile(g.config.projectName & ".deps")
   let f = open(fname.string, fmWrite)
-  for m in g.ifaces:
-    if m.module != nil:
-      f.writeLine(toFullPath(g.config, m.module.position.FileIndex))
+  for m in g.modules:
+    if m != nil:
+      f.writeLine(toFullPath(g.config, m.position.FileIndex))
   for k in g.inclToMod.keys:
     if g.getModule(k).isNil:  # don't repeat includes which are also modules
       f.writeLine(toFullPath(g.config, k))
@@ -55,48 +51,34 @@ proc commandGenDepend(graph: ModuleGraph) =
       ' ' & changeFileExt(project, "dot").string)
 
 proc commandCheck(graph: ModuleGraph) =
-  let conf = graph.config
-  conf.setErrorMaxHighMaybe
-  defineSymbol(conf.symbols, "nimcheck")
-  if optWasNimscript in conf.globalOptions:
-    defineSymbol(conf.symbols, "nimscript")
-    defineSymbol(conf.symbols, "nimconfig")
+  graph.config.setErrorMaxHighMaybe
+  defineSymbol(graph.config.symbols, "nimcheck")
   semanticPasses(graph)  # use an empty backend for semantic checking only
   compileProject(graph)
 
-  if conf.symbolFiles != disabledSf:
-    case conf.ideCmd
-    of ideDef: navDefinition(graph)
-    of ideUse: navUsages(graph)
-    of ideDus: navDefusages(graph)
-    else: discard
-    writeRodFiles(graph)
-
 when not defined(leanCompiler):
-  proc commandDoc2(graph: ModuleGraph; ext: string) =
+  proc commandDoc2(graph: ModuleGraph; json: bool) =
     handleDocOutputOptions graph.config
     graph.config.setErrorMaxHighMaybe
     semanticPasses(graph)
-    case ext:
-    of TexExt:  registerPass(graph, docgen2TexPass)
-    of JsonExt: registerPass(graph, docgen2JsonPass)
-    of HtmlExt: registerPass(graph, docgen2Pass)
-    else: doAssert false, $ext
+    if json: registerPass(graph, docgen2JsonPass)
+    else: registerPass(graph, docgen2Pass)
     compileProject(graph)
     finishDoc2Pass(graph.config.projectName)
 
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
+  setOutFile(conf)
   extccomp.initVars(conf)
   semanticPasses(graph)
-  if conf.symbolFiles == disabledSf:
-    registerPass(graph, cgenPass)
+  registerPass(graph, cgenPass)
 
-    if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"):
-      if not changeDetectedViaJsonBuildInstructions(conf, conf.jsonBuildInstructionsFile):
-        # nothing changed
-        graph.config.notes = graph.config.mainPackageNotes
-        return
+  if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"):
+    let proj = changeFileExt(conf.projectFull, "")
+    if not changeDetectedViaJsonBuildInstructions(conf, proj):
+      # nothing changed
+      graph.config.notes = graph.config.mainPackageNotes
+      return
 
   if not extccomp.ccHasSaneOverflow(conf):
     conf.symbols.defineSymbol("nimEmulateOverflowChecks")
@@ -104,16 +86,8 @@ proc commandCompileToC(graph: ModuleGraph) =
   compileProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933
-  if conf.symbolFiles == disabledSf:
-    cgenWriteModules(graph.backend, conf)
-  else:
-    if isDefined(conf, "nimIcIntegrityChecks"):
-      checkIntegrity(graph)
-    generateCode(graph)
-    # graph.backend can be nil under IC when nothing changed at all:
-    if graph.backend != nil:
-      cgenWriteModules(graph.backend, conf)
-  if conf.cmd != cmdTcc and graph.backend != nil:
+  cgenWriteModules(graph.backend, conf)
+  if conf.cmd != cmdRun:
     extccomp.callCCompiler(conf)
     # for now we do not support writing out a .json file with the build instructions when HCR is on
     if not conf.hcrOn:
@@ -122,20 +96,27 @@ proc commandCompileToC(graph: ModuleGraph) =
       writeDepsFile(graph)
 
 proc commandJsonScript(graph: ModuleGraph) =
-  extccomp.runJsonBuildInstructions(graph.config, graph.config.jsonBuildInstructionsFile)
+  let proj = changeFileExt(graph.config.projectFull, "")
+  extccomp.runJsonBuildInstructions(graph.config, proj)
 
 proc commandCompileToJS(graph: ModuleGraph) =
-  let conf = graph.config
   when defined(leanCompiler):
-    globalError(conf, unknownLineInfo, "compiler wasn't built with JS code generator")
+    globalError(graph.config, unknownLineInfo, "compiler wasn't built with JS code generator")
   else:
+    let conf = graph.config
     conf.exc = excCpp
-    setTarget(conf.target, osJS, cpuJS)
-    defineSymbol(conf.symbols, "ecmascript") # For backward compatibility
+
+    if conf.outFile.isEmpty:
+      conf.outFile = RelativeFile(conf.projectName & ".js")
+
+    #incl(gGlobalOptions, optSafeCode)
+    setTarget(graph.config.target, osJS, cpuJS)
+    #initDefines()
+    defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility
     semanticPasses(graph)
     registerPass(graph, JSgenPass)
     compileProject(graph)
-    if optGenScript in conf.globalOptions:
+    if optGenScript in graph.config.globalOptions:
       writeDepsFile(graph)
 
 proc interactivePasses(graph: ModuleGraph) =
@@ -156,9 +137,12 @@ proc commandInteractive(graph: ModuleGraph) =
   else:
     var m = graph.makeStdinModule()
     incl(m.flags, sfMainModule)
-    var idgen = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0)
-    let s = llStreamOpenStdIn(onPrompt = proc() = flushDot(graph.config))
-    processModule(graph, m, idgen, s)
+    processModule(graph, m, llStreamOpenStdIn())
+
+const evalPasses = [verbosePass, semPass, evalPass]
+
+proc evalNim(graph: ModuleGraph; nodes: PNode, module: PSym) =
+  carryPasses(graph, nodes, module, evalPasses)
 
 proc commandScan(cache: IdentCache, config: ConfigRef) =
   var f = addFileExt(AbsoluteFile mainCommandArg(config), NimExt)
@@ -177,46 +161,19 @@ proc commandScan(cache: IdentCache, config: ConfigRef) =
   else:
     rawMessage(config, errGenerated, "cannot open file: " & f.string)
 
-proc commandView(graph: ModuleGraph) =
-  let f = toAbsolute(mainCommandArg(graph.config), AbsoluteDir getCurrentDir()).addFileExt(RodExt)
-  rodViewer(f, graph.config, graph.cache)
-
 const
   PrintRopeCacheStats = false
 
-proc hashMainCompilationParams*(conf: ConfigRef): string =
-  ## doesn't have to be complete; worst case is a cache hit and recompilation.
-  var state = newSha1State()
-  with state:
-    update os.getAppFilename() # nim compiler
-    update conf.commandLine # excludes `arguments`, as it should
-    update $conf.projectFull # so that running `nim r main` from 2 directories caches differently
-  result = $SecureHash(state.finalize())
-
-proc setOutFile*(conf: ConfigRef) =
-  proc libNameTmpl(conf: ConfigRef): string {.inline.} =
-    result = if conf.target.targetOS == osWindows: "$1.lib" else: "lib$1.a"
-
-  if conf.outFile.isEmpty:
-    var base = conf.projectName
-    if optUseNimcache in conf.globalOptions:
-      base.add "_" & hashMainCompilationParams(conf)
-    let targetName =
-      if conf.backend == backendJs: base & ".js"
-      elif optGenDynLib in conf.globalOptions:
-        platform.OS[conf.target.targetOS].dllFrmt % base
-      elif optGenStaticLib in conf.globalOptions: libNameTmpl(conf) % base
-      else: base & platform.OS[conf.target.targetOS].exeExt
-    conf.outFile = RelativeFile targetName
-
 proc mainCommand*(graph: ModuleGraph) =
   let conf = graph.config
   let cache = graph.cache
 
+  setupModuleCache(graph)
   # In "nim serve" scenario, each command must reset the registered passes
   clearPasses(graph)
   conf.lastCmdTime = epochTime()
   conf.searchPaths.add(conf.libpath)
+  setId(100)
 
   proc customizeForBackend(backend: TBackend) =
     ## Sets backend specific options but don't compile to backend yet in
@@ -240,10 +197,15 @@ proc mainCommand*(graph: ModuleGraph) =
         # A better solution might be to fix system.nim
         undefSymbol(conf.symbols, "useNimRtl")
     of backendInvalid: doAssert false
+    if conf.selectedGC in {gcArc, gcOrc} and conf.backend != backendCpp:
+      conf.exc = excGoto
 
-  proc compileToBackend() =
-    customizeForBackend(conf.backend)
-    setOutFile(conf)
+  var commandAlreadyProcessed = false
+
+  proc compileToBackend(backend: TBackend, cmd = cmdCompileToBackend) =
+    commandAlreadyProcessed = true
+    conf.cmd = cmd
+    customizeForBackend(backend)
     case conf.backend
     of backendC: commandCompileToC(graph)
     of backendCpp: commandCompileToC(graph)
@@ -253,76 +215,86 @@ proc mainCommand*(graph: ModuleGraph) =
 
   template docLikeCmd(body) =
     when defined(leanCompiler):
-      conf.quitOrRaise "compiler wasn't built with documentation generator"
+      quit "compiler wasn't built with documentation generator"
     else:
       wantMainModule(conf)
-      let docConf = if conf.cmd == cmdDoc2tex: DocTexConfig else: DocConfig
-      loadConfigs(docConf, cache, conf, graph.idgen)
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
       defineSymbol(conf.symbols, "nimdoc")
       body
 
-  ## command prepass
-  if conf.cmd == cmdCrun: conf.globalOptions.incl {optRun, optUseNimcache}
-  if conf.cmd notin cmdBackends + {cmdTcc}: customizeForBackend(backendC)
-  if conf.outDir.isEmpty:
-    # doc like commands can generate a lot of files (especially with --project)
-    # so by default should not end up in $PWD nor in $projectPath.
-    var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
-              else: conf.projectPath
-    doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
-    if conf.cmd in cmdDocLike + {cmdRst2html, cmdRst2tex}: ret = ret / htmldocsDir
-    conf.outDir = ret
+  block: ## command prepass
+    var docLikeCmd2 = false # includes what calls `docLikeCmd` + some more
+    case conf.command.normalize
+    of "r": conf.globalOptions.incl {optRun, optUseNimcache}
+    of "doc0",  "doc2", "doc", "rst2html", "rst2tex", "jsondoc0", "jsondoc2",
+      "jsondoc", "ctags", "buildindex": docLikeCmd2 = true
+    else: discard
+    if conf.outDir.isEmpty:
+      # doc like commands can generate a lot of files (especially with --project)
+      # so by default should not end up in $PWD nor in $projectPath.
+      conf.outDir = block:
+        var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
+                  else: conf.projectPath
+        doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
+        if docLikeCmd2: ret = ret / htmldocsDir
+        ret
 
-  ## process all commands
-  case conf.cmd
-  of cmdBackends: compileToBackend()
-  of cmdTcc:
+  ## process all backend commands
+  case conf.command.normalize
+  of "c", "cc", "compile", "compiletoc": compileToBackend(backendC) # compile means compileToC currently
+  of "cpp", "compiletocpp": compileToBackend(backendCpp)
+  of "objc", "compiletooc": compileToBackend(backendObjc)
+  of "js", "compiletojs": compileToBackend(backendJs)
+  of "r": compileToBackend(backendC) # different from `"run"`!
+  of "run":
     when hasTinyCBackend:
       extccomp.setCC(conf, "tcc", unknownLineInfo)
-      if conf.backend != backendC:
+      if conf.backend notin {backendC, backendInvalid}:
         rawMessage(conf, errGenerated, "'run' requires c backend, got: '$1'" % $conf.backend)
-      compileToBackend()
+      compileToBackend(backendC, cmd = cmdRun)
     else:
       rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
-  of cmdDoc0: docLikeCmd commandDoc(cache, conf)
-  of cmdDoc:
+  else: customizeForBackend(backendC) # fallback for other commands
+
+  ## process all other commands
+  case conf.command.normalize # synchronize with `cmdUsingHtmlDocs`
+  of "doc0": docLikeCmd commandDoc(cache, conf)
+  of "doc2", "doc":
     docLikeCmd():
       conf.setNoteDefaults(warnLockLevel, false) # issue #13218
-      conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # issue #13218
+      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
         # because currently generates lots of false positives due to conflation
         # of labels links in doc comments, e.g. for random.rand:
         #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
         #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
-      commandDoc2(graph, HtmlExt)
+      commandDoc2(graph, false)
       if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
         commandBuildIndex(conf, $conf.outDir)
-  of cmdRst2html:
-    # XXX: why are warnings disabled by default for rst2html and rst2tex?
-    for warn in rstWarnings:
-      conf.setNoteDefaults(warn, true)
-    conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # similar to issue #13218
+  of "rst2html":
+    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
     when defined(leanCompiler):
-      conf.quitOrRaise "compiler wasn't built with documentation generator"
+      quit "compiler wasn't built with documentation generator"
     else:
-      loadConfigs(DocConfig, cache, conf, graph.idgen)
+      conf.cmd = cmdRst2html
+      loadConfigs(DocConfig, cache, conf)
       commandRst2Html(cache, conf)
-  of cmdRst2tex, cmdDoc2tex:
-    for warn in rstWarnings:
-      conf.setNoteDefaults(warn, true)
+  of "rst2tex":
     when defined(leanCompiler):
-      conf.quitOrRaise "compiler wasn't built with documentation generator"
+      quit "compiler wasn't built with documentation generator"
     else:
-      if conf.cmd == cmdRst2tex:
-        loadConfigs(DocTexConfig, cache, conf, graph.idgen)
-        commandRst2TeX(cache, conf)
-      else:
-        docLikeCmd commandDoc2(graph, TexExt)
-  of cmdJsondoc0: docLikeCmd commandJson(cache, conf)
-  of cmdJsondoc: docLikeCmd commandDoc2(graph, JsonExt)
-  of cmdCtags: docLikeCmd commandTags(cache, conf)
-  of cmdBuildindex: docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
-  of cmdGendepend: commandGenDepend(graph)
-  of cmdDump:
+      conf.cmd = cmdRst2tex
+      loadConfigs(DocTexConfig, cache, conf)
+      commandRst2TeX(cache, conf)
+  of "jsondoc0": docLikeCmd commandJson(cache, conf)
+  of "jsondoc2", "jsondoc": docLikeCmd commandDoc2(graph, true)
+  of "ctags": docLikeCmd commandTags(cache, conf)
+  of "buildindex": docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
+  of "gendepend":
+    conf.cmd = cmdGenDepend
+    commandGenDepend(graph)
+  of "dump":
+    conf.cmd = cmdDump
     if getConfigVar(conf, "dump.format") == "json":
       wantMainModule(conf)
 
@@ -357,41 +329,77 @@ proc mainCommand*(graph: ModuleGraph) =
         (key: "warnings", val: warnings),
       ]
 
-      msgWriteln(conf, $dumpdata, {msgStdout, msgSkipHook, msgNoUnitSep})
-        # `msgNoUnitSep` to avoid generating invalid json, refs bug #17853
+      msgWriteln(conf, $dumpdata, {msgStdout, msgSkipHook})
     else:
       msgWriteln(conf, "-- list of currently defined symbols --",
-                 {msgStdout, msgSkipHook, msgNoUnitSep})
-      for s in definedSymbolNames(conf.symbols): msgWriteln(conf, s, {msgStdout, msgSkipHook, msgNoUnitSep})
+                 {msgStdout, msgSkipHook})
+      for s in definedSymbolNames(conf.symbols): msgWriteln(conf, s, {msgStdout, msgSkipHook})
       msgWriteln(conf, "-- end of list --", {msgStdout, msgSkipHook})
 
       for it in conf.searchPaths: msgWriteln(conf, it.string)
-  of cmdCheck:
+  of "check":
+    conf.cmd = cmdCheck
     commandCheck(graph)
-  of cmdParse:
+  of "parse":
+    conf.cmd = cmdParse
     wantMainModule(conf)
     discard parseFile(conf.projectMainIdx, cache, conf)
-  of cmdRod:
+  of "scan":
+    conf.cmd = cmdScan
     wantMainModule(conf)
-    commandView(graph)
-    #msgWriteln(conf, "Beware: Indentation tokens depend on the parser's state!")
-  of cmdInteractive: commandInteractive(graph)
-  of cmdNimscript:
-    if conf.projectIsCmd or conf.projectIsStdin: discard
-    elif not fileExists(conf.projectFull):
+    commandScan(cache, conf)
+    msgWriteln(conf, "Beware: Indentation tokens depend on the parser's state!")
+  of "secret":
+    conf.cmd = cmdInteractive
+    commandInteractive(graph)
+  of "e":
+    if not fileExists(conf.projectFull):
       rawMessage(conf, errGenerated, "NimScript file does not exist: " & conf.projectFull.string)
-    # main NimScript logic handled in `loadConfigs`.
-  of cmdNop: discard
-  of cmdJsonscript:
+    elif not conf.projectFull.string.endsWith(".nims"):
+      rawMessage(conf, errGenerated, "not a NimScript file: " & conf.projectFull.string)
+    # main NimScript logic handled in cmdlinehelper.nim.
+  of "nop", "help":
+    # prevent the "success" message:
+    conf.cmd = cmdDump
+  of "jsonscript":
+    conf.cmd = cmdJsonScript
     setOutFile(graph.config)
     commandJsonScript(graph)
-  of cmdUnknown, cmdNone, cmdIdeTools, cmdNimfix:
+  elif commandAlreadyProcessed: discard # already handled
+  else:
     rawMessage(conf, errGenerated, "invalid command: " & conf.command)
 
-  if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop}:
+  if conf.errorCounter == 0 and
+     conf.cmd notin {cmdInterpret, cmdRun, cmdDump}:
+    let mem =
+      when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem"
+      else: formatSize(getTotalMem()) & " totmem"
+    let loc = $conf.linesCompiled
+    let build = if isDefined(conf, "danger"): "Dangerous Release"
+                elif isDefined(conf, "release"): "Release"
+                else: "Debug"
+    let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3)
+    let project = if optListFullPaths in conf.globalOptions: $conf.projectFull else: $conf.projectName
+
+    var output: string
+    if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonScript:
+      output = $conf.jsonBuildFile
+    elif conf.outFile.isEmpty and conf.cmd notin {cmdJsonScript, cmdCompileToBackend, cmdDoc}:
+      # for some cmd we expect a valid absOutFile
+      output = "unknownOutput"
+    else:
+      output = $conf.absOutFile
+    if optListFullPaths notin conf.globalOptions: output = output.AbsoluteFile.extractFilename
     if optProfileVM in conf.globalOptions:
       echo conf.dump(conf.vmProfileData)
-    genSuccessX(conf)
+    rawMessage(conf, hintSuccessX, [
+      "loc", loc,
+      "sec", sec,
+      "mem", mem,
+      "build", build,
+      "project", project,
+      "output", output,
+      ])
 
   when PrintRopeCacheStats:
     echo "rope cache stats: "

compiler/modulegraphs.nim

@@ -9,65 +9,30 @@
 
 ## This module implements the module graph data structure. The module graph
 ## represents a complete Nim project. Single modules can either be kept in RAM
-## or stored in a rod-file.
+## or stored in a Sqlite database.
+##
+## The caching of modules is critical for 'nimsuggest' and is tricky to get
+## right. If module E is being edited, we need autocompletion (and type
+## checking) for E but we don't want to recompile depending
+## modules right away for faster turnaround times. Instead we mark the module's
+## dependencies as 'dirty'. Let D be a dependency of E. If D is dirty, we
+## need to recompile it and all of its dependencies that are marked as 'dirty'.
+## 'nimsuggest sug' actually is invoked for the file being edited so we know
+## its content changed and there is no need to compute any checksums.
+## Instead of a recursive algorithm, we use an iterative algorithm:
+##
+## - If a module gets recompiled, its dependencies need to be updated.
+## - Its dependent module stays the same.
+##
 
-import intsets, tables, hashes, md5, sequtils
-import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages
-import ic / [packed_ast, ic]
+import ast, intsets, tables, options, lineinfos, hashes, idents,
+  incremental, btrees, md5
 
 type
   SigHash* = distinct MD5Digest
 
-  LazySym* = object
-    id*: FullId
-    sym*: PSym
-
-  Iface* = object       ## data we don't want to store directly in the
-                        ## ast.PSym type for s.kind == skModule
-    module*: PSym       ## module this "Iface" belongs to
-    converters*: seq[LazySym]
-    patterns*: seq[LazySym]
-    pureEnums*: seq[LazySym]
-    interf: TStrTable
-    interfHidden: TStrTable
-    uniqueName*: Rope
-
-  Operators* = object
-    opNot*, opContains*, opLe*, opLt*, opAnd*, opOr*, opIsNil*, opEq*: PSym
-    opAdd*, opSub*, opMul*, opDiv*, opLen*: PSym
-
-  FullId* = object
-    module*: int
-    packed*: PackedItemId
-
-  LazyType* = object
-    id*: FullId
-    typ*: PType
-
-  LazyInstantiation* = object
-    module*: int
-    sym*: FullId
-    concreteTypes*: seq[FullId]
-    inst*: PInstantiation
-
-  SymInfoPair* = object
-    sym*: PSym
-    info*: TLineInfo
-    isDecl*: bool
-
   ModuleGraph* {.acyclic.} = ref object
-    ifaces*: seq[Iface]  ## indexed by int32 fileIdx
-    packed*: PackedModuleGraph
-    encoders*: seq[PackedEncoder]
-
-    typeInstCache*: Table[ItemId, seq[LazyType]] # A symbol's ItemId.
-    procInstCache*: Table[ItemId, seq[LazyInstantiation]] # A symbol's ItemId.
-    attachedOps*: array[TTypeAttachedOp, Table[ItemId, PSym]] # Type ID, destructors, etc.
-    methodsPerType*: Table[ItemId, seq[(int, LazySym)]] # Type ID, attached methods
-    enumToStringProcs*: Table[ItemId, LazySym]
-    emittedTypeInfo*: Table[string, FileIndex]
-
-    startupPackedConfig*: PackedConfig
+    modules*: seq[PSym]  ## indexed by int32 fileIdx
     packageSyms*: TStrTable
     deps*: IntSet # the dependency graph or potentially its transitive closure.
     importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies
@@ -85,22 +50,23 @@ type
     doStopCompile*: proc(): bool {.closure.}
     usageSym*: PSym # for nimsuggest
     owners*: seq[PSym]
-    suggestSymbols*: Table[FileIndex, seq[SymInfoPair]]
-    suggestErrors*: Table[FileIndex, seq[Suggest]]
     methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization!
     systemModule*: PSym
     sysTypes*: array[TTypeKind, PType]
     compilerprocs*: TStrTable
     exposed*: TStrTable
-    packageTypes*: TStrTable
+    intTypeCache*: array[-5..64, PType]
+    opContains*, opNot*: PSym
     emptyNode*: PNode
+    incr*: IncrementalCtx
     canonTypes*: Table[SigHash, PType]
     symBodyHashes*: Table[int, SigHash] # symId to digest mapping
     importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.}
     includeFileCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PNode {.nimcall.}
-    cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API; IC: implemented
-    cacheCounters*: Table[string, BiggestInt] # IC: implemented
-    cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented
+    recordStmt*: proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.}
+    cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API
+    cacheCounters*: Table[string, BiggestInt]
+    cacheTables*: Table[string, BTree[string, PNode]]
     passes*: seq[TPass]
     onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
@@ -108,14 +74,11 @@ type
     globalDestructors*: seq[PNode]
     strongSemCheck*: proc (graph: ModuleGraph; owner: PSym; body: PNode) {.nimcall.}
     compatibleProps*: proc (graph: ModuleGraph; formal, actual: PType): bool {.nimcall.}
-    idgen*: IdGenerator
-    operators*: Operators
 
   TPassContext* = object of RootObj # the pass's context
-    idgen*: IdGenerator
   PPassContext* = ref TPassContext
 
-  TPassOpen* = proc (graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nimcall.}
+  TPassOpen* = proc (graph: ModuleGraph; module: PSym): PPassContext {.nimcall.}
   TPassClose* = proc (graph: ModuleGraph; p: PPassContext, n: PNode): PNode {.nimcall.}
   TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}
 
@@ -124,14 +87,6 @@ type
                  close: TPassClose,
                  isFrontend: bool]
 
-proc resetForBackend*(g: ModuleGraph) =
-  initStrTable(g.compilerprocs)
-  g.typeInstCache.clear()
-  g.procInstCache.clear()
-  for a in mitems(g.attachedOps):
-    a.clear()
-  g.methodsPerType.clear()
-  g.enumToStringProcs.clear()
 
 const
   cb64 = [
@@ -167,196 +122,6 @@ proc toBase64a(s: cstring, len: int): string =
     result.add cb64[a shr 2]
     result.add cb64[(a and 3) shl 4]
 
-template interfSelect(iface: Iface, importHidden: bool): TStrTable =
-  var ret = iface.interf.addr # without intermediate ptr, it creates a copy and compiler becomes 15x slower!
-  if importHidden: ret = iface.interfHidden.addr
-  ret[]
-
-template semtab(g: ModuleGraph, m: PSym): TStrTable =
-  g.ifaces[m.position].interf
-
-template semtabAll*(g: ModuleGraph, m: PSym): TStrTable =
-  g.ifaces[m.position].interfHidden
-
-proc initStrTables*(g: ModuleGraph, m: PSym) =
-  initStrTable(semtab(g, m))
-  initStrTable(semtabAll(g, m))
-
-proc strTableAdds*(g: ModuleGraph, m: PSym, s: PSym) =
-  strTableAdd(semtab(g, m), s)
-  strTableAdd(semtabAll(g, m), s)
-
-proc isCachedModule(g: ModuleGraph; module: int): bool {.inline.} =
-  result = module < g.packed.len and g.packed[module].status == loaded
-
-proc isCachedModule(g: ModuleGraph; m: PSym): bool {.inline.} =
-  isCachedModule(g, m.position)
-
-proc simulateCachedModule*(g: ModuleGraph; moduleSym: PSym; m: PackedModule) =
-  when false:
-    echo "simulating ", moduleSym.name.s, " ", moduleSym.position
-  simulateLoadedModule(g.packed, g.config, g.cache, moduleSym, m)
-
-proc initEncoder*(g: ModuleGraph; module: PSym) =
-  let id = module.position
-  if id >= g.encoders.len:
-    setLen g.encoders, id+1
-  ic.initEncoder(g.encoders[id],
-    g.packed[id].fromDisk, module, g.config, g.startupPackedConfig)
-
-type
-  ModuleIter* = object
-    fromRod: bool
-    modIndex: int
-    ti: TIdentIter
-    rodIt: RodIter
-    importHidden: bool
-
-proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym =
-  assert m.kind == skModule
-  mi.modIndex = m.position
-  mi.fromRod = isCachedModule(g, mi.modIndex)
-  mi.importHidden = optImportHidden in m.options
-  if mi.fromRod:
-    result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name, mi.importHidden)
-  else:
-    result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name)
-
-proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym =
-  if mi.fromRod:
-    result = nextRodIter(mi.rodIt, g.packed)
-  else:
-    result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden))
-
-iterator allSyms*(g: ModuleGraph; m: PSym): PSym =
-  let importHidden = optImportHidden in m.options
-  if isCachedModule(g, m):
-    var rodIt: RodIter
-    var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position, importHidden)
-    while r != nil:
-      yield r
-      r = nextRodIter(rodIt, g.packed)
-  else:
-    for s in g.ifaces[m.position].interfSelect(importHidden).data:
-      if s != nil:
-        yield s
-
-proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym =
-  let importHidden = optImportHidden in m.options
-  if isCachedModule(g, m):
-    result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden)
-  else:
-    result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name)
-
-proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym =
-  result = someSym(g, g.systemModule, name)
-
-iterator systemModuleSyms*(g: ModuleGraph; name: PIdent): PSym =
-  var mi: ModuleIter
-  var r = initModuleIter(mi, g, g.systemModule, name)
-  while r != nil:
-    yield r
-    r = nextModuleIter(mi, g)
-
-proc resolveType(g: ModuleGraph; t: var LazyType): PType =
-  result = t.typ
-  if result == nil and isCachedModule(g, t.id.module):
-    result = loadTypeFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed)
-    t.typ = result
-  assert result != nil
-
-proc resolveSym(g: ModuleGraph; t: var LazySym): PSym =
-  result = t.sym
-  if result == nil and isCachedModule(g, t.id.module):
-    result = loadSymFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed)
-    t.sym = result
-  assert result != nil
-
-proc resolveInst(g: ModuleGraph; t: var LazyInstantiation): PInstantiation =
-  result = t.inst
-  if result == nil and isCachedModule(g, t.module):
-    result = PInstantiation(sym: loadSymFromId(g.config, g.cache, g.packed, t.sym.module, t.sym.packed))
-    result.concreteTypes = newSeq[PType](t.concreteTypes.len)
-    for i in 0..high(result.concreteTypes):
-      result.concreteTypes[i] = loadTypeFromId(g.config, g.cache, g.packed,
-          t.concreteTypes[i].module, t.concreteTypes[i].packed)
-    t.inst = result
-  assert result != nil
-
-iterator typeInstCacheItems*(g: ModuleGraph; s: PSym): PType =
-  if g.typeInstCache.contains(s.itemId):
-    let x = addr(g.typeInstCache[s.itemId])
-    for t in mitems(x[]):
-      yield resolveType(g, t)
-
-iterator procInstCacheItems*(g: ModuleGraph; s: PSym): PInstantiation =
-  if g.procInstCache.contains(s.itemId):
-    let x = addr(g.procInstCache[s.itemId])
-    for t in mitems(x[]):
-      yield resolveInst(g, t)
-
-proc getAttachedOp*(g: ModuleGraph; t: PType; op: TTypeAttachedOp): PSym =
-  ## returns the requested attached operation for type `t`. Can return nil
-  ## if no such operation exists.
-  result = g.attachedOps[op].getOrDefault(t.itemId)
-
-proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
-  ## we also need to record this to the packed module.
-  g.attachedOps[op][t.itemId] = value
-
-proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
-  ## we also need to record this to the packed module.
-  g.attachedOps[op][t.itemId] = value
-  # XXX Also add to the packed module!
-
-proc completePartialOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
-  if g.config.symbolFiles != disabledSf:
-    assert module < g.encoders.len
-    assert isActive(g.encoders[module])
-    toPackedGeneratedProcDef(value, g.encoders[module], g.packed[module].fromDisk)
-
-proc getToStringProc*(g: ModuleGraph; t: PType): PSym =
-  result = resolveSym(g, g.enumToStringProcs[t.itemId])
-  assert result != nil
-
-proc setToStringProc*(g: ModuleGraph; t: PType; value: PSym) =
-  g.enumToStringProcs[t.itemId] = LazySym(sym: value)
-
-iterator methodsForGeneric*(g: ModuleGraph; t: PType): (int, PSym) =
-  if g.methodsPerType.contains(t.itemId):
-    for it in mitems g.methodsPerType[t.itemId]:
-      yield (it[0], resolveSym(g, it[1]))
-
-proc addMethodToGeneric*(g: ModuleGraph; module: int; t: PType; col: int; m: PSym) =
-  g.methodsPerType.mgetOrPut(t.itemId, @[]).add (col, LazySym(sym: m))
-
-proc hasDisabledAsgn*(g: ModuleGraph; t: PType): bool =
-  let op = getAttachedOp(g, t, attachedAsgn)
-  result = op != nil and sfError in op.flags
-
-proc copyTypeProps*(g: ModuleGraph; module: int; dest, src: PType) =
-  for k in low(TTypeAttachedOp)..high(TTypeAttachedOp):
-    let op = getAttachedOp(g, src, k)
-    if op != nil:
-      setAttachedOp(g, module, dest, k, op)
-
-proc loadCompilerProc*(g: ModuleGraph; name: string): PSym =
-  if g.config.symbolFiles == disabledSf: return nil
-
-  # slow, linear search, but the results are cached:
-  for module in 0..high(g.packed):
-    #if isCachedModule(g, module):
-    let x = searchForCompilerproc(g.packed[module], name)
-    if x >= 0:
-      result = loadSymFromId(g.config, g.cache, g.packed, module, toPackedItemId(x))
-      if result != nil:
-        strTableAdd(g.compilerprocs, result)
-      return result
-
-proc loadPackedSym*(g: ModuleGraph; s: var LazySym) =
-  if s.sym == nil:
-    s.sym = loadSymFromId(g.config, g.cache, g.packed, s.id.module, s.id.packed)
-
 proc `$`*(u: SigHash): string =
   toBase64a(cast[cstring](unsafeAddr u), sizeof(u))
 
@@ -370,13 +135,27 @@ proc hash*(u: SigHash): Hash =
 
 proc hash*(x: FileIndex): Hash {.borrow.}
 
-template getPContext(): untyped =
-  when c is PContext: c
-  else: c.c
+when defined(nimfind):
+  template onUse*(info: TLineInfo; s: PSym) =
+    when compiles(c.c.graph):
+      if c.c.graph.onUsage != nil: c.c.graph.onUsage(c.c.graph, s, info)
+    else:
+      if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info)
+
+  template onDef*(info: TLineInfo; s: PSym) =
+    when compiles(c.c.graph):
+      if c.c.graph.onDefinition != nil: c.c.graph.onDefinition(c.c.graph, s, info)
+    else:
+      if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info)
+
+  template onDefResolveForward*(info: TLineInfo; s: PSym) =
+    when compiles(c.c.graph):
+      if c.c.graph.onDefinitionResolveForward != nil:
+        c.c.graph.onDefinitionResolveForward(c.c.graph, s, info)
+    else:
+      if c.graph.onDefinitionResolveForward != nil:
+        c.graph.onDefinitionResolveForward(c.graph, s, info)
 
-when defined(nimsuggest):
-  template onUse*(info: TLineInfo; s: PSym) = discard
-  template onDefResolveForward*(info: TLineInfo; s: PSym) = discard
 else:
   template onUse*(info: TLineInfo; s: PSym) = discard
   template onDef*(info: TLineInfo; s: PSym) = discard
@@ -385,84 +164,41 @@ else:
 proc stopCompile*(g: ModuleGraph): bool {.inline.} =
   result = g.doStopCompile != nil and g.doStopCompile()
 
-proc createMagic*(g: ModuleGraph; idgen: IdGenerator; name: string, m: TMagic): PSym =
-  result = newSym(skProc, getIdent(g.cache, name), nextSymId(idgen), nil, unknownLineInfo, {})
+proc createMagic*(g: ModuleGraph; name: string, m: TMagic): PSym =
+  result = newSym(skProc, getIdent(g.cache, name), nil, unknownLineInfo, {})
   result.magic = m
   result.flags = {sfNeverRaises}
 
-proc createMagic(g: ModuleGraph; name: string, m: TMagic): PSym =
-  result = createMagic(g, g.idgen, name, m)
-
-proc registerModule*(g: ModuleGraph; m: PSym) =
-  assert m != nil
-  assert m.kind == skModule
-
-  if m.position >= g.ifaces.len:
-    setLen(g.ifaces, m.position + 1)
-
-  if m.position >= g.packed.len:
-    setLen(g.packed, m.position + 1)
-
-  g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[],
-                               uniqueName: rope(uniqueModuleName(g.config, FileIndex(m.position))))
-  initStrTables(g, m)
-
-proc registerModuleById*(g: ModuleGraph; m: FileIndex) =
-  registerModule(g, g.packed[int m].module)
-
-proc initOperators*(g: ModuleGraph): Operators =
-  # These are safe for IC.
-  # Public because it's used by DrNim.
-  result.opLe = createMagic(g, "<=", mLeI)
-  result.opLt = createMagic(g, "<", mLtI)
-  result.opAnd = createMagic(g, "and", mAnd)
-  result.opOr = createMagic(g, "or", mOr)
-  result.opIsNil = createMagic(g, "isnil", mIsNil)
-  result.opEq = createMagic(g, "==", mEqI)
-  result.opAdd = createMagic(g, "+", mAddI)
-  result.opSub = createMagic(g, "-", mSubI)
-  result.opMul = createMagic(g, "*", mMulI)
-  result.opDiv = createMagic(g, "div", mDivI)
-  result.opLen = createMagic(g, "len", mLengthSeq)
-  result.opNot = createMagic(g, "not", mNot)
-  result.opContains = createMagic(g, "contains", mInSet)
-
-proc initModuleGraphFields(result: ModuleGraph) =
-  # A module ID of -1 means that the symbol is not attached to a module at all,
-  # but to the module graph:
-  result.idgen = IdGenerator(module: -1'i32, symId: 0'i32, typeId: 0'i32)
+proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
+  result = ModuleGraph()
   initStrTable(result.packageSyms)
   result.deps = initIntSet()
   result.importDeps = initTable[FileIndex, seq[FileIndex]]()
-  result.ifaces = @[]
+  result.modules = @[]
   result.importStack = @[]
   result.inclToMod = initTable[FileIndex, FileIndex]()
+  result.config = config
+  result.cache = cache
   result.owners = @[]
-  result.suggestSymbols = initTable[FileIndex, seq[SymInfoPair]]()
-  result.suggestErrors = initTable[FileIndex, seq[Suggest]]()
   result.methods = @[]
   initStrTable(result.compilerprocs)
   initStrTable(result.exposed)
-  initStrTable(result.packageTypes)
+  result.opNot = createMagic(result, "not", mNot)
+  result.opContains = createMagic(result, "contains", mInSet)
   result.emptyNode = newNode(nkEmpty)
+  init(result.incr)
+  result.recordStmt = proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.} =
+    discard
   result.cacheSeqs = initTable[string, PNode]()
   result.cacheCounters = initTable[string, BiggestInt]()
   result.cacheTables = initTable[string, BTree[string, PNode]]()
   result.canonTypes = initTable[SigHash, PType]()
   result.symBodyHashes = initTable[int, SigHash]()
-  result.operators = initOperators(result)
-  result.emittedTypeInfo = initTable[string, FileIndex]()
-
-proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
-  result = ModuleGraph()
-  result.config = config
-  result.cache = cache
-  initModuleGraphFields(result)
 
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.packageSyms)
   g.deps = initIntSet()
-  g.ifaces = @[]
+  g.modules = @[]
   g.importStack = @[]
   g.inclToMod = initTable[FileIndex, FileIndex]()
   g.usageSym = nil
@@ -470,56 +206,16 @@ proc resetAllModules*(g: ModuleGraph) =
   g.methods = @[]
   initStrTable(g.compilerprocs)
   initStrTable(g.exposed)
-  initModuleGraphFields(g)
 
 proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
-  if fileIdx.int32 >= 0:
-    if isCachedModule(g, fileIdx.int32):
-      result = g.packed[fileIdx.int32].module
-    elif fileIdx.int32 < g.ifaces.len:
-      result = g.ifaces[fileIdx.int32].module
-
-proc moduleOpenForCodegen*(g: ModuleGraph; m: FileIndex): bool {.inline.} =
-  if g.config.symbolFiles == disabledSf:
-    result = true
-  else:
-    result = g.packed[m.int32].status notin {undefined, stored, loaded}
-
-proc rememberEmittedTypeInfo*(g: ModuleGraph; m: FileIndex; ti: string) =
-  #assert(not isCachedModule(g, m.int32))
-  if g.config.symbolFiles != disabledSf:
-    #assert g.encoders[m.int32].isActive
-    assert g.packed[m.int32].status != stored
-    g.packed[m.int32].fromDisk.emittedTypeInfo.add ti
-    #echo "added typeinfo ", m.int32, " ", ti, " suspicious ", not g.encoders[m.int32].isActive
-
-proc rememberFlag*(g: ModuleGraph; m: PSym; flag: ModuleBackendFlag) =
-  if g.config.symbolFiles != disabledSf:
-    #assert g.encoders[m.int32].isActive
-    assert g.packed[m.position].status != stored
-    g.packed[m.position].fromDisk.backendFlags.incl flag
-
-proc closeRodFile*(g: ModuleGraph; m: PSym) =
-  if g.config.symbolFiles in {readOnlySf, v2Sf}:
-    # For stress testing we seek to reload the symbols from memory. This
-    # way much of the logic is tested but the test is reproducible as it does
-    # not depend on the hard disk contents!
-    let mint = m.position
-    saveRodFile(toRodFile(g.config, AbsoluteFile toFullPath(g.config, FileIndex(mint))),
-                g.encoders[mint], g.packed[mint].fromDisk)
-    g.packed[mint].status = stored
-
-  elif g.config.symbolFiles == stressTest:
-    # debug code, but maybe a good idea for production? Could reduce the compiler's
-    # memory consumption considerably at the cost of more loads from disk.
-    let mint = m.position
-    simulateCachedModule(g, m, g.packed[mint].fromDisk)
-    g.packed[mint].status = loaded
+  if fileIdx.int32 >= 0 and fileIdx.int32 < g.modules.len:
+    result = g.modules[fileIdx.int32]
 
 proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b
 
 proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) =
   assert m.position == m.info.fileIndex.int32
+  addModuleDep(g.incr, g.config, m.info.fileIndex, dep, isIncludeFile = false)
   if g.suggestMode:
     g.deps.incl m.position.dependsOn(dep.int)
     # we compute the transitive closure later when querying the graph lazily.
@@ -527,13 +223,14 @@ proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) =
     #invalidTransitiveClosure = true
 
 proc addIncludeDep*(g: ModuleGraph; module, includeFile: FileIndex) =
+  addModuleDep(g.incr, g.config, module, includeFile, isIncludeFile = true)
   discard hasKeyOrPut(g.inclToMod, includeFile, module)
 
 proc parentModule*(g: ModuleGraph; fileIdx: FileIndex): FileIndex =
   ## returns 'fileIdx' if the file belonging to this index is
   ## directly used as a module or else the module that first
   ## references this include file.
-  if fileIdx.int32 >= 0 and fileIdx.int32 < g.ifaces.len and g.ifaces[fileIdx.int32].module != nil:
+  if fileIdx.int32 >= 0 and fileIdx.int32 < g.modules.len and g.modules[fileIdx.int32] != nil:
     result = fileIdx
   else:
     result = g.inclToMod.getOrDefault(fileIdx)
@@ -549,19 +246,7 @@ proc transitiveClosure(g: var IntSet; n: int) =
 
 proc markDirty*(g: ModuleGraph; fileIdx: FileIndex) =
   let m = g.getModule fileIdx
-  if m != nil:
-    g.suggestSymbols.del(fileIdx)
-    g.suggestErrors.del(fileIdx)
-    incl m.flags, sfDirty
-
-proc unmarkAllDirty*(g: ModuleGraph) =
-  for i in 0i32..<g.ifaces.len.int32:
-    let m = g.ifaces[i].module
-    if m != nil:
-      m.flags.excl sfDirty
-
-proc isDirty*(g: ModuleGraph; m: PSym): bool =
-  result = g.suggestMode and sfDirty in m.flags
+  if m != nil: incl m.flags, sfDirty
 
 proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
   # we need to mark its dependent modules D as dirty right away because after
@@ -569,91 +254,13 @@ proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
   # cleared but D still needs to be remembered as 'dirty'.
   if g.invalidTransitiveClosure:
     g.invalidTransitiveClosure = false
-    transitiveClosure(g.deps, g.ifaces.len)
+    transitiveClosure(g.deps, g.modules.len)
 
   # every module that *depends* on this file is also dirty:
-  for i in 0i32..<g.ifaces.len.int32:
-    if g.deps.contains(i.dependsOn(fileIdx.int)):
-      let
-        fi = FileIndex(i)
-        module = g.getModule(fi)
-      if module != nil and not g.isDirty(module):
-        g.markDirty(fi)
-        g.markClientsDirty(fi)
+  for i in 0i32..<g.modules.len.int32:
+    let m = g.modules[i]
+    if m != nil and g.deps.contains(i.dependsOn(fileIdx.int)):
+      incl m.flags, sfDirty
 
-proc needsCompilation*(g: ModuleGraph): bool =
-  # every module that *depends* on this file is also dirty:
-  for i in 0i32..<g.ifaces.len.int32:
-    let m = g.ifaces[i].module
-    if m != nil:
-      if sfDirty in m.flags:
-        return true
-
-proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool =
-  let module = g.getModule(fileIdx)
-  if module != nil and g.isDirty(module):
-    return true
-
-  for i in 0i32..<g.ifaces.len.int32:
-    let m = g.ifaces[i].module
-    if m != nil and g.isDirty(m) and g.deps.contains(fileIdx.int32.dependsOn(i)):
-      return true
-
-proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
-  result = s.ast[bodyPos]
-  if result == nil and g.config.symbolFiles in {readOnlySf, v2Sf, stressTest}:
-    result = loadProcBody(g.config, g.cache, g.packed, s)
-    s.ast[bodyPos] = result
-  assert result != nil
-
-proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex;
-                        cachedModules: var seq[FileIndex]): PSym =
-  ## Returns 'nil' if the module needs to be recompiled.
-  if g.config.symbolFiles in {readOnlySf, v2Sf, stressTest}:
-    result = moduleFromRodFile(g.packed, g.config, g.cache, fileIdx, cachedModules)
-
-proc configComplete*(g: ModuleGraph) =
-  rememberStartupConfig(g.startupPackedConfig, g.config)
-
-from std/strutils import repeat, `%`
-
-proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, fromModule: PSym, ) =
-  let conf = graph.config
-  let isNimscript = conf.isDefined("nimscript")
-  if (not isNimscript) or hintProcessing in conf.cmdlineNotes:
-    let path = toFilenameOption(conf, fileIdx, conf.filenameOption)
-    let indent = ">".repeat(graph.importStack.len)
-    let fromModule2 = if fromModule != nil: $fromModule.name.s else: "(toplevel)"
-    let mode = if isNimscript: "(nims) " else: ""
-    rawMessage(conf, hintProcessing, "$#$# $#: $#: $#" % [mode, indent, fromModule2, moduleStatus, path])
-
-proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
-  ## Returns a package symbol for yet to be defined module for fileIdx.
-  ## The package symbol is added to the graph if it doesn't exist.
-  let pkgSym = getPackage(graph.config, graph.cache, fileIdx)
-  # check if the package is already in the graph
-  result = graph.packageSyms.strTableGet(pkgSym.name)
-  if result == nil:
-     # the package isn't in the graph, so create and add it
-    result = pkgSym
-    graph.packageSyms.strTableAdd(pkgSym)
-
-func belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool =
-  ## Check if symbol belongs to the 'stdlib' package.
-  sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId
-
-proc `==`*(a, b: SymInfoPair): bool =
-  result = a.sym == b.sym and a.info.exactEquals(b.info)
-
-proc fileSymbols*(graph: ModuleGraph, fileIdx: FileIndex): seq[SymInfoPair] =
-  result = graph.suggestSymbols.getOrDefault(fileIdx, @[])
-
-iterator suggestSymbolsIter*(g: ModuleGraph): SymInfoPair =
-  for xs in g.suggestSymbols.values:
-    for x in xs:
-      yield x
-
-iterator suggestErrorsIter*(g: ModuleGraph): Suggest =
-  for xs in g.suggestErrors.values:
-    for x in xs:
-      yield x
+proc isDirty*(g: ModuleGraph; m: PSym): bool =
+  result = g.suggestMode and sfDirty in m.flags