bump NimVersion to 1.4.8

(cherry picked from commit df429fa287)

.builds/freebsd.yml

@@ -0,0 +1,31 @@
+# see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd
+image: freebsd/latest
+packages:
+- databases/sqlite3
+- devel/boehm-gc-threaded
+- devel/pcre
+- devel/sdl20
+- devel/sfml
+- www/node
+- devel/gmake
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpu)
+    bin/nim c --skipUserCfg --skipParentCfg koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_0.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "0_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_1.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "1_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_2.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "2_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.gitattributes

@@ -4,6 +4,3 @@
 # duplicated, which is easily identifiable and fixable.
 /changelog.md merge=union
 
-# bug https://github.com/dom96/choosenim/issues/256 for WSL CRLF
-*.sh text eol=lf
-/config/build_config.txt text eol=lf

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,51 @@
+---
+name: Bug report
+about: Have you found an unexpected behavior? Use this template.
+title: Think about the title, twice
+labels: ''
+assignees: ''
+
+---
+
+Function `echo` outputs the wrong string.
+
+### Example
+```nim
+echo "Hello World!"
+# This code should be a minimum reproducible example:
+# try to simplify and minimize as much as possible. If it's a compiler
+# issue, try to minimize further by removing any imports if possible.
+```
+
+### Current Output
+please check whether the problem still exists in git head before posting,
+see [rebuilding the compiler](https://nim-lang.github.io/Nim/intern.html#rebuilding-the-compiler).
+```
+Hola mundo!
+```
+
+### Expected Output
+```
+Hello World!
+```
+
+### Possible Solution
+
+* In file xyz there is a call that might be the cause of it.
+
+### Additional Information
+If it's a regression, you can help us by identifying which version introduced
+the bug, see [Bisecting for regressions](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions),
+or at least try known past releases (eg `choosenim 1.2.0`).
+
+If it's a pre-existing compiler bug, see [Debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler)
+which should give more context on a compiler crash.
+
+* Issue #abc is related, but different because of ...
+* This issue is blocking my project xyz
+
+```
+$ nim -v
+Nim Compiler Version 0.1.2
+# make sure to include the git hash if not using a tagged release
+```

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,75 +0,0 @@
-name:        "Bug Report"
-description: "Create a new bug report. Have you found an unexpected behavior? Use this form."
-title:       "Think about the title, twice."
-labels:      ["unconfirmed"]
-body:
-
-- type: markdown
-  attributes:
-    value: |
-      - **Please provide a minimal code example that reproduces the Bug!** :bug:
-        Reports with a reproducible example and descriptive detailed information will likely receive fixes faster.
-
-- type: textarea
-  id: description
-  attributes:
-    label: Description
-    description: |
-      Use DETAILED DESCRIPTIVE information about the problem.
-      Here, you go into more details about your Bug report. This section can be a few paragraphs long.
-    placeholder: Bug reports with reproducible code and detailed information will be fixed faster.
-  validations:
-    required: true
-    
-- type: textarea
-  id: nim-version
-  attributes:
-    label: Nim Version
-    description: Copy and paste the output of `nim -v` on the command line. For development versions, make sure to include the commit hash.
-  validations:
-    required: true
-
-- type: textarea
-  id: current-logs
-  attributes:
-    label: Current Output
-    description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
-    placeholder: Bug reports with reproducible code and detailed information will be fixed faster.
-    render: text
-
-- type: textarea
-  id: expected-logs
-  attributes:
-    label: Expected Output
-    description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
-    placeholder: Bug reports with reproducible code and detailed information will be fixed faster.
-    render: text
-
-- type: textarea
-  id: possible-solution
-  attributes:
-    label: Possible Solution
-    description: Propose a possible solution.
-  validations:
-    required: false
-
-- type: textarea
-  id: extra-info
-  attributes:
-    label: Additional Information
-    description: Any additional relevant information.
-  validations:
-    required: false
-
-- type: markdown
-  attributes:
-    value: |
-      - Thanks for your contributions!, your Bug report will receive feedback from the community soon...
-      - Please check whether the problem still exists in the devel branch, see [rebuilding the compiler](https://nim-lang.github.io/Nim/intern.html#rebuilding-the-compiler).
-      - Consider writing a PR targetting devel branch after filing this, see [contributing](https://nim-lang.github.io/Nim/contributing.html).
-      - If it's a pre-existing compiler bug, see [Debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler)
-        which should give more context on a compiler crash.
-      - If it's a regression, you can help us by identifying which version introduced the bug,
-        see [Bisecting for regressions](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions),
-        or at least try known past releases (eg `choosenim 1.2.0`).
-      - [Please, consider a Donation for the Nim project.](https://nim-lang.org/donate.html)

.github/ISSUE_TEMPLATE/config.yml

@@ -1,10 +0,0 @@
-contact_links:
-  - name: Forum
-    url: https://forum.nim-lang.org
-    about: Please ask and answer questions here.
-  - name: Discord
-    url: https://discord.gg/nim
-    about: Please ask and answer questions here.
-  - name: Stack Overflow
-    url: https://stackoverflow.com/questions/tagged/nim-lang
-    about: Please ask and answer questions here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,35 @@
+---
+name: Feature request
+about: Do you want to suggest a new feature? Use this template.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+<!---
+Notice there is now a separate repository for the detailed RFCs and proposals:
+https://github.com/nim-lang/RFCs
+
+If you have a simple feature request, you can post it here using this template,
+but bear in mind that adding new features to the language is currently a low priority.
+-->
+
+
+### Summary
+<!--- Short summary of your proposed feature -->
+
+
+### Description
+<!--- Describe your solution, what problem does it fix? -->
+
+
+### Alternatives
+<!--- Are there any alternatives you've considered? -->
+
+
+### Additional Information
+<!--- For Example:
+* A link to a project where the issue is relevant.
+* A link to a related issue or discussion.
+-->

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,72 +0,0 @@
-name:        "Feature request"
-description: "Create a new Feature Request. Do you want to suggest a new feature? Use this form."
-title:       "Think about the title, twice."
-labels:      ["unconfirmed"]
-body:
-
-- type: markdown
-  attributes:
-    value: |
-      - Please provide a minimal code example that illustrates the basic idea behind your feature request.
-        Reports with full repro code and descriptive detailed information will likely receive feedback faster.
-      - There is a separate repository for the detailed RFCs and proposals: https://github.com/nim-lang/RFCs.
-        If you have a simple feature request, you can post it here using this form,
-        but bear in mind that adding new features to the language is currently a low priority.
-
-- type: textarea
-  id: summary
-  attributes:
-    label: Summary
-    description: |
-      Use DETAILED DESCRIPTIVE information about the feature request.
-      Here, you go into more details about your ideas. This section can be a few paragraphs long.
-    placeholder: Short summary of your proposed feature.
-  validations:
-    required: true
-
-- type: textarea
-  id: description
-  attributes:
-    label: Description
-    description: Describe your solution, what problem does it fix?
-  validations:
-    required: true
-
-- type: textarea
-  id: alternatives
-  attributes:
-    label: Alternatives
-    description: Are there any alternatives you've considered?
-  validations:
-    required: false
-
-- type: textarea
-  id: Examples
-  attributes:
-    label: Examples
-    description: Provide examples for your feature request here.
-    placeholder: Example code here.
-
-- type: textarea
-  id: incompatibility
-  attributes:
-    label: Backwards Compatibility
-    description: If your Feature Request introduces backward-incompatible changes, describe them and propose how to deal with them.
-  validations:
-    required: false
-
-- type: textarea
-  id: links
-  attributes:
-    label: Links
-    description: Please copy and paste any relevant links to projects, issues, discussions, technical documentations, code samples, etc.
-  validations:
-    required: false
-
-- type: markdown
-  attributes:
-    value: |
-      - Thanks for your contributions!, your ideas will receive feedback from the community soon...
-      - **Remember to :star: Star the Nim project on GitHub!.**
-      - Consider writing a PR targetting devel branch after filing this, see [contributing](https://nim-lang.github.io/Nim/contributing.html).
-      - [Please, consider a Donation for the Nim project.](https://nim-lang.org/donate.html)

.github/workflows/ci.yml.disabled

@@ -0,0 +1,122 @@
+name: Continous Integration
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64, i386]
+        cpp: ['false', 'true']
+        pkg: ['false', 'true']
+        exclude:
+          - os: ubuntu-18.04
+            cpp: 'true'
+          - os: ubuntu-18.04
+            pkg: 'true'
+          - os: macos-10.15
+            cpu: i386
+          - os: macos-10.15
+            pkg: 'true'
+          - os: windows-2019
+            cpu: i386
+          - os: windows-2019
+            cpp: 'true'
+    name: '${{ matrix.os }} (${{ matrix.cpu }}, cpp: ${{ matrix.cpp }}, pkg: ${{ matrix.pkg }})'
+    runs-on: ${{ matrix.os }}
+    env:
+      NIM_COMPILE_TO_CPP: ${{ matrix.cpp }}
+      NIM_TEST_PACKAGES: ${{ matrix.pkg }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install node.js 8.x'
+        uses: actions/setup-node@v1
+        with:
+          node-version: '8.x'
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -yq \
+              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
+              valgrind libc6-dbg
+      - name: 'Install dependencies (Linux i386)'
+        if: runner.os == 'Linux' && matrix.cpu == 'i386'
+        run: |
+          sudo dpkg --add-architecture i386
+
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+              g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
+              libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 \
+              libffi-dev:i386
+
+          cat << EOF > bin/gcc
+          #!/bin/bash
+
+          exec $(which gcc) -m32 "\$@"
+          EOF
+          cat << EOF > bin/g++
+          #!/bin/bash
+
+          exec $(which g++) -m32 "\$@"
+          EOF
+
+          chmod 755 bin/gcc
+          chmod 755 bin/g++
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install boehmgc make sfml
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+      - name: 'Run CI'
+        shell: bash
+        run: ./koch runCI
+
+      - name: 'Show failed tests'
+        if: failure()
+        shell: bash
+        run: nim c -r tools/ci_testresults.nim

.github/workflows/ci_docs.yml

@@ -6,7 +6,6 @@ on:
       - 'compiler/renderverbatim.nim'
       - 'config/nimdoc.cfg'
       - 'doc/**.rst'
-      - 'doc/**.md'
       - 'doc/nimdoc.css'
       - 'lib/**.nim'
       - 'nimdoc/testproject/expected/testproject.html'
@@ -14,7 +13,6 @@ on:
       - 'tools/kochdocs.nim'
       - '.github/workflows/ci_docs.yml'
       - 'koch.nim'
-
   pull_request:
     # Run only on changes on these files.
     paths:
@@ -22,42 +20,36 @@ on:
       - 'compiler/renderverbatim.nim'
       - 'config/nimdoc.cfg'
       - 'doc/**.rst'
-      - 'doc/**.md'
       - 'doc/nimdoc.css'
       - 'lib/**.nim'
       - 'nimdoc/testproject/expected/testproject.html'
       - 'tools/dochack/dochack.nim'
       - 'tools/kochdocs.nim'
       - '.github/workflows/ci_docs.yml'
-      - 'koch.nim'
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
 
 jobs:
   build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
     strategy:
       fail-fast: false
       matrix:
         target: [linux, windows, osx]
         include:
           - target: linux
-            os: ubuntu-20.04
+            os: ubuntu-18.04
           - target: windows
             os: windows-2019
           - target: osx
-            os: macos-11
+            os: macos-10.15
 
     name: ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 60 # refs bug #18178
 
     steps:
       - name: 'Checkout'
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
+        uses: actions/checkout@v2
 
       - name: 'Install build dependencies (macOS)'
         if: runner.os == 'macOS'
@@ -67,33 +59,64 @@ jobs:
         if: runner.os == 'Windows'
         shell: bash
         run: |
-          set -e
-          . ci/funs.sh
-          nimInternalInstallDepsWindows
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
           echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'System information'
+      - name: 'Get current csources version'
+        id: csources-version
         shell: bash
-        run: . ci/funs.sh && nimCiSystemInfo
+        run: |
+          sha=$(git ls-remote https://github.com/nim-lang/csources_v1 master | cut -f 1)
+          echo "::set-output name=sha::$sha"
 
-      - name: 'Build csourcesAny (posix)'
-        # this would work on windows and other CI use this on windows,
-        # but we ensure here that `ci/build_autogen.bat` keeps working on windows.
-        if: runner.os != 'Windows'
+      - name: 'Get prebuilt csources from cache'
+        id: csources-cache
+        uses: actions/cache@v1
+        with:
+          path: bin
+          key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}'
+
+      - name: 'Checkout csources'
+        if: steps.csources-cache.outputs.cache-hit != 'true'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources_v1
+          path: csources
+
+      - name: 'Build 1-stage compiler from csources'
         shell: bash
-        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc
-          # was previously using caching via `actions/cache@v1` but this wasn't
-          # used in other CI pipelines and it's unclear the added complexity
-          # was worth the saving; can be revisited if needed.
+        run: |
+          ext=
+          [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe
+          if [[ ! -x bin/nim-csources$ext ]]; then
+            ncpu=
+            case '${{ runner.os }}' in
+            'Linux')
+              ncpu=$(nproc)
+              ;;
+            'macOS')
+              ncpu=$(sysctl -n hw.ncpu)
+              ;;
+            'Windows')
+              ncpu=$NUMBER_OF_PROCESSORS
+              ;;
+            esac
+            [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
 
-      - name: 'Build csourcesAny (windows)'
-        if: runner.os == 'Windows'
-        shell: cmd
-        run: ci/build_autogen.bat
+            make -C csources -j $ncpu CC=gcc
+            cp bin/nim{,-csources}$ext
+          else
+            echo 'Cache hit, using prebuilt csources'
+            cp bin/nim{-csources,}$ext
+          fi
 
       - name: 'Build koch'
         shell: bash
@@ -103,6 +126,10 @@ jobs:
         shell: bash
         run: ./koch boot -d:release
 
+      - name: 'Clone fusion'
+        shell: bash
+        run: ./koch fusion
+
       - name: 'Build documentation'
         shell: bash
         run: ./koch doc --git.commit:devel
@@ -111,7 +138,7 @@ jobs:
         if: |
           github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
           matrix.target == 'linux'
-        uses: crazy-max/ghaction-github-pages@v3
+        uses: crazy-max/ghaction-github-pages@v1
         with:
           build_dir: doc/html
         env:

.github/workflows/ci_packages.yml

@@ -1,41 +1,33 @@
 name: Packages CI
-on:
-  pull_request:
-  push:
-    branches:
-      - 'devel'
-      - 'version-1-6'
-      - 'version-1-2'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+on: [push, pull_request]
 
 jobs:
   build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-11]
+        os: [ubuntu-18.04, macos-10.15]
         cpu: [amd64]
-        batch: ["allowed_failures", "0_3", "1_3", "2_3"] # list of `index_num`
-    name: '${{ matrix.os }} (batch: ${{ matrix.batch }})'
+        pkg: [1, 2]
+    name: '${{ matrix.os }} (pkg: ${{ matrix.pkg }})'
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 60 # refs bug #18178
     env:
-      NIM_TEST_PACKAGES: "1"
-      NIM_TESTAMENT_BATCH: ${{ matrix.batch }}
+      NIM_TEST_PACKAGES: ${{ matrix.pkg }}
     steps:
       - name: 'Checkout'
         uses: actions/checkout@v2
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
         with:
-          fetch-depth: 2
+          repository: nim-lang/csources
+          path: csources
 
-      - name: 'Install node.js 16.x'
-        uses: actions/setup-node@v2
+      - name: 'Install node.js 12.x'
+        uses: actions/setup-node@v1
         with:
-          node-version: '16.x'
-
+          node-version: '12.x'
       - name: 'Install dependencies (Linux amd64)'
         if: runner.os == 'Linux' && matrix.cpu == 'amd64'
         run: |
@@ -51,23 +43,43 @@ jobs:
         if: runner.os == 'Windows'
         shell: bash
         run: |
-          set -e
-          . ci/funs.sh
-          nimInternalInstallDepsWindows
-          echo_run echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'System information'
+      - name: 'Build csources'
         shell: bash
-        run: . ci/funs.sh && nimCiSystemInfo
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
 
-      - name: 'Build csourcesAny'
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+      - name: 'Build koch'
         shell: bash
-        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
+        run: nim c koch
+      - name: 'Run CI'
+        shell: bash
+        run: ./koch runCI
 
-      - name: 'koch, Run CI'
+      - name: 'Show failed tests'
+        if: failure()
         shell: bash
-        run: . ci/funs.sh && nimInternalBuildKochAndRunCI
+        run: nim c -r tools/ci_testresults.nim

.github/workflows/ci_publish.yml

@@ -1,90 +0,0 @@
-name: Tracking orc-booting compiler memory usage
-
-on:
-  pull_request_target:
-    types: [closed]
-
-
-jobs:
-  build:
-    if: github.event.pull_request.merged == true
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-20.04]
-        cpu: [amd64]
-    name: '${{ matrix.os }}'
-    runs-on: ${{ matrix.os }}
-    steps:
-      - name: 'Checkout'
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-
-      - name: 'Install node.js 16.x'
-        uses: actions/setup-node@v2
-        with:
-          node-version: '16.x'
-
-      - name: 'Install dependencies (Linux amd64)'
-        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
-        run: |
-          sudo apt-fast update -qq
-          DEBIAN_FRONTEND='noninteractive' \
-            sudo apt-fast install --no-install-recommends -yq \
-              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
-              valgrind libc6-dbg libblas-dev xorg-dev
-      - name: 'Install dependencies (macOS)'
-        if: runner.os == 'macOS'
-        run: brew install boehmgc make sfml gtk+3
-      - name: 'Install dependencies (Windows)'
-        if: runner.os == 'Windows'
-        shell: bash
-        run: |
-          set -e
-          . ci/funs.sh
-          nimInternalInstallDepsWindows
-          echo_run echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
-
-      - name: 'Add build binaries to PATH'
-        shell: bash
-        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
-
-      - name: 'System information'
-        shell: bash
-        run: . ci/funs.sh && nimCiSystemInfo
-
-      - name: 'Build csourcesAny'
-        shell: bash
-        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
-
-      - name: 'Build koch'
-        shell: bash
-        run: nim c koch
-
-      - name: 'Build Nim'
-        shell: bash
-        run: ./koch boot -d:release -d:nimStrictMode --lib:lib
-
-      - name: 'Action'
-        shell: bash
-        run: nim c -r -d:release ci/action.nim
-
-      - name: 'Comment'
-        uses: actions/github-script@v6
-        with:
-          script: |
-            const fs = require('fs');
-
-            try {
-              const data = fs.readFileSync('ci/nimcache/results.txt', 'utf8');
-              github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: data
-            })
-            } catch (err) {
-              console.error(err);
-            }
-

.github/workflows/ci_ssl.yml

@@ -0,0 +1,93 @@
+name: Nim SSL CI
+on:
+  pull_request:
+    # Run only on changes on related files
+    paths:
+      - 'lib/pure/httpclient.nim'
+      - 'lib/pure/net.nim'
+      - 'lib/pure/ssl_certs.nim'
+      - 'lib/wrappers/openssl.nim'
+      - 'tests/stdlib/thttpclient_ssl*'
+      - 'tests/untestable/thttpclient_ssl*'
+
+jobs:
+  build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64]
+    name: '${{ matrix.os }} (${{ matrix.cpu }})'
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -y libssl1.1
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install make
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build 1-stage compiler from csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build the real compiler'
+        shell: bash
+        run: ./koch boot
+
+      - name: 'Run SSL nimDisableCertificateValidation integration tests'
+        shell: bash
+        run: nim c -d:nimDisableCertificateValidation -d:ssl -r -p:. tests/untestable/thttpclient_ssl_disabled.nim
+
+      - name: 'Run SSL certificate check integration tests'
+        # Not supported on Windows due to old openssl version
+        if: runner.os != 'Windows'
+        shell: bash
+        run: nim c -d:ssl -p:. --threads:on -r tests/untestable/thttpclient_ssl.nim

.gitignore

@@ -3,9 +3,9 @@
 !*.*
 
 # Cache
-nimcache*/
-rnimcache*/
-dnimcache*/
+nimcache/
+rnimcache/
+dnimcache/
 
 *.o
 !/icons/*.o
@@ -30,7 +30,7 @@ deinstall.sh
 
 doc/html/
 doc/*.html
-doc/pdf
+doc/*.pdf
 doc/*.idx
 /web/upload
 /build/*
@@ -64,29 +64,22 @@ lib/**/*.html
 testament.db
 /tests/**/*.json
 /tests/**/*.js
-
 /csources
-/csources_v1
-
 /dist/
-# /lib/fusion # fusion is now unbundled; `git status` should reveal if it's there so users can act on it
+/lib/fusion
 
 # Private directories and files (IDEs)
 .*/
 ~*
 
-# testament cruft; TODO: generate these in a gitignore'd dir (./build) in the first place.
+# testament cruft; TODO: generate these in a gitignore'd dir in the first place.
 testresults/
 test.txt
 /test.ini
 
 tweeter.db
 tweeter_test.db
-
-/tests/megatest.nim
-/tests/ic/*_temp.nim
-/tests/navigator/*_temp.nim
-
+megatest.nim
 
 /outputExpected.txt
 /outputGotten.txt
@@ -107,5 +100,3 @@ htmldocs
 
 ## these are not needed anymore unless checkout old older versions
 nimdoc.out.css
-# except here:
-!/nimdoc/testproject/expected/*

.gitlab-ci.yml

@@ -1,6 +1,4 @@
-# xxx unused, out of date
-
-image: ubuntu:18.04
+image: ubuntu:16.04
 
 stages:
   - pre-build
@@ -56,7 +54,7 @@ test-windows:
   script:
     - call ci\deps.bat
     - nim c testament\tester
-    - testament\tester.exe all
+    - testament\tester.exe --pedantic all
   tags:
     - windows
     - fast

.travis.yml

@@ -0,0 +1,52 @@
+sudo: false
+language: c
+
+dist: xenial
+
+matrix:
+  include:
+    - os: linux
+      env:
+        - NIM_COMPILE_TO_CPP=false
+        - CPU=amd64
+
+addons:
+  apt:
+    # update the list above if more deps are introduced
+    packages:
+    - libcurl4-openssl-dev
+    - libsdl1.2-dev
+    - libgc-dev
+    - libsfml-dev
+    - libc6-dbg
+    - valgrind
+
+before_script:
+  - git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
+  - export PATH="$PWD/bin${PATH:+:$PATH}"
+  - make -C csources -j 2 LD=$CC ucpu=$CPU
+
+script:
+  - echo "travis_fold:start:nim_c_koch"
+  - nim c koch
+  - echo "travis_fold:end:nim_c_koch"
+  - echo "travis_fold:start:koch_boot"
+  - ./koch boot
+  - echo "travis_fold:end:koch_boot"
+  - echo "travis_fold:start:koch_doc"
+  - ./koch doc
+  - echo "travis_fold:end:koch_doc"
+
+before_deploy:
+  # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html
+  - cp -f ./doc/html/overview.html ./doc/html/index.html
+
+deploy:                         # https://nim-lang.github.io/Nim
+  provider: pages
+  # local-dir *has* to be a relative path from the repo root.
+  local-dir: "doc/html"
+  skip-cleanup: true
+  github-token: "$GITHUB_OAUTH_TOKEN"
+  keep-history: false
+  on:
+    branch: devel

appveyor.yml

@@ -0,0 +1,37 @@
+version: '{build}'
+
+environment:
+  DLLS_URL: https://nim-lang.org/download/dlls.zip
+  DLLS_ARCHIVE: dlls.zip
+  MINGW_DIR: mingw64
+  MINGW_URL: https://nim-lang.org/download/mingw64.7z
+  MINGW_ARCHIVE: mingw64.7z
+
+  matrix:
+    - NIM_TEST_PACKAGES: false
+    - NIM_TEST_PACKAGES: true
+
+cache:
+    - '%MINGW_ARCHIVE%'
+    - '%DLLS_ARCHIVE%'
+
+
+install:
+  - ps: Install-Product node 8 # node 8 or later is required to test js async stuff
+  - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%"
+  - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\BIN"> nul
+  - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%"
+  - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul
+  - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH%
+  - git clone --depth 1 https://github.com/nim-lang/csources
+  - cd csources
+  - build64.bat
+  - cd ..
+
+build_script:
+  - openssl version
+  - openssl version -d
+  - bin\nim c koch
+  - koch runCI
+
+deploy: off

azure-pipelines.yml

@@ -1,17 +1,12 @@
 trigger:
   branches:
     include:
-    - 'devel'
-    - 'version-*'
+    - '*'
 pr:
   branches:
     include:
     - '*'
 
-variables:
-- name: skipci
-  value: false 
-
 jobs:
 - job: packages
 
@@ -20,19 +15,16 @@ jobs:
   strategy:
     matrix:
       Linux_amd64:
-        vmImage: 'ubuntu-20.04'
+        vmImage: 'ubuntu-16.04'
         CPU: amd64
-      # regularly breaks, refs bug #17325
       Linux_i386:
-        # on 'ubuntu-16.04' (not supported anymore anyways) it errored with:
-        # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed
-        vmImage: 'ubuntu-18.04'
+        vmImage: 'ubuntu-16.04'
         CPU: i386
       OSX_amd64:
-        vmImage: 'macOS-11'
+        vmImage: 'macOS-10.15'
         CPU: amd64
       OSX_amd64_cpp:
-        vmImage: 'macOS-11'
+        vmImage: 'macOS-10.15'
         CPU: amd64
         NIM_COMPILE_TO_CPP: true
       Windows_amd64_batch0_3:
@@ -58,54 +50,43 @@ jobs:
   steps:
     - bash: git config --global core.autocrlf false
       displayName: 'Disable auto conversion to CRLF by git (Windows-only)'
-      condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
 
     - checkout: self
-      fetchDepth: 2 # see D20210329T004830
+      fetchDepth: 1
 
-    - bash: |
-        set -e
-        . ci/funs.sh
-        if nimIsCiSkip; then
-          echo '##vso[task.setvariable variable=skipci]true'
-        fi
-      displayName: 'Check whether to skip CI'
+    - bash: git clone --depth 1 https://github.com/nim-lang/csources_v1 csources
+      displayName: 'Checkout Nim csources'
 
     - task: NodeTool@0
       inputs:
-        versionSpec: '16.x'
-      displayName: 'Install node.js 16.x'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
+        versionSpec: '12.x'
+      displayName: 'Install node.js 12.x'
 
     - bash: |
-        set -e
-        . ci/funs.sh
-        echo_run sudo apt-fast update -qq
+        sudo apt-fast update -qq
         DEBIAN_FRONTEND='noninteractive' \
-          echo_run sudo apt-fast install --no-install-recommends -yq \
+          sudo apt-fast install --no-install-recommends -yq \
             libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg
       displayName: 'Install dependencies (amd64 Linux)'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
 
     - bash: |
-        set -e
-        . ci/funs.sh
-        echo_run sudo dpkg --add-architecture i386
+        sudo dpkg --add-architecture i386
+
         # Downgrade llvm:
         # - llvm has to be downgraded to have 32bit version installed for sfml.
-
         cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
         Package: libllvm6.0
         Pin: origin "azure.archive.ubuntu.com"
         Pin-Priority: 1001
         EOF
 
-        # echo_run sudo apt-fast update -qq
-        echo_run sudo apt-fast update -qq || echo "failed, see bug #17343"
+        sudo apt-fast update -qq
         # `:i386` (e.g. in `libffi-dev:i386`) is needed otherwise you may get:
         # `could not load: libffi.so` during dynamic loading.
         DEBIAN_FRONTEND='noninteractive' \
-          echo_run sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+          sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
             g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
             libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 libffi-dev:i386
 
@@ -114,55 +95,86 @@ jobs:
 
         exec $(which gcc) -m32 "\$@"
         EOF
-
         cat << EOF > bin/g++
         #!/bin/bash
 
         exec $(which g++) -m32 "\$@"
         EOF
 
-        echo_run chmod 755 bin/gcc
-        echo_run chmod 755 bin/g++
+        chmod 755 bin/gcc
+        chmod 755 bin/g++
 
       displayName: 'Install dependencies (i386 Linux)'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
 
     - bash: brew install boehmgc make sfml
       displayName: 'Install dependencies (OSX)'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Darwin'))
+      condition: eq(variables['Agent.OS'], 'Darwin')
 
     - bash: |
-        set -e
-        . ci/funs.sh
-        nimInternalInstallDepsWindows
-        echo_run echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
+        mkdir dist
+        curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+        curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+        7z x dist/mingw64.7z -odist
+        7z x dist/dlls.zip -obin
+        echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
 
       displayName: 'Install dependencies (Windows)'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Windows_NT'))
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
 
     - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
       displayName: 'Add build binaries to PATH'
 
-    - bash: . ci/funs.sh && nimCiSystemInfo
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
+    - bash: |
+        echo 'PATH:' "$PATH"
+        echo '##[section]gcc version'
+        gcc -v
+        echo '##[section]nodejs version'
+        node -v
+        echo '##[section]make version'
+        make -v
       displayName: 'System information'
 
-    - bash: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu=$(CPU)
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'Build csourcesAny'
+    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
+      displayName: 'Get csources version'
 
-    # this could be revived if performance justifies it (needs a few updates)
-    # - task: Cache@2
-    #   inputs:
-    #     key: 'csourcesAny | "$(Agent.OS)" | $(CPU) | $(csources_version)'
-    #     path: $(nim_csources)
-    #   condition: and(succeeded(), eq(variables['skipci'], 'false'))
-    #   displayName: 'Restore built csourcesAny'
+    - task: Cache@2
+      inputs:
+        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+        path: csources/bin
+      displayName: 'Restore built csources'
 
-    - bash: . ci/funs.sh && nimInternalBuildKochAndRunCI
-      # would could also show on failure: echo '##vso[task.complete result=Failed]'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'koch, Run CI'
+    - bash: |
+        ncpu=
+        ext=
+        case '$(Agent.OS)' in
+        'Linux')
+          ncpu=$(nproc)
+          ;;
+        'Darwin')
+          ncpu=$(sysctl -n hw.ncpu)
+          ;;
+        'Windows_NT')
+          ncpu=$NUMBER_OF_PROCESSORS
+          ext=.exe
+          ;;
+        esac
+        [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+        if [[ -x csources/bin/nim$ext ]]; then
+          echo "Found cached compiler, skipping build"
+        else
+          make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
+        fi
+
+        cp csources/bin/nim$ext bin
+      displayName: 'Build 1-stage compiler from csources'
+
+    - bash: nim c koch
+      displayName: 'Build koch'
+
+      # set result to omit the "bash exited with error code '1'" message
+    - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
+      displayName: 'Run CI'
       env:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)

bin/nim-gdb

@@ -7,7 +7,7 @@ which nim > /dev/null || (echo "nim not in PATH"; exit 1)
 which gdb > /dev/null || (echo "gdb not in PATH"; exit 1)
 which readlink > /dev/null || (echo "readlink not in PATH."; exit 1)
 
-if [[ $(uname -s) == Darwin || $(uname -s) == *BSD ]]; then
+if [[ "$(uname -s)" == "Darwin" || "(uname -s)" == *"BSD" ]]; then
   NIM_SYSROOT=$(dirname $(dirname $(readlink -f $(which nim))))
 else
   NIM_SYSROOT=$(dirname $(dirname $(readlink -e $(which nim))))

bin/nim-gdb.bash

@@ -0,0 +1 @@
+nim-gdb

build_all.bat

@@ -1,29 +1,17 @@
 @echo off
-rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
-rem Build development version of the compiler; can be rerun safely
-rem bare bones version of ci/funs.sh adapted for windows.
-
-rem Read in some common shared variables (shared with other tools),
-rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file
-for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H
-SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe
-echo "building from csources: %nim_csources%"
-
-if not exist %nim_csourcesDir% (
-  git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir%
+rem build development version of the compiler; can be rerun safely
+if not exist csources (
+  git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 )
-
-if not exist %nim_csources% (
-  cd %nim_csourcesDir%
-  git checkout %nim_csourcesHash%
-  echo "%PROCESSOR_ARCHITECTURE%"
-  if "%PROCESSOR_ARCHITECTURE%"=="AMD64" (
+if not exist bin\nim.exe (
+  cd csources
+  if PROCESSOR_ARCHITECTURE == AMD64 (
     SET ARCH=64
   )
   CALL build.bat
   cd ..
-  copy /y bin\nim.exe  %nim_csources%
 )
- bin\nim.exe c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch
- koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
- koch tools --skipUserCfg --skipParentCfg --hints:off
+bin\nim.exe c --skipUserCfg --skipParentCfg koch
+koch.exe boot -d:release --skipUserCfg --skipParentCfg
+koch.exe tools --skipUserCfg --skipParentCfg
+

build_all.sh

@@ -1,17 +1,52 @@
-#!/bin/sh
-# DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
+#! /bin/sh
 
 # build development version of the compiler; can be rerun safely.
-# arguments can be passed, e.g.:
-# CC=gcc ucpu=amd64 uos=darwin
+# arguments can be passed, e.g. `--os freebsd`
 
 set -u # error on undefined variables
 set -e # exit on first error
 
-. ci/funs.sh
-nimBuildCsourcesIfNeeded "$@"
+echo_run(){
+  echo "$*"
+  "$@"
+}
 
-echo_run bin/nim c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch
-echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
-echo_run ./koch tools --skipUserCfg --skipParentCfg --hints:off
+[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources_v1.git csources
+
+nim_csources=bin/nim_csources
+
+build_nim_csources_via_script(){
+  echo_run cd csources
+  echo_run sh build.sh "$@"
+}
+
+build_nim_csources(){
+  # avoid changing dir in case of failure
+  (
+    if [ $# -ne 0 ]; then
+      # some args were passed (e.g.: `--cpu i386`), need to call build.sh
+      build_nim_csources_via_script "$@"
+    else
+      # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s)
+      makeX=make
+      unamestr=$(uname)
+      if [ "$unamestr" = 'FreeBSD' ]; then
+        makeX=gmake
+      fi
+      nCPU=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
+      which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script
+    fi
+  )
+  # keep $nim_csources in case needed to investigate bootstrap issues
+  # without having to rebuild from csources
+  echo_run cp bin/nim $nim_csources
+}
+
+[ -f $nim_csources ] || echo_run build_nim_csources $@
+
+# Note: if fails, may need to `cd csources && git pull`
+echo_run bin/nim c --skipUserCfg --skipParentCfg koch
+
+echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg
+echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools.
 

changelog.md

@@ -1,273 +1,83 @@
-# v2.0.0 - yyyy-mm-dd
+# v1.6.x - yyyy-mm-dd
+
 
 
 ## Changes affecting backward compatibility
-- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response, it followed Apache HttpClient(Java), http(go) and .Net HttpWebResponse(C#) behavior. Previously it raised `ValueError`.
 
-- `addr` is now available for all addressable locations,
-  `unsafeAddr` is now deprecated and an alias for `addr`.
+- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected.
+  The downside is that these defines now have custom logic that doesn't apply for
+  other defines.
 
-- Certain definitions from the default `system` module have been moved to
-  the following new modules:
-
-  - `std/syncio`
-  - `std/assertions`
-  - `std/formatfloat`
-  - `std/objectdollar`
-  - `std/widestrs`
-  - `std/typedthreads`
-  - `std/sysatomics`
-
-  In the future, these definitions will be removed from the `system` module,
-  and their respective modules will have to be imported to use them.
-  Currently, to make these imports required, the `-d:nimPreviewSlimSystem` option
-  may be used.
-
-- Enabling `-d:nimPreviewSlimSystem` also removes the following deprecated
-  symbols in the `system` module:
-  - Aliases with `Error` suffix to exception types that have a `Defect` suffix
-    (see [exceptions](https://nim-lang.org/docs/exceptions.html)):
-    `ArithmeticError`, `DivByZeroError`, `OverflowError`,
-    `AccessViolationError`, `AssertionError`, `OutOfMemError`, `IndexError`,
-    `FieldError`, `RangeError`, `StackOverflowError`, `ReraiseError`,
-    `ObjectAssignmentError`, `ObjectConversionError`, `FloatingPointError`,
-    `FloatOverflowError`, `FloatUnderflowError`, `FloatInexactError`,
-    `DeadThreadError`, `NilAccessError`
-  - `addQuitProc`, replaced by `exitprocs.addExitProc`
-  - Legacy unsigned conversion operations: `ze`, `ze64`, `toU8`, `toU16`, `toU32`
-  - `TaintedString`, formerly a distinct alias to `string`
-  - `PInt32`, `PInt64`, `PFloat32`, `PFloat64`, aliases to
-    `ptr int32`, `ptr int64`, `ptr float32`, `ptr float64`
-
-- Enabling `-d:nimPreviewSlimSystem` removes the import of `channels_builtin` in
-  in the `system` module.
-
-- Enabling `-d:nimPreviewCstringConversion`, `ptr char`, `ptr array[N, char]` and `ptr UncheckedArray[N, char]` don't support conversion to cstring anymore.
-
-- The `gc:v2` option is removed.
-
-- The `mainmodule` and `m` options are removed.
-
-- The `threads:on` option is now the default.
-
-- Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via
-  `experimental:flexibleOptionalParams`.
-
-- Automatic dereferencing (experimental feature) is removed.
-
-- The `Math.trunc` polyfill for targeting Internet Explorer was
-  previously included in most JavaScript output files.
-  Now, it is only included with `-d:nimJsMathTruncPolyfill`.
-  If you are targeting Internet Explorer, you may choose to enable this option
-  or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma).
-  Nim uses `Math.trunc` for the division and modulo operators for integers.
-
-- `shallowCopy` and `shallow` are removed for ARC/ORC. Use `move` when possible or combine assignment and
-`sink` for optimization purposes.
-
-- The `nimPreviewDotLikeOps` define is going to be removed or deprecated.
-
-- The `{.this.}` pragma, deprecated since 0.19, has been removed.
-- `nil` literals can no longer be directly assigned to variables or fields of `distinct` pointer types. They must be converted instead.
-  ```nim
-  type Foo = distinct ptr int
-
-  # Before:
-  var x: Foo = nil
-  # After:
-  var x: Foo = Foo(nil)
-  ```
-- Removed two type pragma syntaxes deprecated since 0.20, namely
-  `type Foo = object {.final.}`, and `type Foo {.final.} [T] = object`.
-
-- [Overloadable enums](https://nim-lang.github.io/Nim/manual.html#overloadable-enum-value-names) and Unicode Operators
-  are no longer experimental.
-
-- Removed the `nimIncrSeqV3` define.
-
-- `macros.getImpl` for `const` symbols now returns the full definition node
-  (as `nnkConstDef`) rather than the AST of the constant value.
-
-- Lock levels are deprecated, now a noop.
-
-- ORC is now the default memory management strategy. Use
-  `--mm:refc` for a transition period.
-
-- `strictEffects` are no longer experimental.
-  Use `legacy:laxEffects` to keep backward compatibility.
-
-- The `gorge`/`staticExec` calls will now return a descriptive message in the output
-  if the execution fails for whatever reason. To get back legacy behaviour use `-d:nimLegacyGorgeErrors`.
-
-- Pointer to `cstring` conversion now triggers a `[PtrToCstringConv]` warning.
-  This warning will become an error in future versions! Use a `cast` operation
-  like `cast[cstring](x)` instead.
-
-- `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`.
-
-- Object fields now support default values, see https://nim-lang.github.io/Nim/manual.html#types-default-values-for-object-fields for details.
-
-- Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead.
 
 ## Standard library additions and changes
+- Added support for parenthesized expressions in `strformat`
 
-[//]: # "Changes:"
-- OpenSSL 3 is now supported.
-- `macros.parseExpr` and `macros.parseStmt` now accept an optional
-  filename argument for more informative errors.
-- Module `colors` expanded with missing colors from the CSS color standard.
-  `colPaleVioletRed` and `colMediumPurple` have also been changed to match the CSS color standard.
-- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)).
-- The `md5` module now works at compile time and in JavaScript.
-- Changed `mimedb` to use an `OrderedTable` instead of `OrderedTableRef` to support `const` tables.
-- `strutils.find` now uses and defaults to `last = -1` for whole string searches,
-  making limiting it to just the first char (`last = 0`) valid.
-- `random.rand` now works with `Ordinal`s.
-- Undeprecated `os.isvalidfilename`.
-- `std/oids` now uses `int64` to store time internally (before it was int32).
-- `std/uri.Uri` dollar `$` improved, precalculates the `string` result length from the `Uri`.
-- `std/uri.Uri.isIpv6` is now exported.
-- `std/logging.ConsoleLogger` and `FileLogger` now have a `flushThreshold` attribute to set what log message levels are automatically flushed. For Nim v1 use `-d:nimFlushAllLogs` to automatically flush all message levels. Flushing all logs is the default behavior for Nim v2.
+- Fixed buffer overflow bugs in `net`
+
+- Added `sections` iterator in `parsecfg`.
+
+- Make custom op in macros.quote work for all statements.
+
+- On Windows the SSL library now checks for valid certificates.
+  It uses the `cacert.pem` file for this purpose which was extracted
+  from `https://curl.se/ca/cacert.pem`. Besides
+  the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
+  now also need to ship `cacert.pem` with your `.exe` file.
 
 
-- `std/net.IpAddress` dollar `$` improved, uses a fixed capacity for the `string` result based from the `IpAddressFamily`.
-- `std/jsfetch.newFetchOptions` now has default values for all parameters
-- `std/jsformdata` now accepts `Blob` data type.
+- Make `{.requiresInit.}` pragma to work for `distinct` types.
 
+- Added a macros `enumLen` for returning the number of items in an enum to the
+  `typetraits.nim` module.
 
-[//]: # "Additions:"
-- Added ISO 8601 week date utilities in `times`:
-  - Added `IsoWeekRange`, a range type for weeks in a week-based year.
-  - Added `IsoYear`, a distinct type for a week-based year in contrast to a regular year.
-  - Added a `initDateTime` overload to create a datetime from an ISO week date.
-  - Added `getIsoWeekAndYear` to get an ISO week number and week-based year from a datetime.
-  - Added `getIsoWeeksInYear` to return the number of weeks in a week-based year.
-- Added new modules which were part of `std/os`:
-  - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling.
-  - Added `std/paths`, `std/dirs`, `std/files`, `std/symlinks` and `std/appdirs`.
-- Added `sep` parameter in `std/uri` to specify the query separator.
-- Added bindings to [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift)
-  and [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask)
-  in `jscore` for JavaScript targets.
-- Added `UppercaseLetters`, `LowercaseLetters`, `PunctuationChars`, `PrintableChars` sets to `std/strutils`.
-- Added `complex.sgn` for obtaining the phase of complex numbers.
-- Added `insertAdjacentText`, `insertAdjacentElement`, `insertAdjacentHTML`,
-  `after`, `before`, `closest`, `append`, `hasAttributeNS`, `removeAttributeNS`,
-  `hasPointerCapture`, `releasePointerCapture`, `requestPointerLock`,
-  `replaceChildren`, `replaceWith`, `scrollIntoViewIfNeeded`, `setHTML`,
-  `toggleAttribute`, and `matches` to `std/dom`.
-- Added [`jsre.hasIndices`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/hasIndices)
-- Added `capacity` for `string` and `seq` to return the current capacity, see https://github.com/nim-lang/RFCs/issues/460
-- Added `openArray[char]` overloads for `std/parseutils` allowing more code reuse.
-- Added `openArray[char]` overloads for `std/unicode` allowing more code reuse.
-- Added `safe` parameter to `base64.encodeMime`.
+- `prelude` now works with the JavaScript target.
 
-[//]: # "Deprecations:"
-- Deprecated `selfExe` for Nimscript.
-- Deprecated `std/sums`.
-- Deprecated `std/base64.encode` for collections of arbitrary integer element type.
-  Now only `byte` and `char` are supported.
+- Added `ioutils` module containing `duplicate` and `duplicateTo` to duplicate `FileHandle` using C function `dup` and `dup2`.
 
-[//]: # "Removals:"
-- Removed deprecated module `parseopt2`.
-- Removed deprecated module `sharedstrings`.
-- Removed deprecated module `dom_extensions`.
-- Removed deprecated module `LockFreeHash`.
-- Removed deprecated module `events`.
-- Removed deprecated `oids.oidToString`.
-- Removed define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`.
-- Removed deprecated `jsre.test` and `jsre.toString`.
-- Removed deprecated `math.c_frexp`.
-- Removed deprecated `` httpcore.`==` ``.
-- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types.
-- Removed deprecated `osproc.poDemon`, symbol with typo.
+- The JSON module can now handle integer literals and floating point literals of arbitrary length and precision.
+  Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are kept as string literals and
+  one can use external BigNum libraries to handle these. The `parseFloat` family of functions also has now optional
+  `rawIntegers` and `rawFloats` parameters that can be used to enforce that all integer or float literals remain
+  in the "raw" string form so that client code can easily treat small and large numbers uniformly.
+
+- Added `randState` template that exposes the default random number generator. Useful for library authors.
+
+- Added `asyncdispatch.activeDescriptors` that returns the number of currently
+  active async event handles/file descriptors
+
+- Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place.
 
 ## Language changes
 
-- [Tag tracking](https://nim-lang.github.io/Nim/manual.html#effect-system-tag-tracking) supports the definition of forbidden tags by the `.forbids` pragma
-  which can be used to disable certain effects in proc types.
-- [Case statement macros](https://nim-lang.github.io/Nim/manual.html#macros-case-statement-macros) are no longer experimental,
-  meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them.
-- Full command syntax and block arguments i.e. `foo a, b: c` are now allowed
-  for the right-hand side of type definitions in type sections. Previously
-  they would error with "invalid indentation".
-- `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`.
-  In the command line, this is defined as `-d:a.b.c`. Older versions can
-  use accents as in ``defined(`a.b.c`)`` to access such defines.
-- [Macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas) changes:
-  - Templates now accept macro pragmas.
-  - Macro pragmas for var/let/const sections have been redesigned in a way that works
-    similarly to routine macro pragmas. The new behavior is documented in the
-    [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#extended-macro-pragmas).
-  - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`,
-    allowing multiple type definitions to be injected in place of the original type definition.
+- `nimscript` now handles `except Exception as e`
 
-    ```nim
-    import macros
-    macro multiply(amount: static int, s: untyped): untyped =
-      let name = $s[0].basename
-      result = newNimNode(nnkTypeSection)
-      for i in 1 .. amount:
-        result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2]))
-    type
-      Foo = object
-      Bar {.multiply: 3.} = object
-        x, y, z: int
-      Baz = object
-    # becomes
-    type
-      Foo = object
-      Bar1 = object
-        x, y, z: int
-      Bar2 = object
-        x, y, z: int
-      Bar3 = object
-        x, y, z: int
-      Baz = object
-    ```
+- The `cstring` doesn't support `[]=` operator in JS backend.
 
-- Redefining templates with the same signature implicitly was previously
-  allowed to support certain macro code. A `{.redefine.}` pragma has been
-  added to make this work explicitly, and a warning is generated in the case
-  where it is implicit. This behavior only applies to templates, redefinition
-  is generally disallowed for other symbols.
-
-- A new form of type inference called [top-down inference](https://nim-lang.github.io/Nim/manual_experimental.html#topminusdown-type-inference)
-  has been implemented for a variety of basic cases. For example, code like the following now compiles:
-
-  ```nim
-  let foo: seq[(float, byte, cstring)] = @[(1, 2, "abc")]
-  ```
-
-- `cstring` is now accepted as a selector in `case` statements, removing the
-  need to convert to `string`. On the JS backend, this is translated directly
-  to a `switch` statement.
-
-- Nim now supports `out` parameters and ["strict definitions"](https://nim-lang.github.io/Nim/manual_experimental.html#strict-definitions-and-nimout-parameters).
-- Nim now offers a [strict mode](https://nim-lang.github.io/Nim/manual_experimental.html#strict-case-objects) for `case objects`.
 
 
 ## Compiler changes
 
-- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the
-  reality better. (Nim moved away from all techniques based on "tracing".)
 
-- Defines the `gcRefc` symbol which allows writing specific code for the refc GC.
 
-- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`,
-  without requiring `-d:nimVersion140` which is now a noop.
+- Added `unsafeIsolate` and `extract` to `std/isolation`.
 
-- `--styleCheck`, `--hintAsError` and `--warningAsError` now only apply to the current package.
+- `--hint:CC` now goes to stderr (like all other hints) instead of stdout.
 
-- The switch `--nimMainPrefix:prefix` has been added to add a prefix to the names of `NimMain` and
-  related functions produced on the backend. This prevents conflicts with other Nim
-  static libraries.
+- json build instructions are now generated in `$nimcache/outFileBasename.json`
+  instead of `$nimcache/projectName.json`. This allows avoiding recompiling a given project
+  compiled with different options if the output file differs.
 
-- When compiling for Release the flag `-fno-math-errno` is used for GCC.
+- `--usenimcache` (implied by `nim r main`) now generates an output file that includes a hash of
+  some of the compilation options, which allows caching generated binaries:
+  nim r main # recompiles
+  nim r -d:foo main # recompiles
+  nim r main # uses cached binary
+  nim r main arg1 arg2 # ditto (runtime arguments are irrelevant)
 
+- The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch
+  enforces that every symbol is written as it was declared, not enforcing
+  the official Nim style guide. To be enabled, this has to be combined either
+  with `--styleCheck:error` or `--styleCheck:hint`.
 
 ## Tool changes
 
-- Nim now supports Nimble version 0.14 which added support for lock-files. This is done by
-  a simple configuration change setting that you can do yourself too. In `$nim/config/nim.cfg`
-  replace `pkgs` by `pkgs2`.

changelogs/changelog.md

@@ -1,33 +0,0 @@
-# v1.xx.x - yyyy-mm-dd
-
-## Changes affecting backward compatibility
-
-## Standard library additions and changes
-
-- `std/sharedlist` and `std/sharedtables` are now deprecated, see RFC [#433](https://github.com/nim-lang/RFCs/issues/433).
-
-### New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall
-
-This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17.
-
-On linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall.
-
-When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be:
-```sh
-$ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details
-$ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8
-$ sh build.sh  # per unix install instructions
-$ bin/nim c koch  # per unix install instructions
-$ ./koch boot -d:release  # per unix install instructions
-$ ./koch tools -d:nimNoGetRandom  # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall
-```
-
-This is necessary to pass when building nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for stdlib (sysrand in particular).
-
-## Language changes
-
-
-## Compiler changes
-
-
-## Tool changes

changelogs/changelog_0_19_0.md

@@ -15,7 +15,7 @@
   at runtime. Use ``ref object`` with inheritance rather than ``object`` with
   inheritance to prevent this issue.
 - The ``not nil`` type annotation now has to be enabled explicitly
-  via ``{.experimental: "notnil".}`` as we are still not pleased with how this
+  via ``{.experimental: "notnil"}`` as we are still not pleased with how this
   feature works with Nim's containers.
 - The parser now warns about inconsistent spacing around binary operators as
   these can easily be confused with unary operators. This warning will likely

changelogs/changelog_1_4_0.md

@@ -282,7 +282,7 @@
 - Removed the deprecated `asyncdispatch.newAsyncNativeSocket`.
 - Removed the deprecated `dom.releaseEvents` and `dom.captureEvents`.
 
-- Removed `sharedlist.initSharedList`, was deprecated and produces undefined behaviour.
+- Removed `sharedlists.initSharedList`, was deprecated and produces undefined behaviour.
 
 - There is a new experimental feature called "strictFuncs" which makes the definition of
   `.noSideEffect` stricter. [See here](manual_experimental.html#stricts-funcs)

changelogs/changelog_1_6_0.md

@@ -1,957 +0,0 @@
-# v1.6.0 - 2021-10-14
-
-
-
-# Major new features
-
-With so many new features, pinpointing the most salient ones is a subjective exercise,
-but here are a select few:
-
-
-## `iterable[T]`
-
-The `iterable[T]` type class was added to match called iterators,
-which solves a number of long-standing issues related to iterators.
-Example:
-
-```nim
-iterator iota(n: int): int =
-  for i in 0..<n: yield i
-
-# previously, you'd need `untyped`, which caused other problems such as lack
-# of type inference, overloading issues, and MCS.
-template sumOld(a: untyped): untyped = # no type inference possible
-  var result: typeof(block:(for ai in a: ai))
-  for ai in a: result += ai
-  result
-
-assert sumOld(iota(3)) == 0 + 1 + 2
-
-# now, you can write:
-template sum[T](a: iterable[T]): T =
-  # `template sum(a: iterable): auto =` would also be possible
-  var result: T
-  for ai in a: result += ai
-  result
-
-assert sum(iota(3)) == 0 + 1 + 2 # or `iota(3).sum`
-```
-
-In particular iterable arguments can now be used with the method call syntax. For example:
-```nim
-import std/[sequtils, os]
-echo walkFiles("*").toSeq # now works
-```
-See PR [#17196](https://github.com/nim-lang/Nim/pull/17196) for additional details.
-
-
-## Strict effects
-
-The effect system was refined and there is a new `.effectsOf` annotation that does
-explicitly what was previously done implicitly. See the
-[manual](https://nim-lang.github.io/Nim/manual.html#effect-system-effectsof-annotation)
-for more details.
-To write code that is portable with older Nim versions, use this idiom:
-
-```nim
-when defined(nimHasEffectsOf):
-  {.experimental: "strictEffects".}
-else:
-  {.pragma: effectsOf.}
-
-proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.}
-```
-
-To enable the new effect system, compile with `--experimental:strictEffects`.
-See also [#18777](https://github.com/nim-lang/Nim/pull/18777) and RFC
-[#408](https://github.com/nim-lang/RFCs/issues/408).
-
-
-## Private imports and private field access
-
-A new import syntax `import foo {.all.}` now allows importing all symbols
-(public or private) from `foo`.
-This can be useful for testing purposes or for more flexibility in project organization.
-
-Example:
-```nim
-from system {.all.} as system2 import nil
-echo system2.ThisIsSystem # ThisIsSystem is private in `system`
-import os {.all.} # weirdTarget is private in `os`
-echo weirdTarget # or `os.weirdTarget`
-```
-
-Added a new module `std/importutils`, and an API `privateAccess`, which allows access
-to private fields for an object type in the current scope.
-
-Example:
-```nim
-import times
-from std/importutils import privateAccess
-block:
-  let t = now()
-  # echo t.monthdayZero # Error: undeclared field: 'monthdayZero' for type times.DateTime
-  privateAccess(typeof(t)) # enables private access in this scope
-  echo t.monthdayZero # ok
-```
-
-See PR [#17706](https://github.com/nim-lang/Nim/pull/17706) for additional details.
-
-
-## `nim --eval:cmd`
-
-Added `nim --eval:cmd` to evaluate a command directly, e.g.: `nim --eval:"echo 1"`.
-It defaults to `e` (nimscript) but can also work with other commands, e.g.:
-```bash
-find . | nim r --eval:'import strutils; for a in stdin.lines: echo a.toUpper'
-```
-
-```bash
-# use as a calculator:
-nim --eval:'echo 3.1 / (1.2+7)'
-# explore a module's APIs, including private symbols:
-nim --eval:'import os {.all.}; echo weirdTarget'
-# use a custom backend:
-nim r -b:js --eval:"import std/jsbigints; echo 2'big ** 64'big"
-```
-
-See PR [#15687](https://github.com/nim-lang/Nim/pull/15687) for more details.
-
-
-## Round-trip float to string
-
-`system.addFloat` and `system.$` now can produce string representations of
-floating point numbers that are minimal in size and possess round-trip and correct
-rounding guarantees (via the
-[Dragonbox](https://raw.githubusercontent.com/jk-jeon/dragonbox/master/other_files/Dragonbox.pdf) algorithm).
-This currently has to be enabled via `-d:nimPreviewFloatRoundtrip`.
-It is expected that this behavior becomes the new default in upcoming versions,
-as with other `nimPreviewX` define flags.
-
-Example:
-```nim
-from math import round
-let a = round(9.779999999999999, 2)
-assert a == 9.78
-echo a # with `-d:nimPreviewFloatRoundtrip`: 9.78, like in python3 (instead of  9.779999999999999)
-```
-
-
-## New `std/jsbigints` module
-
-Provides arbitrary precision integers for the JS target. See PR
-[#16409](https://github.com/nim-lang/Nim/pull/16409).
-Example:
-```nim
-import std/jsbigints
-assert 2'big ** 65'big == 36893488147419103232'big
-echo 0xdeadbeef'big shl 4'big # 59774856944n
-```
-
-
-## New `std/sysrand` module
-Cryptographically secure pseudorandom number generator,
-allows generating random numbers from a secure source provided by the operating system.
-Example:
-```nim
-import std/sysrand
-assert urandom(1234) != urandom(1234) # unlikely to fail in practice
-```
-See PR [#16459](https://github.com/nim-lang/Nim/pull/16459).
-
-
-## New module: `std/tempfiles`
-
-Allows creating temporary files and directories, see PR
-[#17361](https://github.com/nim-lang/Nim/pull/17361) and followups.
-```nim
-import std/tempfiles
-let tmpPath = genTempPath("prefix", "suffix.log", "/tmp/")
-# tmpPath looks like: /tmp/prefixpmW1P2KLsuffix.log
-
-let dir = createTempDir("tmpprefix_", "_end")
-# created dir looks like: getTempDir() / "tmpprefix_YEl9VuVj_end"
-
-let (cfile, path) = createTempFile("tmpprefix_", "_end.tmp")
-# path looks like: getTempDir() / "tmpprefix_FDCIRZA0_end.tmp"
-cfile.write "foo"
-cfile.setFilePos 0
-assert readAll(cfile) == "foo"
-close cfile
-assert readFile(path) == "foo"
-```
-
-
-## User-defined literals
-
-Custom numeric literals (e.g. `-128'bignum`) are now supported.
-Additionally, the unary minus in `-1` is now part of the integer literal, i.e.
-it is now parsed as a single token.
-This implies that edge cases like `-128'i8` finally work correctly.
-Example:
-```nim
-func `'big`*(num: cstring): JsBigInt {.importjs: "BigInt(#)".}
-assert 0xffffffffffffffff'big == (1'big shl 64'big) - 1'big
-```
-
-
-## Dot-like operators
-
-With `-d:nimPreviewDotLikeOps`, dot-like operators (operators starting with `.`,
-but not with `..`) now have the same precedence as `.`, so that `a.?b.c` is now
-parsed as `(a.?b).c` instead of `a.?(b.c)`.
-A warning is generated when a dot-like operator is used without `-d:nimPreviewDotLikeOps`.
-
-An important use case is to enable dynamic fields without affecting the
-built-in `.` operator, e.g. for `std/jsffi`, `std/json`, `pkg/nimpy`. Example:
-```nim
-import std/json
-template `.?`(a: JsonNode, b: untyped{ident}): JsonNode =
-  a[astToStr(b)]
-let j = %*{"a1": {"a2": 10}}
-assert j.?a1.?a2.getInt == 10
-```
-
-
-## Block arguments now support optional parameters
-
-This solves a major pain point for routines accepting block parameters,
-see PR [#18631](https://github.com/nim-lang/Nim/pull/18631) for details:
-
-```nim
-template fn(a = 1, b = 2, body) = discard
-fn(1, 2): # already works
-  bar
-fn(a = 1): # now works
-  bar
-```
-
-Likewise with multiple block arguments via `do`:
-```nim
-template fn(a = 1, b = 2, body1, body2) = discard
-fn(a = 1): # now works
-  bar1
-do:
-  bar2
-```
-
-
-# Other new features
-
-## New and deprecated modules
-
-The following modules were added (they are discussed in the rest of the text):
-- `std/enumutils`
-- `std/genasts`
-- `std/importutils`
-- `std/jsbigints`
-- `std/jsfetch`
-- `std/jsformdata`
-- `std/jsheaders`
-- `std/packedsets`
-- `std/setutils`
-- `std/socketstreams`
-- `std/strbasics`
-- `std/sysrand`
-- `std/tasks`
-- `std/tempfiles`
-- `std/vmutils`
-
-- Deprecated `std/mersenne`.
-- Removed deprecated `std/iup` module from stdlib; it has already moved to
-  [nimble](https://github.com/nim-lang/iup).
-
-
-## New `std/jsfetch` module
-
-Provides a wrapper for JS Fetch API.
-Example:
-```nim
-# requires -d:nimExperimentalAsyncjsThen
-import std/[jsfetch, asyncjs, jsconsole, jsffi, sugar]
-proc fn {.async.} =
-  await fetch("https://api.github.com/users/torvalds".cstring)
-    .then((response: Response) => response.json())
-    .then((json: JsObject) => console.log(json))
-    .catch((err: Error) => console.log("Request Failed", err))
-discard fn()
-```
-
-
-## New `std/tasks` module
-
-Provides basic primitives for creating parallel programs.
-Example:
-```nim
-import std/tasks
-var num = 0
-proc hello(a: int) = num+=a
-
-let b = toTask hello(13) # arguments must be isolated, see `std/isolation`
-b.invoke()
-assert num == 13
-b.invoke() # can be called again
-assert num == 26
-```
-
-
-## New module: `std/genasts`
-
-Provides an API `genAst` that avoids the problems inherent with `quote do` and can
-be used as a replacement.
-Example showing how this could be used for writing a simplified version of `unittest.check`:
-```nim
-import std/[genasts, macros, strutils]
-macro check2(cond: bool): untyped =
-  assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind
-  result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]):
-    # each local symbol we access must be explicitly captured
-    if not cond:
-      doAssert false, "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs]
-let a = 3
-check2 a*2 == a+3
-if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4'
-```
-
-See PR [#17426](https://github.com/nim-lang/Nim/pull/17426) for details.
-
-
-## New module: `std/setutils`
-
-- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`,
-  if the iterable yields a built-in settable type.
-- Added `setutils.fullSet` which returns a full built-in `set` for a valid type.
-- Added `setutils.complement` which returns the complement of a built-in `set`.
-- Added `setutils.[]=`.
-
-
-## New module: `std/enumutils`
-
-- Added `genEnumCaseStmt` macro that generates
-  case statement to parse string to enum.
-- Added `items` for enums with holes.
-- Added `symbolName` to return the `enum` symbol name ignoring the human-readable name.
-- Added `symbolRank` to return the index in which an `enum` member is listed in an enum.
-
-
-## `system`
-
-- Added `system.prepareMutation` for better support of low
-  level `moveMem`, `copyMem` operations for `gc:orc`'s copy-on-write string
-  implementation.
-- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible
-  with most other languages.
-- Added `cmpMem` to `system`.
-- `doAssertRaises` now correctly handles foreign exceptions.
-- `addInt` now supports unsigned integers.
-
-Compatibility notes:
-- `system.delete` had surprising behavior when the index passed to it was out of
-  bounds (it would delete the last entry then). Compile with `-d:nimStrictDelete` so
-  that an index error is produced instead. Be aware however that your code might depend on
-  this quirky behavior so a review process is required on your part before you can
-  use `-d:nimStrictDelete`. To make this review easier, use the `-d:nimAuditDelete`
-  switch, which pretends that `system.delete` is deprecated so that it is easier
-  to see where it was used in your code.
-  `-d:nimStrictDelete` will become the default in upcoming versions.
-- `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`.
-  Please use `char` or `uint8` instead.
-- `repr` now doesn't insert trailing newlines; the previous behavior was very inconsistent,
-  see [#16034](https://github.com/nim-lang/Nim/pull/16034).
-  Use `-d:nimLegacyReprWithNewline` for the previous behavior.
-  `repr` now also renders ASTs correctly for user defined literals, setters, `do`, etc.
-- Deprecated `any`. See RFC [#281](https://github.com/nim-lang/RFCs/issues/281).
-- The unary slice `..b` was deprecated, use `0..b` instead.
-
-
-## `std/math`
-
-- Added `almostEqual` for comparing two float values using a machine epsilon.
-- Added `clamp` which allows using a `Slice` to clamp to a value.
-- Added `ceilDiv` for integer division that rounds up.
-- Added `isNaN`.
-- Added `copySign`.
-- Added `euclDiv` and `euclMod`.
-- Added `signbit`.
-- Added `frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead.
-
-Compatibility notes:
-- `math.round` now rounds "away from zero" in the JS backend, which is consistent
-  with other backends. See [#9125](https://github.com/nim-lang/Nim/pull/9125).
-  Use `-d:nimLegacyJsRound` for the previous behavior.
-
-
-## Random number generators: `std/random`, `std/sysrand`, `std/oids`
-
-- Added `std/sysrand` module (see details above).
-- Added `randState` template that exposes the default random number generator.
-  Useful for library authors.
-- Added `initRand()` overload with no argument which uses the current time as a seed.
-- `initRand(seed)` now allows `seed == 0`.
-- Fixed overflow bugs.
-- Fix `initRand` to avoid random number sequences overlapping, refs
-  [#18744](https://github.com/nim-lang/Nim/pull/18744).
-- `std/oids` now uses `std/random`.
-
-Compatibility notes:
-- Deprecated `std/mersenne`.
-- `random.initRand(seed)` now produces non-skewed values for the first call to
-  `rand()` after initialization with a small (< 30000) seed.
-  Use `-d:nimLegacyRandomInitRand` to restore previous behavior for a transition
-  time, see PR [#17467](https://github.com/nim-lang/Nim/pull/17467).
-
-
-## `std/json`, `std/jsonutils`
-
-- With `-d:nimPreviewJsonutilsHoleyEnum`, `jsonutils` now can serialize/deserialize
-  holey enums as regular enums (via `ord`) instead of as strings.
-  It is expected that this behavior becomes the new default in upcoming versions.
- `toJson` now serializes `JsonNode` as is via reference (without a deep copy)
-   instead of treating `JsonNode` as a regular ref object,
-  this can be customized via `jsonNodeMode`.
-- `std/json` and `std/jsonutils` now serialize `NaN`, `Inf`, `-Inf` as strings,
-  so that `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]`
-  which was invalid JSON.
-- `std/json` can now handle integer literals and floating point literals of
-  arbitrary length and precision.
-  Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are
-  kept as string literals and one can use external BigNum libraries to handle these.
-  The `parseFloat` family of functions also has now optional `rawIntegers` and
-  `rawFloats` parameters that can be used to enforce that all integer or float
-  literals remain in the "raw" string form so that client code can easily treat
-  small and large numbers uniformly.
-- Added `BackwardsIndex` overload for `JsonNode`.
-- `json.%`,`json.to`, `jsonutils.fromJson`,`jsonutils.toJson` now work with `uint|uint64`
-  instead of raising (as in 1.4) or giving wrong results (as in 1.2).
-- `std/jsonutils` now handles `cstring` (including as Table key), and `set`.
-- Added `jsonutils.jsonTo` overload with `opt = Joptions()` param.
-- `jsonutils.toJson` now supports customization via `ToJsonOptions`.
-- `std/json`, `std/jsonutils` now support round-trip serialization when
-  `-d:nimPreviewFloatRoundtrip` is used.
-
-
-## `std/typetraits`, `std/compilesettings`
-
-- `distinctBase` now is identity instead of error for non distinct types.
-- `distinctBase` now allows controlling whether to be recursive or not.
-- Added `enumLen` to return the number of elements in an enum.
-- Added `HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes.
-- Added `hasClosure`.
-- Added `pointerBase` to return `T` for `ref T | ptr T`.
-- Added `compilesettings.SingleValueSetting.libPath`.
-
-
-## networking: `std/net`, `std/asyncnet`, `std/htmlgen`, `std/httpclient`, `std/asyncdispatch`, `std/asynchttpserver`, `std/httpcore`
-
-- Fixed buffer overflow bugs in `std/net`.
-- Exported `sslHandle` from `std/net` and `std/asyncnet`.
-- Added `hasDataBuffered` to `std/asyncnet`.
-- Various functions in `std/httpclient` now accept `url` of type `Uri`.
-  Moreover, the `request` function's `httpMethod` argument of type `string` was
-  deprecated in favor of `HttpMethod` `enum` type; see
-  [#15919](https://github.com/nim-lang/Nim/pull/15919).
-- Added `asyncdispatch.activeDescriptors` that returns the number of currently
-  active async event handles/file descriptors.
-- Added `getPort` to `std/asynchttpserver` to resolve OS-assigned `Port(0)`;
-  this is usually recommended instead of hardcoding ports which can lead to
-  "Address already in use" errors.
-- Fixed premature garbage collection in `std/asyncdispatch`, when a stacktrace
-  override is in place.
-- Added `httpcore.is1xx` and missing HTTP codes.
-- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals).
-
-Compatibility notes:
-- On Windows, the SSL library now checks for valid certificates.
-  For this purpose it uses the `cacert.pem` file, which was extracted
-  from `https://curl.se/ca/cacert.pem`. Besides
-  the OpenSSL DLLs (e.g. `libssl-1_1-x64.dll`, `libcrypto-1_1-x64.dll`) you
-  now also need to ship `cacert.pem` with your `.exe` file.
-
-
-## `std/hashes`
-
-- `hashes.hash` can now support `object` and `ref` (can be overloaded in user code),
-  if `-d:nimPreviewHashRef` is used. It is expected that this behavior
-  becomes the new default in upcoming versions.
-- `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`.
-  `hashes.hash(closure)` has also been improved.
-
-
-## OS: `std/os`, `std/io`, `std/socketstream`, `std/linenoise`, `std/tempfiles`
-
-- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`,
-  determining preferred I/O block size for this file object.
-- Added `os.getCacheDir()` to return platform specific cache directory.
-- Improved `os.getTempDir()`, see PR [#16914](https://github.com/nim-lang/Nim/pull/16914).
-- Added `os.isAdmin` to tell whether the caller's process is a member of the
-  Administrators local group (on Windows) or a root (on POSIX).
-- Added optional `options` argument to `copyFile`, `copyFileToDir`, and
-  `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are
-  followed (copy files symlinks point to); on Windows, `options` argument is
-  ignored and symlinks are skipped.
-- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as
-  symlinks (instead of skipping them as it was before); on Windows symlinks are
-  skipped.
-- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks
-  (instead of skipping them sometimes as it was before).
-- Added optional `followSymlinks` argument to `setFilePermissions`.
-- Added a simpler to use `io.readChars` overload.
-- Added `socketstream` module that wraps sockets in the stream interface.
-- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C).
-
-
-## Environment variable handling
-
-- Empty environment variable values are now supported across OS's and backends.
-- Environment variable APIs now work in multithreaded scenarios, by delegating to
-  direct OS calls instead of trying to keep track of the environment.
-- `putEnv`, `delEnv` now work at compile time.
-- NodeJS backend now supports osenv: `getEnv`, `putEnv`, `envPairs`, `delEnv`, `existsEnv`.
-
-Compatibility notes:
-- `std/os`: `putEnv` now raises if the first argument contains a `=`.
-
-
-## POSIX
-
-- On POSIX systems, the default signal handlers used for Nim programs (it's
-  used for printing the stacktrace on fatal signals) will now re-raise the
-  signal for the OS default handlers to handle.
-  This lets the OS perform its default actions, which might include core
-  dumping (on select signals) and notifying the parent process about the cause
-  of termination.
-- On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler`
-  for previous behavior.
-- Added `posix_utils.osReleaseFile` to get system identification from `os-release`
-  file on Linux and the BSDs.
-  ([link](https://www.freedesktop.org/software/systemd/man/os-release.html))
-- Removed undefined behavior for `posix.open`.
-
-
-## `std/prelude`
-
-- `std/strformat` is now part of `include std/prelude`.
-- Added `std/sequtils` import to `std/prelude`.
-- `std/prelude` now works with the JS target.
-- `std/prelude` can now be used via `include std/prelude`, but `include prelude` still works.
-
-
-## String manipulation: `std/strformat`, `std/strbasics`
-
-- Added support for parenthesized expressions.
-- Added support for const strings instead of just string literals.
-- Added `std/strbasics` for high-performance string operations.
-- Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`.
-
-
-## `std/wrapnils`
-
-- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding
-  issues like bug [#13063](https://github.com/nim-lang/Nim/issues/13063)
-  (which affected error messages) for modules importing `std/wrapnils`.
-- Added `??.` macro which returns an `Option`.
-- `std/wrapnils` can now be used to protect against `FieldDefect` errors in
-  case objects, generates optimal code (no overhead compared to manual
-  if-else branches), and preserves lvalue semantics which allows modifying
-  an expression.
-
-
-## Containers: `std/algorithm`, `std/lists`, `std/sequtils`, `std/options`, `std/packedsets`
-
-- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix`
-  iterators (it never worked reliably).
-- Added `algorithm.merge`.
-- In `std/lists`: renamed `append` to `add` and retained `append` as an alias;
-  added `prepend` and `prependMoved` analogously to `add` and `addMoved`;
-  added `remove` for `SinglyLinkedList`s.
-- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList`
-  and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements
-  shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes
-  its argument, `addMoved`, is also supplied.
-  See PRs [#16362](https://github.com/nim-lang/Nim/pull/16362),
-  [#16536](https://github.com/nim-lang/Nim/pull/16536).
-- New module: `std/packedsets`.
-  Generalizes `std/intsets`, see PR [#15564](https://github.com/nim-lang/Nim/pull/15564).
-
-Compatibility notes:
-- Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises
-  a defect if the slice is out of bounds, likewise with `strutils.delete`.
-- Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]`
-  in `std/algorithm`.
-- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"`
-  and `$none(int)` to `"none(int)"` instead of `"None[int]"`.
-
-
-## `std/times`
-
-- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time
-  zone offsets without colons, e.g. `UTC+7 -> +0700`.
-- Added `dateTime` and deprecated `initDateTime`.
-
-
-## `std/macros` and AST
-
-- New module `std/genasts`, see description above.
-- The required name of case statement macros for the experimental
-  `caseStmtMacros` feature has changed from `match` to `` `case` ``.
-- Tuple expressions are now parsed consistently as
-  `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`.
-- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes
-  in a collapsed way.
-  Use `-d:nimLegacyMacrosCollapseSymChoice` to get the previous behavior.
-- Made custom op in `macros.quote` work for all statements.
-
-
-## `std/sugar`
-
-- Added `sugar.dumpToString` which improves on `sugar.dump`.
-- Added an overload for the `collect` macro that infers the container type based
-  on the syntax of the last expression. Works with std seqs, tables and sets.
-
-Compatibility notes:
-- Removed support for named procs in `sugar.=>`.
-
-
-## Parsing: `std/parsecfg`, `std/strscans`, `std/uri`
-
-- Added `sections` iterator in `parsecfg`.
-- `strscans.scanf` now supports parsing single characters.
-- Added `strscans.scanTuple` which uses `strscans.scanf` internally,
-  returning a tuple which can be unpacked for easier usage of `scanf`.
-- Added `decodeQuery` to `std/uri`.
-- `parseopt.initOptParser` has been made available and `parseopt` has been
-  added back to `std/prelude` for all backends. Previously `initOptParser` was
-  unavailable if the `std/os` module did not have `paramCount` or `paramStr`,
-  but the use of these in `initOptParser` were conditionally to the runtime
-  arguments passed to it, so `initOptParser` has been changed to raise
-  `ValueError` when the real command line is not available. `parseopt` was
-  previously excluded from `std/prelude` for JS, as it could not be imported.
-
-Compatibility notes:
-- Changed the behavior of `uri.decodeQuery` when there are unencoded `=`
-  characters in the decoded values. Prior versions would raise an error. This is
-  no longer the case to comply with the HTML spec and other languages'
-  implementations. Old behavior can be obtained with
-  `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same
-  underlying code is also updated the same way.
-
-
-## JS stdlib changes
-
-- Added `std/jsbigints` module, which provides arbitrary precision integers for the JS target.
-- Added `setCurrentException` for the JS backend.
-- `writeStackTrace` is available in the JS backend now.
-- Added `then`, `catch` to `std/asyncjs` for promise pipelining, for now hidden
-  behind `-d:nimExperimentalAsyncjsThen`.
-- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API)
-  wrapper for the JS target.
-- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers)
-  wrapper for the JS target.
-- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData)
-  wrapper for the JS target.
-- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger).
-- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`.
-- Added dollar `$` and `len` for `jsre.RegExp`.
-- Added `jsconsole.jsAssert` for the JS target.
-- Added `**` to `std/jsffi`.
-- Added `copyWithin` [for `seq` and `array` for JS targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin).
-- In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`.
-  Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated.
-- Added `dom.scrollIntoView` proc with options.
-- Added `dom.setInterval`, `dom.clearInterval` overloads.
-- Merged `std/dom_extensions` into the `std/dom` module,
-  as it was a module with a single line, see RFC [#413](https://github.com/nim-lang/RFCs/issues/413).
-- `$` now gives more correct results on the JS backend.
-
-
-## JS compiler changes
-
-- `cstring` doesn't support the `[]=` operator anymore in the JS backend.
-- Array literals now use JS typed arrays when the corresponding JS typed array exists,
-  for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`.
-
-
-## VM and nimscript backend
-
-- VM now supports `addr(mystring[ind])` (index + index assignment).
-- `nimscript` now handles `except Exception as e`.
-- `nil` dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time.
-- `static[T]` now works better, refs [#17590](https://github.com/nim-lang/Nim/pull/17590),
-  [#15853](https://github.com/nim-lang/Nim/pull/15853).
-- `distinct T` conversions now work in VM.
-- `items(cstring)` now works in VM.
-- Fix `addr`, `len`, `high` in VM ([#16002](https://github.com/nim-lang/Nim/pull/16002),
-  [#16610](https://github.com/nim-lang/Nim/pull/16610)).
-- `std/cstrutils` now works in VM.
-
-
-## OS/platform-specific notes
-
-- Support for Apple silicon/M1.
-- Support for 32-bit RISC-V, refs [#16231](https://github.com/nim-lang/Nim/pull/16231).
-- Support for armv8l, refs [#18901](https://github.com/nim-lang/Nim/pull/18901).
-- Support for CROSSOS, refs [#18889](https://github.com/nim-lang/Nim/pull/18889).
-- The allocator for Nintendo Switch, which was nonfunctional because
-  of breaking changes in libnx, was removed, in favor of the new `-d:nimAllocPagesViaMalloc` option.
-- Allow reading parameters when compiling for Nintendo Switch.
-- `--nimcache` now correctly works in a cross-compilation setting.
-- Cross compilation targeting Windows was improved.
-- This now works from macOS/Linux: `nim r -d:mingw main`
-
-
-
-## Performance / memory optimizations
-
-- The comment field in PNode AST was moved to a side channel, reducing overall
-  memory usage during compilation by a factor 1.25x
-- `std/jsonutils` deserialization is now up to 20x faster.
-- `os.copyFile` is now 2.5x faster on macOS, by using `copyfile` from `copyfile.h`;
-  use `-d:nimLegacyCopyFile` for macOS < 10.5.
-- Float to string conversion is now 10x faster thanks to the Dragonbox algorithm,
-  with `-d:nimPreviewFloatRoundtrip`.
-- `newSeqWith` is 3x faster.
-- CI now supports batching (making Windows CI 2.3X faster).
-- Sets now uses the optimized `countSetBits` proc, see PR
-  [#17334](https://github.com/nim-lang/Nim/pull/17334).
-
-
-## Debugging
-
-- You can now enable/disable VM tracing in user code via `vmutils.vmTrace`.
-- `koch tools` now builds `bin/nim_dbg` which allows easy access to a debug version
-  of Nim without recompiling.
-- Added new module `compiler/debugutils` to help with debugging Nim compiler.
-- Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints` and
-  used it in more contexts; this flag which works in tandem with `--stackTraceMsgs`
-  to show user code context in compiler stacktraces.
-
-
-## Type system
-
-- `typeof(voidStmt)` now works and returns `void`.
-- `enum` values can now be overloaded. This needs to be enabled
-  via `{.experimental: "overloadableEnums".}`. We hope that this feature allows
-  for the development of more fluent (less ugly) APIs.
-  See RFC [#373](https://github.com/nim-lang/RFCs/issues/373) for more details.
-- A type conversion from one `enum` type to another now produces an `[EnumConv]` warning.
-  You should use `ord` (or `cast`, but the compiler won't help, if you misuse it) instead.
-  ```nim
-  type A = enum a1, a2
-  type B = enum b1, b2
-  echo a1.B # produces a warning
-  echo a1.ord.B # produces no warning
-  ```
-- A dangerous implicit conversion to `cstring` now triggers a `[CStringConv]` warning.
-  This warning will become an error in future versions! Use an explicit conversion
-  like `cstring(x)` in order to silence the warning.
-- There is a new warning for *any* type conversion to `enum` that can be enabled via
-  `.warning[AnyEnumConv]:on` or `--warning:AnyEnumConv:on`.
-- Reusing a type name in a different scope now works, refs [#17710](https://github.com/nim-lang/Nim/pull/17710).
-- Fixed implicit and explicit generics in procedures, refs [#18808](https://github.com/nim-lang/Nim/pull/18808).
-
-
-## New-style concepts
-
-Example:
-```nim
-type
-  Comparable = concept # no T, an atom
-    proc cmp(a, b: Self): int
-```
-The new design does not rely on `system.compiles` and may compile faster.
-See PR [#15251](https://github.com/nim-lang/Nim/pull/15251)
-and RFC [#168](https://github.com/nim-lang/RFCs/issues/168) for details.
-
-
-## Lexical / syntactic
-
-- Nim now supports a small subset of Unicode operators as operator symbols.
-  The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔".
-  To enable this feature, use `--experimental:unicodeOperators`. Note that due
-  to parser limitations you **cannot** enable this feature via a
-  pragma `{.experimental: "unicodeOperators".}` reliably, you need to enable
-  it via the command line or in a configuration file.
-- `var a {.foo.} = expr` now works inside templates (except when `foo` is overloaded).
-
-
-## Compiler messages, error messages, hints, warnings
-
-- Significant improvement to error messages involving effect mismatches,
-  see PRs [#18384](https://github.com/nim-lang/Nim/pull/18384),
-  [#18418](https://github.com/nim-lang/Nim/pull/18418).
-- Added `--declaredLocs` to show symbol declaration location in error messages.
-- Added `--spellSuggest` to show spelling suggestions on typos.
-- Added `--processing:dots|filenames|off` which customizes `hintProcessing`;
-  `--processing:filenames` shows which include/import modules are being compiled as an import stack.
-- `FieldDefect` messages now shows discriminant value + lineinfo, in all backends (C, JS, VM)
-- Added `--hintAsError` with similar semantics as `--warningAsError`.
-- Added `--unitsep:on|off` to control whether to add ASCII unit separator `\31`
-  before a newline for every generated message (potentially multiline),
-  so tooling can tell when messages start and end.
-- Added `--filenames:abs|canonical|legacyRelProj` which replaces `--listFullPaths:on|off`
-- `--hint:all:on|off` is now supported to select or deselect all hints; it
-  differs from `--hints:on|off` which acts as a (reversible) gate.
-  Likewise with `--warning:all:on|off`.
-- The style checking of the compiler now supports a `--styleCheck:usages` switch.
-  This switch enforces that every symbol is written as it was declared, not enforcing
-  the official Nim style guide. To be enabled, this has to be combined either
-  with `--styleCheck:error` or `--styleCheck:hint`.
-- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for
-  previous behavior.
-- `typedesc[Foo]` now renders as such instead of `type Foo` in compiler messages.
-- `runnableExamples` now show originating location in stacktraces on failure.
-- `SuccessX` message now shows more useful information.
-- New `DuplicateModuleImport` warning; improved `UnusedImport` and
-  `XDeclaredButNotUsed` accuracy.
-
-Compatibility notes:
-- `--hint:CC` now prints to stderr (like all other hints) instead of stdout.
-
-
-## Building and running Nim programs, configuration system
-
-- JSON build instructions are now generated in `$nimcache/outFileBasename.json`
-  instead of `$nimcache/projectName.json`. This allows avoiding recompiling a
-  given project compiled with different options if the output file differs.
-- `--usenimcache` (implied by `nim r main`) now generates an output file that includes
-  a hash of some of the compilation options, which allows caching generated binaries:
-  ```bash
-    nim r main # recompiles
-    nim r -d:foo main # recompiles
-    nim r main # uses cached binary
-    nim r main arg1 arg2 # likewise (runtime arguments are irrelevant)
-  ```
-- `nim r` now supports cross compilation from unix to windows when specifying
-  `-d:mingw` by using Wine, e.g.:
-  `nim r --eval:'import os; echo "a" / "b"'` prints `a\b`.
-- `nim` can compile version 1.4.0 as follows:
-  `nim c --lib:lib --stylecheck:off -d:nimVersion140 compiler/nim`.
-  `-d:nimVersion140` is not needed for bootstrapping, only for building 1.4.0 from devel.
-- `nim e` now accepts arbitrary file extensions for the nimscript file,
-  although `.nims` is still the preferred extension in general.
-- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected.
-  The downside is that these defines now have custom logic that doesn't apply for
-  other defines.
-
-
-## Multithreading
-
-- TLS: macOS now uses native TLS (`--tlsEmulation:off`). TLS now works with
-  `importcpp` non-POD types; such types must use `.cppNonPod` and
-  `--tlsEmulation:off`should be used.
-- Added `unsafeIsolate` and `extract` to `std/isolation`.
-- Added `std/tasks`, see description above.
-
-
-## Memory management
-
-- `--gc:arc` now bootstraps (PR [#17342](https://github.com/nim-lang/Nim/pull/17342)).
-- Lots of improvements to `gc:arc`, `gc:orc`, see PR
-  [#15697](https://github.com/nim-lang/Nim/pull/15697),
-  [#16849](https://github.com/nim-lang/Nim/pull/16849),
-  [#17993](https://github.com/nim-lang/Nim/pull/17993).
-- `--gc:orc` is now 10% faster than previously for common workloads.
-  If you have trouble with its changed behavior, compile with `-d:nimOldOrc`.
-- The `--gc:orc` algorithm was refined so that custom container types can participate in the
-  cycle collection process. See the documentation of `=trace` for more details.
-- On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`.
-  This is only supported for `--gc:orc` or `--gc:arc`.
-
-Compatibility notes:
-- `--newruntime` and `--refchecks` are deprecated,
-  use `--gc:arc`, `--gc:orc`, or `--gc:none` as appropriate instead.
-
-
-## Docgen
-
-- docgen: RST files can now use single backticks instead of double backticks and
-  correctly render in both `nim rst2html` (as before) as well as common tools rendering
-  RST directly (e.g. GitHub).
-  This is done by adding the `default-role:: code` directive inside the RST file
-  (which is now handled by `nim rst2html`).
-- Source+Edit links now appear on top of every docgen'd page when
-  `nim doc --git.url:url ...` is given.
-- Latex doc generation is revised: output `.tex` files should be compiled
-  by `xelatex` (not by `pdflatex` as before). Now default Latex settings
-  provide support for Unicode and better avoid margin overflows.
-  The minimum required version is TeXLive 2018 (or an equivalent MikTeX version).
-- The RST parser now supports footnotes, citations, admonitions, and short style
-  references with symbols.
-- The RST parser now supports Markdown table syntax.
-  Known limitations:
-  - cell alignment is not supported, i.e. alignment annotations in a delimiter
-    row (`:---`, `:--:`, `---:`) are ignored
-  - every table row must start with `|`, e.g. `| cell 1 | cell 2 |`.
-- Implemented `doc2tex` compiler command which converts documentation in
-  `.nim` files to Latex.
-- docgen now supports syntax highlighting for inline code.
-- docgen now supports same-line doc comments:
-  ```nim
-  func fn*(a: int): int = 42  ## Doc comment
-  ```
-- docgen now renders `deprecated` and other pragmas.
-- `runnableExamples` now works with templates and nested templates.
-- `runnableExamples: "-r:off"` now works for examples that should compile but not run.
-- `runnableExamples` now renders code verbatim, and produces correct code in all cases.
-- docgen now shows correct, canonical import paths in docs.
-- docgen now shows all routines in sidebar, and the proc signature is now shown in sidebar.
-
-
-## Effects and checks
-
-- Significant improvement to error messages involving effect mismatches
-- There is a new `cast` section `{.cast(uncheckedAssign).}: body` that disables some
-  compiler checks regarding `case objects`. This allows serialization libraries
-  to avoid ugly, non-portable solutions. See RFC
-  [#407](https://github.com/nim-lang/RFCs/issues/407) for more details.
-
-Compatibility notes:
-- Fixed effect tracking for borrowed procs (see [#18882](https://github.com/nim-lang/Nim/pull/18882)).
-  One consequence is that, under some circumstances, Nim could previously permit a procedure with side effects to be written with `func` - you may need to change some occurrences of `func` to `proc`.
-  To illustrate, Nim versions before 1.6.0 compile the below without error
-  ```nim
-  proc print(s: string) =
-    echo s
-
-  type
-    MyString = distinct string
-
-  proc print(s: MyString) {.borrow.}
-
-  func foo(s: MyString) =
-    print(s)
-  ```
-  but Nim 1.6.0 produces the error
-  ```
-  Error: 'foo' can have side effects
-  ```
-  similar to how we expect that
-  ```nim
-  func print(s: string) =
-    echo s
-  ```
-  produces
-  ```
-  Error: 'print' can have side effects
-  ```
-
-
-## Tools
-
-- Major improvements to `nimgrep`, see PR [#15612
-](https://github.com/nim-lang/Nim/pull/15612).
-- `fusion` is now un-bundled from Nim, `./koch fusion` will
-  install it via Nimble at a fixed hash.
-- `testament`: added `nimoutFull: bool` spec to compare full output of compiler
-  instead of a subset; many bugfixes to testament.
-
-
-## Misc/cleanups
-
-- Deprecated `TaintedString` and `--taintmode`.
-- Deprecated `--nilseqs` which is now a noop.
-- Added `-d:nimStrictMode` in CI in several places to ensure code doesn't have
-  certain hints/warnings.
-- Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`.
-- `[skip ci]` now works in azure and CI pipelines, see detail in PR
-  [#17561](https://github.com/nim-lang/Nim/pull/17561).

changelogs/changelog_X_XX_X.md

@@ -3,16 +3,13 @@
 This is an example file.
 The changes should go to changelog.md!
 
-## Changes affecting backward compatibility
-
-- `foo` now behaves differently, use `-d:nimLegacyFoo` for previous behavior.
 
 ## Standard library additions and changes
 
 - Added `example.exampleProc`.
-
 - Changed `example.foo` to take additional `bar` parameter.
 
+
 ## Language changes
 
 

ci/action.nim

@@ -1,26 +0,0 @@
-import std/[strutils, os, osproc, parseutils, strformat]
-
-
-proc main() =
-  var msg = ""
-  const cmd = "./koch boot --gc:orc -d:release"
-
-  let (output, exitCode) = execCmdEx(cmd)
-
-  doAssert exitCode == 0, output
-
-  let start = rfind(output, "Hint: mm")
-  doAssert parseUntil(output, msg, "; proj", start) > 0, output
-
-  let (commitHash, _) = execCmdEx("""git log --format="%H" -n 1""")
-
-  let welcomeMessage = fmt"""Thanks for your hard work on this PR!
-The lines below are statistics of the Nim compiler built from {commitHash}
-
-{msg}
-"""
-  createDir "ci/nimcache"
-  writeFile "ci/nimcache/results.txt", welcomeMessage
-
-when isMainModule:
-  main()

ci/build.bat

@@ -0,0 +1,14 @@
+REM Some debug info
+echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%."
+gcc -v
+
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
+cd csources
+call build64.bat
+cd ..
+set PATH=%CD%\bin;%PATH%
+nim -v
+nim c koch
+koch.exe boot
+copy bin/nim bin/nimd
+koch.exe boot -d:release

ci/build.sh

@@ -0,0 +1,15 @@
+sh ci/deps.sh
+
+# Build from C sources.
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
+cd csources
+sh build.sh
+cd ..
+# Add Nim to the PATH
+export PATH=$(pwd)/bin${PATH:+:$PATH}
+# Bootstrap.
+nim -v
+nim c koch
+./koch boot
+cp bin/nim bin/nimd
+./koch boot -d:release

ci/build_autogen.bat

@@ -1,26 +0,0 @@
-@echo off
-rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
-rem Build development version of the compiler; can be rerun safely
-rem bare bones version of ci/funs.sh adapted for windows.
-
-rem Read in some common shared variables (shared with other tools),
-rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file
-for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H
-SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe
-echo "building from csources: %nim_csources%"
-
-if not exist %nim_csourcesDir% (
-  git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir%
-)
-
-if not exist %nim_csources% (
-  cd %nim_csourcesDir%
-  git checkout %nim_csourcesHash%
-  echo "%PROCESSOR_ARCHITECTURE%"
-  if "%PROCESSOR_ARCHITECTURE%"=="AMD64" (
-    SET ARCH=64
-  )
-  CALL build.bat
-  cd ..
-  copy /y bin\nim.exe  %nim_csources%
-)

ci/deps.bat

@@ -0,0 +1,4 @@
+nim e install_nimble.nims
+nim e tests/test_nimscript.nims
+nimble update
+nimble install -y zip opengl sdl1 jester@#head niminst

ci/deps.sh

@@ -0,0 +1,16 @@
+# Some debug info
+echo "Running on $CI_RUNNER_ID ($CI_RUNNER_DESCRIPTION) with tags $CI_RUNNER_TAGS."
+
+# Packages
+apt-get update -qq
+apt-get install -y -qq build-essential git libcurl4-openssl-dev libsdl1.2-dev libgc-dev nodejs
+
+gcc -v
+
+export PATH=$(pwd)/bin${PATH:+:$PATH}
+
+# Nimble deps
+nim e install_nimble.nims
+nim e tests/test_nimscript.nims
+nimble update
+nimble install zip opengl sdl1 jester@#head niminst

ci/funs.sh

@@ -1,143 +0,0 @@
-# Utilities used in CI pipelines and tooling to avoid duplication.
-# Avoid top-level statements.
-# Prefer nim scripts whenever possible.
-# functions starting with `_` are considered internal, less stable.
-
-echo_run () {
-  # echo's a command before running it, which helps understanding logs
-  echo ""
-  echo "cmd: $@" # in azure we could also use this: echo '##[section]"$@"'
-  "$@"
-}
-
-nimGetLastCommit() {
-  git log --no-merges -1 --pretty=format:"%s"
-}
-
-nimIsCiSkip(){
-  # D20210329T004830:here refs https://github.com/microsoft/azure-pipelines-agent/issues/2944
-  # `--no-merges` is needed to avoid merge commits which occur for PR's.
-  # $(Build.SourceVersionMessage) is not helpful
-  # nor is `github.event.head_commit.message` for github actions.
-  # Note: `[skip ci]` is now handled automatically for github actions, see https://github.blog/changelog/2021-02-08-github-actions-skip-pull-request-and-push-workflows-with-skip-ci/
-  commitMsg=$(nimGetLastCommit)
-  echo commitMsg: "$commitMsg"
-  if [[ $commitMsg == *"[skip ci]"* ]]; then
-    echo "skipci: true"
-    return 0
-  else
-    echo "skipci: false"
-    return 1
-  fi
-}
-
-nimInternalInstallDepsWindows(){
-  echo_run mkdir dist
-  echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
-  echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
-  echo_run 7z x dist/mingw64.7z -odist
-  echo_run 7z x dist/dlls.zip -obin
-}
-
-nimInternalBuildKochAndRunCI(){
-  echo_run nim c koch
-  if ! echo_run ./koch runCI; then
-    echo_run echo "runCI failed"
-    echo_run nim r tools/ci_testresults.nim
-    return 1
-  fi
-}
-
-nimDefineVars(){
-  . config/build_config.txt
-  nim_csources=bin/nim_csources_$nim_csourcesHash
-}
-
-_nimNumCpu(){
-  # linux: $(nproc)
-  # FreeBSD | macOS: $(sysctl -n hw.ncpu)
-  # OpenBSD: $(sysctl -n hw.ncpuonline)
-  # windows: $NUMBER_OF_PROCESSORS ?
-  echo $(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
-}
-
-_nimBuildCsourcesIfNeeded(){
-  # if some systems cannot use make or gmake, we could add support for calling `build.sh`
-  # but this is slower (not parallel jobs) and would require making build.sh
-  # understand the arguments passed to the makefile (e.g. `CC=gcc ucpu=amd64 uos=darwin`),
-  # instead of `--cpu amd64 --os darwin`.
-  unamestr=$(uname)
-  # uname values: https://en.wikipedia.org/wiki/Uname
-  if [ "$unamestr" = 'FreeBSD' ]; then
-    makeX=gmake
-  elif [ "$unamestr" = 'OpenBSD' ]; then
-    makeX=gmake
-  elif [ "$unamestr" = 'NetBSD' ]; then
-    makeX=gmake
-  elif [ "$unamestr" = 'CROSSOS' ]; then
-    makeX=gmake
-  elif [ "$unamestr" = 'SunOS' ]; then
-    makeX=gmake
-  else
-    makeX=make
-  fi
-  nCPU=$(_nimNumCpu)
-  echo_run which $makeX
-  # parallel jobs (5X faster on 16 cores: 10s instead of 50s)
-  echo_run $makeX -C $nim_csourcesDir -j $((nCPU + 2)) -l $nCPU "$@"
-  # keep $nim_csources in case needed to investigate bootstrap issues
-  # without having to rebuild
-  echo_run cp bin/nim $nim_csources
-}
-
-nimCiSystemInfo(){
-  nimDefineVars
-  echo_run eval echo '$'nim_csources
-  echo_run pwd
-  echo_run date
-  echo_run uname -a
-  echo_run git log --no-merges -1 --pretty=oneline
-  echo_run eval echo '$'PATH
-  echo_run gcc -v
-  echo_run node -v
-  echo_run make -v
-}
-
-nimCsourcesHash(){
-  nimDefineVars
-  echo $nim_csourcesHash
-}
-
-nimBuildCsourcesIfNeeded(){
-  # goal: allow cachine each tagged version independently
-  # to avoid rebuilding, so that tools like `git bisect`
-  # can grab a cached past version without rebuilding.
-  nimDefineVars
-  (
-    set -e
-    # avoid polluting caller scope with internal variable definitions.
-    if test -f "$nim_csources"; then
-      echo "$nim_csources exists."
-    else
-      if test -d "$nim_csourcesDir"; then
-        echo "$nim_csourcesDir exists."
-      else
-        # Note: using git tags would allow fetching just what's needed, unlike git hashes, e.g.
-        # via `git clone -q --depth 1 --branch $tag $nim_csourcesUrl`.
-        echo_run git clone -q --depth 1 -b $nim_csourcesBranch \
-            $nim_csourcesUrl "$nim_csourcesDir"
-        # old `git` versions don't support -C option, using `cd` explicitly:
-        echo_run cd "$nim_csourcesDir"
-        echo_run git checkout $nim_csourcesHash
-        echo_run cd "$OLDPWD"
-        # if needed we could also add: `git reset --hard $nim_csourcesHash`
-      fi
-      _nimBuildCsourcesIfNeeded "$@"
-    fi
-
-    echo_run rm -f bin/nim
-      # fixes bug #17913, but it's unclear why it's needed, maybe specific to MacOS Big Sur 11.3 on M1 arch?
-    echo_run cp $nim_csources bin/nim
-    echo_run $nim_csources -v
-  )
-}

ci/nsis_build.bat

@@ -0,0 +1,59 @@
+REM - Run the full testsuite;  testament\tester all
+
+REM - Uncomment the list of changes in news.txt
+REM - write a news ticker entry
+REM - Update the version
+
+REM - Generate the full docs;  koch web0
+REM - Generate the installers;
+REM - Update the version in system.nim
+REM - Test the installers
+REM - Tag the release
+REM - Merge devel into master
+REM - Update csources
+
+set NIMVER=%1
+
+Rem Build -docs file:
+koch web0
+cd web\upload
+7z a -tzip docs-%NIMVER%.zip *.html
+move /y docs-%NIMVER%.zip download
+cd ..\..
+
+Rem Build csources
+koch csources -d:release || exit /b
+
+rem Grab C sources and nimsuggest
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
+
+set PATH=%CD%\bin;%PATH%
+
+ReM Build Win32 version:
+
+set PATH=C:\Users\araq\projects\mingw32\bin;%PATH%
+cd csources
+call build.bat
+cd ..
+ReM Rebuilding koch is necessary because it uses its pointer size to determine
+ReM which mingw link to put in the NSIS installer.
+nim c --out:koch_temp koch || exit /b
+koch_temp boot -d:release || exit /b
+koch_temp nsis -d:release || exit /b
+koch_temp zip -d:release || exit /b
+dir build
+move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x32.exe || exit /b
+move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x32.zip || exit /b
+
+
+ReM Build Win64 version:
+set PATH=C:\Users\araq\projects\mingw64\bin;%PATH%
+cd csources
+call build64.bat
+cd ..
+nim c --out:koch_temp koch || exit /b
+koch_temp boot -d:release || exit /b
+koch_temp nsis -d:release || exit /b
+koch_temp zip -d:release || exit /b
+move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x64.exe || exit /b
+move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x64.zip || exit /b

compiler.nimble

@@ -0,0 +1,9 @@
+
+version = system.NimVersion
+author = "Andreas Rumpf"
+description = "Compiler package providing the compiler sources as a library."
+license = "MIT"
+
+installDirs = @["compiler", "nimsuggest"]
+
+requires "nim >= 0.14.0"

compiler/aliasanalysis.nim

@@ -1,128 +0,0 @@
-
-import ast
-
-import std / assertions
-
-const
-  PathKinds0* = {nkDotExpr, nkCheckedFieldExpr,
-                 nkBracketExpr, nkDerefExpr, nkHiddenDeref,
-                 nkAddr, nkHiddenAddr,
-                 nkObjDownConv, nkObjUpConv}
-  PathKinds1* = {nkHiddenStdConv, nkHiddenSubConv}
-
-proc skipConvDfa*(n: PNode): PNode =
-  result = n
-  while true:
-    case result.kind
-    of nkObjDownConv, nkObjUpConv:
-      result = result[0]
-    of PathKinds1:
-      result = result[1]
-    else: break
-
-proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
-  var n = orig
-  while true:
-    case n.kind
-    of PathKinds0 - {nkHiddenDeref, nkDerefExpr}:
-      n = n[0]
-    of PathKinds1:
-      n = n[1]
-    of nkHiddenDeref, nkDerefExpr:
-      # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
-      # pointer indirection.
-      # bug #14159, we cannot reason about sinkParam[].location as it can
-      # still be shared for tyRef.
-      n = n[0]
-      return n.kind == nkSym and n.sym.owner == owner and
-         (n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
-    else: break
-  # XXX Allow closure deref operations here if we know
-  # the owner controlled the closure allocation?
-  result = n.kind == nkSym and n.sym.owner == owner and
-    {sfGlobal, sfThread, sfCursor} * n.sym.flags == {} and
-    (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar)
-  # Note: There is a different move analyzer possible that checks for
-  # consume(param.key); param.key = newValue  for all paths. Then code like
-  #
-  #   let splited = split(move self.root, x)
-  #   self.root = merge(splited.lower, splited.greater)
-  #
-  # could be written without the ``move self.root``. However, this would be
-  # wrong! Then the write barrier for the ``self.root`` assignment would
-  # free the old data and all is lost! Lesson: Don't be too smart, trust the
-  # lower level C++ optimizer to specialize this code.
-
-type AliasKind* = enum
-  yes, no, maybe
-
-proc aliases*(obj, field: PNode): AliasKind =
-  # obj -> field:
-  # x -> x: true
-  # x -> x.f: true
-  # x.f -> x: false
-  # x.f -> x.f: true
-  # x.f -> x.v: false
-  # x -> x[]: true
-  # x[] -> x: false
-  # x -> x[0]: true
-  # x[0] -> x: false
-  # x[0] -> x[0]: true
-  # x[0] -> x[1]: false
-  # x -> x[i]: true
-  # x[i] -> x: false
-  # x[i] -> x[i]: maybe; Further analysis could make this return true when i is a runtime-constant
-  # x[i] -> x[j]: maybe; also returns maybe if only one of i or j is a compiletime-constant
-  template collectImportantNodes(result, n) =
-    var result: seq[PNode]
-    var n = n
-    while true:
-      case n.kind
-      of PathKinds0 - {nkDotExpr, nkBracketExpr, nkDerefExpr, nkHiddenDeref}:
-        n = n[0]
-      of PathKinds1:
-        n = n[1]
-      of nkDotExpr, nkBracketExpr, nkDerefExpr, nkHiddenDeref:
-        result.add n
-        n = n[0]
-      of nkSym:
-        result.add n
-        break
-      else: return no
-
-  collectImportantNodes(objImportantNodes, obj)
-  collectImportantNodes(fieldImportantNodes, field)
-
-  # If field is less nested than obj, then it cannot be part of/aliased by obj
-  if fieldImportantNodes.len < objImportantNodes.len: return no
-
-  result = yes
-  for i in 1..objImportantNodes.len:
-    # We compare the nodes leading to the location of obj and field
-    # with each other.
-    # We continue until they diverge, in which case we return no, or
-    # until we reach the location of obj, in which case we do not need
-    # to look further, since field must be part of/aliased by obj now.
-    # If we encounter an element access using an index which is a runtime value,
-    # we simply return maybe instead of yes; should further nodes not diverge.
-    let currFieldPath = fieldImportantNodes[^i]
-    let currObjPath = objImportantNodes[^i]
-
-    if currFieldPath.kind != currObjPath.kind:
-      return no
-
-    case currFieldPath.kind
-    of nkSym:
-      if currFieldPath.sym != currObjPath.sym: return no
-    of nkDotExpr:
-      if currFieldPath[1].sym != currObjPath[1].sym: return no
-    of nkDerefExpr, nkHiddenDeref:
-      discard
-    of nkBracketExpr:
-      if currFieldPath[1].kind in nkLiterals and currObjPath[1].kind in nkLiterals:
-        if currFieldPath[1].intVal != currObjPath[1].intVal:
-          return no
-      else:
-        result = maybe
-    else: assert false # unreachable
-

compiler/aliases.nim

@@ -12,9 +12,6 @@
 import
   ast, astalgo, types, trees, intsets
 
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
 type
   TAnalysisResult* = enum
     arNo, arMaybe, arYes
@@ -77,29 +74,24 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
   ## cases:
   ##
   ## YES-cases:
-  ##  ```
   ##  x    <| x   # for general trees
   ##  x[]  <| x
   ##  x[i] <| x
   ##  x.f  <| x
-  ##  ```
   ##
   ## NO-cases:
-  ## ```
   ## x           !<| y    # depending on type and symbol kind
   ## x[constA]   !<| x[constB]
   ## x.f         !<| x.g
   ## x.f         !<| y.f  iff x !<= y
-  ## ```
   ##
   ## MAYBE-cases:
   ##
-  ##  ```
   ##  x[] ?<| y[]   iff compatible type
   ##
   ##
   ##  x[]  ?<| y  depending on type
-  ##  ```
+  ##
   if a.kind == b.kind:
     case a.kind
     of nkSym:

compiler/asciitables.nim

@@ -1,6 +1,6 @@
 #[
-move to std/asciitables.nim once stable, or to a fusion package
-once compiler can depend on fusion
+move to std/asciitables.nim once stable, or to a nimble paackage
+once compiler can depend on nimble
 ]#
 
 type Cell* = object
@@ -8,7 +8,7 @@ type Cell* = object
   width*, row*, col*, ncols*, nrows*: int
 
 iterator parseTableCells*(s: string, delim = '\t'): Cell =
-  ## Iterates over all cells in a `delim`-delimited `s`, after a 1st
+  ## iterates over all cells in a `delim`-delimited `s`, after a 1st
   ## pass that computes number of rows, columns, and width of each column.
   var widths: seq[int]
   var cell: Cell
@@ -69,7 +69,7 @@ iterator parseTableCells*(s: string, delim = '\t'): Cell =
     finishRow()
 
 proc alignTable*(s: string, delim = '\t', fill = ' ', sep = " "): string =
-  ## Formats a `delim`-delimited `s` representing a table; each cell is aligned
+  ## formats a `delim`-delimited `s` representing a table; each cell is aligned
   ## to a width that's computed for each column; consecutive columns are
   ## delimited by `sep`, and alignment space is filled using `fill`.
   ## More customized formatting can be done by calling `parseTableCells` directly.

compiler/astalgo.nim

@@ -12,17 +12,9 @@
 # the data structures here are used in various places of the compiler.
 
 import
-  ast, hashes, intsets, options, lineinfos, ropes, idents, rodutils,
+  ast, hashes, intsets, strutils, options, lineinfos, ropes, idents, rodutils,
   msgs
 
-import strutils except addf
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
-when not defined(nimHasCursor):
-  {.pragma: cursor.}
-
 proc hashNode*(p: RootRef): Hash
 proc treeToYaml*(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope
   # Convert a tree into its YAML representation; this is used by the
@@ -229,10 +221,11 @@ proc getNamedParamFromList*(list: PNode, ident: PIdent): PSym =
   ## Named parameters are special because a named parameter can be
   ## gensym'ed and then they have '\`<number>' suffix that we need to
   ## ignore, see compiler / evaltempl.nim, snippet:
-  ##   ```
-  ##   result.add newIdentNode(getIdent(c.ic, x.name.s & "\`gensym" & $x.id),
+  ##
+  ##..code-block:: nim
+  ##
+  ##    result.add newIdentNode(getIdent(c.ic, x.name.s & "\`gensym" & $x.id),
   ##            if c.instLines: actual.info else: templ.info)
-  ##   ```
   for i in 1..<list.len:
     let it = list[i].sym
     if it.name.id == ident.id or
@@ -260,7 +253,7 @@ proc makeYamlString*(s: string): Rope =
   # this could trigger InternalError(111). See the ropes module for
   # further information.
   const MaxLineLength = 64
-  result = ""
+  result = nil
   var res = "\""
   for i in 0..<s.len:
     if (i + 1) mod MaxLineLength == 0:
@@ -276,9 +269,9 @@ proc flagsToStr[T](flags: set[T]): Rope =
   if flags == {}:
     result = rope("[]")
   else:
-    result = ""
+    result = nil
     for x in items(flags):
-      if result != "": result.add(", ")
+      if result != nil: result.add(", ")
       result.add(makeYamlString($x))
     result = "[" & result & "]"
 
@@ -592,9 +585,6 @@ proc value(this: var DebugPrinter; value: PNode) =
   this.openCurly
   this.key "kind"
   this.value  value.kind
-  if value.comment.len > 0:
-    this.key "comment"
-    this.value  value.comment
   when defined(useNodeIds):
     this.key "id"
     this.value value.id
@@ -769,9 +759,9 @@ proc strTableAdd*(t: var TStrTable, n: PSym) =
 
 proc strTableInclReportConflict*(t: var TStrTable, n: PSym;
                                  onConflictKeepOld = false): PSym =
-  # if `t` has a conflicting symbol (same identifier as `n`), return it
-  # otherwise return `nil`. Incl `n` to `t` unless `onConflictKeepOld = true`
-  # and a conflict was found.
+  # returns true if n is already in the string table:
+  # It is essential that `n` is written nevertheless!
+  # This way the newest redefinition is picked by the semantic analyses!
   assert n.name != nil
   var h: Hash = n.name.h and high(t.data)
   var replaceSlot = -1
@@ -787,10 +777,9 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym;
       replaceSlot = h
     h = nextTry(h, high(t.data))
   if replaceSlot >= 0:
-    result = t.data[replaceSlot] # found it
     if not onConflictKeepOld:
       t.data[replaceSlot] = n # overwrite it with newer definition!
-    return result # but return the old one
+    return t.data[replaceSlot] # found it
   elif mustRehash(t.data.len, t.counter):
     strTableEnlarge(t)
     strTableRawInsert(t.data, n)
@@ -819,21 +808,16 @@ type
     name*: PIdent
 
 proc nextIdentIter*(ti: var TIdentIter, tab: TStrTable): PSym =
-  # hot spots
   var h = ti.h and high(tab.data)
   var start = h
-  var p {.cursor.} = tab.data[h]
-  while p != nil:
-    if p.name.id == ti.name.id: break
+  result = tab.data[h]
+  while result != nil:
+    if result.name.id == ti.name.id: break
     h = nextTry(h, high(tab.data))
     if h == start:
-      p = nil
+      result = nil
       break
-    p = tab.data[h]
-  if p != nil:
-    result = p # increase the count
-  else:
-    result = nil
+    result = tab.data[h]
   ti.h = nextTry(h, high(tab.data))
 
 proc initIdentIter*(ti: var TIdentIter, tab: TStrTable, s: PIdent): PSym =

compiler/astmsgs.nim

@@ -1,39 +0,0 @@
-# this module avoids ast depending on msgs or vice versa
-import std/strutils
-import options, ast, msgs
-
-proc typSym*(t: PType): PSym =
-  result = t.sym
-  if result == nil and t.kind == tyGenericInst: # this might need to be refined
-    result = t[0].sym
-
-proc addDeclaredLoc*(result: var string, conf: ConfigRef; sym: PSym) =
-  result.add " [$1 declared in $2]" % [sym.kind.toHumanStr, toFileLineCol(conf, sym.info)]
-
-proc addDeclaredLocMaybe*(result: var string, conf: ConfigRef; sym: PSym) =
-  if optDeclaredLocs in conf.globalOptions and sym != nil:
-    addDeclaredLoc(result, conf, sym)
-
-proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) =
-  # xxx figure out how to resolve `tyGenericParam`, e.g. for
-  # proc fn[T](a: T, b: T) = discard
-  # fn(1.1, "a")
-  let typ = typ.skipTypes(abstractInst + {tyStatic, tySequence, tyArray, tySet, tyUserTypeClassInst, tyVar, tyRef, tyPtr} - {tyRange})
-  result.add " [$1" % typ.kind.toHumanStr
-  if typ.sym != nil:
-    result.add " declared in " & toFileLineCol(conf, typ.sym.info)
-  result.add "]"
-
-proc addDeclaredLocMaybe*(result: var string, conf: ConfigRef; typ: PType) =
-  if optDeclaredLocs in conf.globalOptions: addDeclaredLoc(result, conf, typ)
-
-template quoteExpr*(a: string): untyped =
-  ## can be used for quoting expressions in error msgs.
-  "'" & a & "'"
-
-proc genFieldDefect*(conf: ConfigRef, field: string, disc: PSym): string =
-  let obj = disc.owner.name.s # `types.typeToString` might be better, eg for generics
-  result = "field '$#' is not accessible for type '$#'" % [field, obj]
-  if optDeclaredLocs in conf.globalOptions:
-    result.add " [discriminant declared in $#]" % toFileLineCol(conf, disc.info)
-  result.add " using '$# = " % disc.name.s

compiler/bitsets.nim

@@ -10,9 +10,6 @@
 # this unit handles Nim sets; it implements bit sets
 # the code here should be reused in the Nim standard library
 
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
 type
   ElemType = byte
   TBitSet* = seq[ElemType]    # we use byte here to avoid issues with
@@ -26,40 +23,53 @@ const
 template modElemSize(arg: untyped): untyped = arg and 7
 template divElemSize(arg: untyped): untyped = arg shr 3
 
-proc bitSetIn*(x: TBitSet, e: BiggestInt): bool =
+proc bitSetInit*(b: var TBitSet, length: int)
+proc bitSetUnion*(x: var TBitSet, y: TBitSet)
+proc bitSetDiff*(x: var TBitSet, y: TBitSet)
+proc bitSetSymDiff*(x: var TBitSet, y: TBitSet)
+proc bitSetIntersect*(x: var TBitSet, y: TBitSet)
+proc bitSetIncl*(x: var TBitSet, elem: BiggestInt)
+proc bitSetExcl*(x: var TBitSet, elem: BiggestInt)
+proc bitSetIn*(x: TBitSet, e: BiggestInt): bool
+proc bitSetEquals*(x, y: TBitSet): bool
+proc bitSetContains*(x, y: TBitSet): bool
+proc bitSetCard*(x: TBitSet): BiggestInt
+# implementation
+
+proc bitSetIn(x: TBitSet, e: BiggestInt): bool =
   result = (x[int(e.divElemSize)] and (One shl e.modElemSize)) != Zero
 
-proc bitSetIncl*(x: var TBitSet, elem: BiggestInt) =
+proc bitSetIncl(x: var TBitSet, elem: BiggestInt) =
   assert(elem >= 0)
   x[int(elem.divElemSize)] = x[int(elem.divElemSize)] or
       (One shl elem.modElemSize)
 
-proc bitSetExcl*(x: var TBitSet, elem: BiggestInt) =
+proc bitSetExcl(x: var TBitSet, elem: BiggestInt) =
   x[int(elem.divElemSize)] = x[int(elem.divElemSize)] and
       not(One shl elem.modElemSize)
 
-proc bitSetInit*(b: var TBitSet, length: int) =
+proc bitSetInit(b: var TBitSet, length: int) =
   newSeq(b, length)
 
-proc bitSetUnion*(x: var TBitSet, y: TBitSet) =
+proc bitSetUnion(x: var TBitSet, y: TBitSet) =
   for i in 0..high(x): x[i] = x[i] or y[i]
 
-proc bitSetDiff*(x: var TBitSet, y: TBitSet) =
+proc bitSetDiff(x: var TBitSet, y: TBitSet) =
   for i in 0..high(x): x[i] = x[i] and not y[i]
 
-proc bitSetSymDiff*(x: var TBitSet, y: TBitSet) =
+proc bitSetSymDiff(x: var TBitSet, y: TBitSet) =
   for i in 0..high(x): x[i] = x[i] xor y[i]
 
-proc bitSetIntersect*(x: var TBitSet, y: TBitSet) =
+proc bitSetIntersect(x: var TBitSet, y: TBitSet) =
   for i in 0..high(x): x[i] = x[i] and y[i]
 
-proc bitSetEquals*(x, y: TBitSet): bool =
+proc bitSetEquals(x, y: TBitSet): bool =
   for i in 0..high(x):
     if x[i] != y[i]:
       return false
   result = true
 
-proc bitSetContains*(x, y: TBitSet): bool =
+proc bitSetContains(x, y: TBitSet): bool =
   for i in 0..high(x):
     if (x[i] and not y[i]) != Zero:
       return false
@@ -86,6 +96,6 @@ const populationCount: array[uint8, uint8] = block:
 
     arr
 
-proc bitSetCard*(x: TBitSet): BiggestInt =
+proc bitSetCard(x: TBitSet): BiggestInt =
   for it in x:
     result.inc int(populationCount[it])

compiler/btrees.nim

@@ -10,9 +10,6 @@
 ## BTree implementation with few features, but good enough for the
 ## Nim compiler's needs.
 
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
 const
   M = 512    # max children per B-tree node = M-1
              # (must be even and greater than 2)
@@ -68,10 +65,7 @@ proc copyHalf[Key, Val](h, result: Node[Key, Val]) =
       result.links[j] = h.links[Mhalf + j]
   else:
     for j in 0..<Mhalf:
-      when defined(gcArc) or defined(gcOrc):
-        result.vals[j] = move h.vals[Mhalf + j]
-      else:
-        shallowCopy(result.vals[j], h.vals[Mhalf + j])
+      shallowCopy(result.vals[j], h.vals[Mhalf + j])
 
 proc split[Key, Val](h: Node[Key, Val]): Node[Key, Val] =
   ## split node in half
@@ -91,10 +85,7 @@ proc insert[Key, Val](h: Node[Key, Val], key: Key, val: Val): Node[Key, Val] =
       if less(key, h.keys[j]): break
       inc j
     for i in countdown(h.entries, j+1):
-      when defined(gcArc) or defined(gcOrc):
-        h.vals[i] = move h.vals[i-1]
-      else:
-        shallowCopy(h.vals[i], h.vals[i-1])
+      shallowCopy(h.vals[i], h.vals[i-1])
     h.vals[j] = val
   else:
     var newLink: Node[Key, Val] = nil

compiler/canonicalizer.nim

@@ -0,0 +1,421 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements the canonalization for the various caching mechanisms.
+
+import strutils, db_sqlite, md5
+
+var db: DbConn
+
+# We *hash* the relevant information into 128 bit hashes. This should be good
+# enough to prevent any collisions.
+
+type
+  TUid = distinct MD5Digest
+
+# For name mangling we encode these hashes via a variant of base64 (called
+# 'base64a') and prepend the *primary* identifier to ease the debugging pain.
+# So a signature like:
+#
+#   proc gABI(c: PCtx; n: PNode; opc: TOpcode; a, b: TRegister; imm: BiggestInt)
+#
+# is mangled into:
+#   gABI_MTdmOWY5MTQ1MDcyNGQ3ZA
+#
+# This is a good compromise between correctness and brevity. ;-)
+
+const
+  cb64 = [
+    "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N",
+    "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
+    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n",
+    "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
+    "0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
+    "_A", "_B"]
+
+proc toBase64a(s: cstring, len: int): string =
+  ## encodes `s` into base64 representation. After `lineLen` characters, a
+  ## `newline` is added.
+  result = newStringOfCap(((len + 2) div 3) * 4)
+  var i = 0
+  while i < s.len - 2:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    let c = ord(s[i+2])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2) or ((c and 0xC0) shr 6)]
+    result.add cb64[c and 0x3F]
+    inc(i, 3)
+  if i < s.len-1:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2)]
+  elif i < s.len:
+    let a = ord(s[i])
+    result.add cb64[a shr 2]
+    result.add cb64[(a and 3) shl 4]
+
+proc toBase64a(u: TUid): string = toBase64a(cast[cstring](u), sizeof(u))
+
+proc `&=`(c: var MD5Context, s: string) = md5Update(c, s, s.len)
+
+proc hashSym(c: var MD5Context, s: PSym) =
+  if sfAnon in s.flags or s.kind == skGenericParam:
+    c &= ":anon"
+  else:
+    var it = s.owner
+    while it != nil:
+      hashSym(c, it)
+      c &= "."
+      it = s.owner
+    c &= s.name.s
+
+proc hashTree(c: var MD5Context, n: PNode) =
+  if n == nil:
+    c &= "\255"
+    return
+  var k = n.kind
+  md5Update(c, cast[cstring](addr(k)), 1)
+  # we really must not hash line information. 'n.typ' is debatable but
+  # shouldn't be necessary for now and avoids potential infinite recursions.
+  case n.kind
+  of nkEmpty, nkNilLit, nkType: discard
+  of nkIdent:
+    c &= n.ident.s
+  of nkSym:
+    hashSym(c, n.sym)
+  of nkCharLit..nkUInt64Lit:
+    var v = n.intVal
+    md5Update(c, cast[cstring](addr(v)), sizeof(v))
+  of nkFloatLit..nkFloat64Lit:
+    var v = n.floatVal
+    md5Update(c, cast[cstring](addr(v)), sizeof(v))
+  of nkStrLit..nkTripleStrLit:
+    c &= n.strVal
+  else:
+    for i in 0..<n.len: hashTree(c, n[i])
+
+proc hashType(c: var MD5Context, t: PType) =
+  # modelled after 'typeToString'
+  if t == nil:
+    c &= "\254"
+    return
+
+  var k = t.kind
+  md5Update(c, cast[cstring](addr(k)), 1)
+
+  if t.sym != nil and sfAnon notin t.sym.flags:
+    # t.n for literals, but not for e.g. objects!
+    if t.kind in {tyFloat, tyInt}: c.hashNode(t.n)
+    c.hashSym(t.sym)
+
+  case t.kind
+  of tyGenericBody, tyGenericInst, tyGenericInvocation:
+    for i in 0..<t.len-ord(t.kind != tyGenericInvocation):
+      c.hashType t[i]
+  of tyUserTypeClass:
+    internalAssert t.sym != nil and t.sym.owner != nil
+    c &= t.sym.owner.name.s
+  of tyUserTypeClassInst:
+    let body = t.base
+    c.hashSym body.sym
+    for i in 1..<t.len-1:
+      c.hashType t[i]
+  of tyFromExpr:
+    c.hashTree(t.n)
+  of tyArray:
+    c.hashTree(t[0].n)
+    c.hashType(t[1])
+  of tyTuple:
+    if t.n != nil:
+      assert(t.n.len == t.len)
+      for i in 0..<t.n.len:
+        assert(t.n[i].kind == nkSym)
+        c &= t.n[i].sym.name.s
+        c &= ":"
+        c.hashType(t[i])
+        c &= ","
+    else:
+      for i in 0..<t.len: c.hashType t[i]
+  of tyRange:
+    c.hashTree(t.n)
+    c.hashType(t[0])
+  of tyProc:
+    c &= (if tfIterator in t.flags: "iterator " else: "proc ")
+    for i in 0..<t.len: c.hashType(t[i])
+    md5Update(c, cast[cstring](addr(t.callConv)), 1)
+
+    if tfNoSideEffect in t.flags: c &= ".noSideEffect"
+    if tfThread in t.flags: c &= ".thread"
+  else:
+    for i in 0..<t.len: c.hashType(t[i])
+  if tfNotNil in t.flags: c &= "not nil"
+
+proc canonConst(n: PNode): TUid =
+  var c: MD5Context
+  md5Init(c)
+  c.hashTree(n)
+  c.hashType(n.typ)
+  md5Final(c, MD5Digest(result))
+
+proc canonSym(s: PSym): TUid =
+  var c: MD5Context
+  md5Init(c)
+  c.hashSym(s)
+  md5Final(c, MD5Digest(result))
+
+proc pushType(w: PRodWriter, t: PType) =
+  # check so that the stack does not grow too large:
+  if iiTableGet(w.index.tab, t.id) == InvalidKey:
+    w.tstack.add(t)
+
+proc pushSym(w: PRodWriter, s: PSym) =
+  # check so that the stack does not grow too large:
+  if iiTableGet(w.index.tab, s.id) == InvalidKey:
+    w.sstack.add(s)
+
+proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
+                result: var string) =
+  if n == nil:
+    # nil nodes have to be stored too:
+    result.add("()")
+    return
+  result.add('(')
+  encodeVInt(ord(n.kind), result)
+  # we do not write comments for now
+  # Line information takes easily 20% or more of the filesize! Therefore we
+  # omit line information if it is the same as the father's line information:
+  if fInfo.fileIndex != n.info.fileIndex:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+    result.add(',')
+    encodeVInt(n.info.line, result)
+    result.add(',')
+    encodeVInt(fileIdx(w, toFilename(n.info)), result)
+  elif fInfo.line != n.info.line:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+    result.add(',')
+    encodeVInt(n.info.line, result)
+  elif fInfo.col != n.info.col:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+  var f = n.flags * PersistentNodeFlags
+  if f != {}:
+    result.add('$')
+    encodeVInt(cast[int32](f), result)
+  if n.typ != nil:
+    result.add('^')
+    encodeVInt(n.typ.id, result)
+    pushType(w, n.typ)
+  case n.kind
+  of nkCharLit..nkInt64Lit:
+    if n.intVal != 0:
+      result.add('!')
+      encodeVBiggestInt(n.intVal, result)
+  of nkFloatLit..nkFloat64Lit:
+    if n.floatVal != 0.0:
+      result.add('!')
+      encodeStr($n.floatVal, result)
+  of nkStrLit..nkTripleStrLit:
+    if n.strVal != "":
+      result.add('!')
+      encodeStr(n.strVal, result)
+  of nkIdent:
+    result.add('!')
+    encodeStr(n.ident.s, result)
+  of nkSym:
+    result.add('!')
+    encodeVInt(n.sym.id, result)
+    pushSym(w, n.sym)
+  else:
+    for i in 0..<n.len:
+      encodeNode(w, n.info, n[i], result)
+  result.add(')')
+
+proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
+  var oldLen = result.len
+  result.add('<')
+  if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
+  if loc.s != low(loc.s):
+    result.add('*')
+    encodeVInt(ord(loc.s), result)
+  if loc.flags != {}:
+    result.add('$')
+    encodeVInt(cast[int32](loc.flags), result)
+  if loc.t != nil:
+    result.add('^')
+    encodeVInt(cast[int32](loc.t.id), result)
+    pushType(w, loc.t)
+  if loc.r != nil:
+    result.add('!')
+    encodeStr($loc.r, result)
+  if loc.a != 0:
+    result.add('?')
+    encodeVInt(loc.a, result)
+  if oldLen + 1 == result.len:
+    # no data was necessary, so remove the '<' again:
+    setLen(result, oldLen)
+  else:
+    result.add('>')
+
+proc encodeType(w: PRodWriter, t: PType, result: var string) =
+  if t == nil:
+    # nil nodes have to be stored too:
+    result.add("[]")
+    return
+  # we need no surrounding [] here because the type is in a line of its own
+  if t.kind == tyForward: internalError("encodeType: tyForward")
+  # for the new rodfile viewer we use a preceding [ so that the data section
+  # can easily be disambiguated:
+  result.add('[')
+  encodeVInt(ord(t.kind), result)
+  result.add('+')
+  encodeVInt(t.id, result)
+  if t.n != nil:
+    encodeNode(w, unknownLineInfo, t.n, result)
+  if t.flags != {}:
+    result.add('$')
+    encodeVInt(cast[int32](t.flags), result)
+  if t.callConv != low(t.callConv):
+    result.add('?')
+    encodeVInt(ord(t.callConv), result)
+  if t.owner != nil:
+    result.add('*')
+    encodeVInt(t.owner.id, result)
+    pushSym(w, t.owner)
+  if t.sym != nil:
+    result.add('&')
+    encodeVInt(t.sym.id, result)
+    pushSym(w, t.sym)
+  if t.size != - 1:
+    result.add('/')
+    encodeVBiggestInt(t.size, result)
+  if t.align != - 1:
+    result.add('=')
+    encodeVInt(t.align, result)
+  encodeLoc(w, t.loc, result)
+  for i in 0..<t.len:
+    if t[i] == nil:
+      result.add("^()")
+    else:
+      result.add('^')
+      encodeVInt(t[i].id, result)
+      pushType(w, t[i])
+
+proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) =
+  result.add('|')
+  encodeVInt(ord(lib.kind), result)
+  result.add('|')
+  encodeStr($lib.name, result)
+  result.add('|')
+  encodeNode(w, info, lib.path, result)
+
+proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
+  if s == nil:
+    # nil nodes have to be stored too:
+    result.add("{}")
+    return
+  # we need no surrounding {} here because the symbol is in a line of its own
+  encodeVInt(ord(s.kind), result)
+  result.add('+')
+  encodeVInt(s.id, result)
+  result.add('&')
+  encodeStr(s.name.s, result)
+  if s.typ != nil:
+    result.add('^')
+    encodeVInt(s.typ.id, result)
+    pushType(w, s.typ)
+  result.add('?')
+  if s.info.col != -1'i16: encodeVInt(s.info.col, result)
+  result.add(',')
+  if s.info.line != -1'i16: encodeVInt(s.info.line, result)
+  result.add(',')
+  encodeVInt(fileIdx(w, toFilename(s.info)), result)
+  if s.owner != nil:
+    result.add('*')
+    encodeVInt(s.owner.id, result)
+    pushSym(w, s.owner)
+  if s.flags != {}:
+    result.add('$')
+    encodeVInt(cast[int32](s.flags), result)
+  if s.magic != mNone:
+    result.add('@')
+    encodeVInt(ord(s.magic), result)
+  if s.options != w.options:
+    result.add('!')
+    encodeVInt(cast[int32](s.options), result)
+  if s.position != 0:
+    result.add('%')
+    encodeVInt(s.position, result)
+  if s.offset != - 1:
+    result.add('`')
+    encodeVInt(s.offset, result)
+  encodeLoc(w, s.loc, result)
+  if s.annex != nil: encodeLib(w, s.annex, s.info, result)
+  if s.constraint != nil:
+    result.add('#')
+    encodeNode(w, unknownLineInfo, s.constraint, result)
+  # lazy loading will soon reload the ast lazily, so the ast needs to be
+  # the last entry of a symbol:
+  if s.ast != nil:
+    # we used to attempt to save space here by only storing a dummy AST if
+    # it is not necessary, but Nim's heavy compile-time evaluation features
+    # make that unfeasible nowadays:
+    encodeNode(w, s.info, s.ast, result)
+
+
+proc createDb() =
+  db.exec(sql"""
+    create table if not exists Module(
+      id integer primary key,
+      name varchar(256) not null,
+      fullpath varchar(256) not null,
+      interfHash varchar(256) not null,
+      fullHash varchar(256) not null,
+
+      created timestamp not null default (DATETIME('now'))
+    );""")
+
+  db.exec(sql"""
+    create table if not exists Backend(
+      id integer primary key,
+      strongdeps varchar(max) not null,
+      weakdeps varchar(max) not null,
+      header varchar(max) not null,
+      code varchar(max) not null
+    )
+
+    create table if not exists Symbol(
+      id integer primary key,
+      module integer not null,
+      backend integer not null,
+      name varchar(max) not null,
+      data varchar(max) not null,
+      created timestamp not null default (DATETIME('now')),
+
+      foreign key (module) references Module(id),
+      foreign key (backend) references Backend(id)
+    );""")
+
+  db.exec(sql"""
+    create table if not exists Type(
+      id integer primary key,
+      module integer not null,
+      name varchar(max) not null,
+      data varchar(max) not null,
+      created timestamp not null default (DATETIME('now')),
+
+      foreign key (module) references module(id)
+    );""")
+
+

compiler/ccgcalls.nim

@@ -14,15 +14,14 @@ proc canRaiseDisp(p: BProc; n: PNode): bool =
   if n.kind == nkSym and {sfNeverRaises, sfImportc, sfCompilerProc} * n.sym.flags != {}:
     result = false
   elif optPanics in p.config.globalOptions or
-      (n.kind == nkSym and sfSystemModule in getModule(n.sym).flags and
-       sfSystemRaisesDefect notin n.sym.flags):
+      (n.kind == nkSym and sfSystemModule in getModule(n.sym).flags):
     # we know we can be strict:
     result = canRaise(n)
   else:
     # we have to be *very* conservative:
     result = canRaiseConservative(n)
 
-proc preventNrvo(p: BProc; dest, le, ri: PNode): bool =
+proc preventNrvo(p: BProc; le, ri: PNode): bool =
   proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
     var n = le
     while true:
@@ -55,11 +54,6 @@ proc preventNrvo(p: BProc; dest, le, ri: PNode): bool =
     if canRaise(ri[0]) and
         locationEscapes(p, le, p.nestedTryStmts.len > 0):
       message(p.config, le.info, warnObservableStores, $le)
-  # bug #19613 prevent dangerous aliasing too:
-  if dest != nil and dest != le:
-    for i in 1..<ri.len:
-      let r = ri[i]
-      if isPartOf(dest, r) != arNo: return true
 
 proc hasNoInit(call: PNode): bool {.inline.} =
   result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
@@ -78,32 +72,32 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
                callee, params: Rope) =
   let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
   genLineDir(p, ri)
-  var pl = callee & "(" & params
+  var pl = callee & ~"(" & params
   # getUniqueType() is too expensive here:
   var typ = skipTypes(ri[0].typ, abstractInst)
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ):
-      if params.len != 0: pl.add(", ")
+    if isInvalidReturnType(p.config, typ[0]):
+      if params != nil: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not preventNrvo(p, d.lode, le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
           discard "resetLoc(p, d)"
         pl.add(addrLoc(p.config, d))
-        pl.add(");\n")
+        pl.add(~");$n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
         getTemp(p, typ[0], tmp, needsInit=true)
         pl.add(addrLoc(p.config, tmp))
-        pl.add(");\n")
+        pl.add(~");$n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
       if canRaise: raiseExit(p)
     else:
-      pl.add(")")
+      pl.add(~")")
       if p.module.compileToCpp:
         if lfSingleUse in d.flags:
           # do not generate spurious temporaries for C++! For C we're better off
@@ -141,7 +135,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
         if canRaise: raiseExit(p)
         genAssignment(p, d, tmp, {})
   else:
-    pl.add(");\n")
+    pl.add(~");$n")
     line(p, cpsStmts, pl)
     if canRaise: raiseExit(p)
 
@@ -174,10 +168,8 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope,
       result = ("($3*)(($1)+($2))" % [rdLoc(a), rdLoc(b), dest],
                 lengthExpr)
     else:
-      var lit = newRopeAppender()
-      intLiteral(first, lit)
       result = ("($4*)($1)+(($2)-($3))" %
-        [rdLoc(a), rdLoc(b), lit, dest],
+        [rdLoc(a), rdLoc(b), intLiteral(first), dest],
         lengthExpr)
   of tyOpenArray, tyVarargs:
     if reifiedOpenArray(q[1]):
@@ -186,7 +178,7 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope,
     else:
       result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest],
                 lengthExpr)
-  of tyUncheckedArray, tyCstring:
+  of tyUncheckedArray, tyCString:
     result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest],
               lengthExpr)
   of tyString, tySequence:
@@ -195,17 +187,15 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope,
         optSeqDestructors in p.config.globalOptions:
       linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
     if atyp.kind in {tyVar} and not compileToCpp(p.module):
-      result = ("(($5) ? (($4*)(*$1)$3+($2)) : NIM_NIL)" %
-                  [rdLoc(a), rdLoc(b), dataField(p), dest, dataFieldAccessor(p, "*" & rdLoc(a))],
+      result = ("($4*)(*$1)$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
                 lengthExpr)
     else:
-      result = ("(($5) ? (($4*)$1$3+($2)) : NIM_NIL)" %
-                  [rdLoc(a), rdLoc(b), dataField(p), dest, dataFieldAccessor(p, rdLoc(a))],
+      result = ("($4*)$1$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
                 lengthExpr)
   else:
     internalError(p.config, "openArrayLoc: " & typeToString(a.t))
 
-proc openArrayLoc(p: BProc, formalType: PType, n: PNode; result: var Rope) =
+proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
   var q = skipConv(n)
   var skipped = false
   while q.kind == nkStmtListExpr and q.len > 0:
@@ -220,19 +210,19 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode; result: var Rope) =
           genStmts(p, q[i])
         q = q.lastSon
     let (x, y) = genOpenArraySlice(p, q, formalType, n.typ[0])
-    result.add x & ", " & y
+    result = x & ", " & y
   else:
     var a: TLoc
     initLocExpr(p, if n.kind == nkHiddenStdConv: n[1] else: n, a)
-    case skipTypes(a.t, abstractVar+{tyStatic}).kind
+    case skipTypes(a.t, abstractVar).kind
     of tyOpenArray, tyVarargs:
       if reifiedOpenArray(n):
         if a.t.kind in {tyVar, tyLent}:
-          result.add "$1->Field0, $1->Field1" % [rdLoc(a)]
+          result = "$1->Field0, $1->Field1" % [rdLoc(a)]
         else:
-          result.add "$1.Field0, $1.Field1" % [rdLoc(a)]
+          result = "$1.Field0, $1.Field1" % [rdLoc(a)]
       else:
-        result.add "$1, $1Len_0" % [rdLoc(a)]
+        result = "$1, $1Len_0" % [rdLoc(a)]
     of tyString, tySequence:
       let ntyp = skipTypes(n.typ, abstractInst)
       if formalType.skipTypes(abstractInst).kind in {tyVar} and ntyp.kind == tyString and
@@ -241,24 +231,19 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode; result: var Rope) =
       if ntyp.kind in {tyVar} and not compileToCpp(p.module):
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
-        result.add "($4) ? ((*$1)$3) : NIM_NIL, $2" %
-                     [a.rdLoc, lenExpr(p, t), dataField(p),
-                      dataFieldAccessor(p, "*" & a.rdLoc)]
+        result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
       else:
-        result.add "($4) ? ($1$3) : NIM_NIL, $2" %
-                     [a.rdLoc, lenExpr(p, a), dataField(p), dataFieldAccessor(p, a.rdLoc)]
+        result = "$1$3, $2" % [a.rdLoc, lenExpr(p, a), dataField(p)]
     of tyArray:
-      result.add "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, a.t))]
+      result = "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, a.t))]
     of tyPtr, tyRef:
       case lastSon(a.t).kind
       of tyString, tySequence:
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
-        result.add "($4) ? ((*$1)$3) : NIM_NIL, $2" %
-                     [a.rdLoc, lenExpr(p, t), dataField(p),
-                      dataFieldAccessor(p, "*" & a.rdLoc)]
+        result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
       of tyArray:
-        result.add "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, lastSon(a.t)))]
+        result = "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, lastSon(a.t)))]
       else:
         internalError(p.config, "openArrayLoc: " & typeToString(a.t))
     else: internalError(p.config, "openArrayLoc: " & typeToString(a.t))
@@ -274,28 +259,21 @@ proc withTmpIfNeeded(p: BProc, a: TLoc, needsTmp: bool): TLoc =
   else:
     result = a
 
-proc literalsNeedsTmp(p: BProc, a: TLoc): TLoc =
-  getTemp(p, a.lode.typ, result, needsInit=false)
-  genAssignment(p, result, a, {})
-
-proc genArgStringToCString(p: BProc, n: PNode; result: var Rope; needsTmp: bool) {.inline.} =
+proc genArgStringToCString(p: BProc, n: PNode, needsTmp: bool): Rope {.inline.} =
   var a: TLoc
   initLocExpr(p, n[0], a)
-  appcg(p.module, result, "#nimToCStringConv($1)", [withTmpIfNeeded(p, a, needsTmp).rdLoc])
+  ropecg(p.module, "#nimToCStringConv($1)", [withTmpIfNeeded(p, a, needsTmp).rdLoc])
 
-proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Rope; needsTmp = false) =
+proc genArg(p: BProc, n: PNode, param: PSym; call: PNode, needsTmp = false): Rope =
   var a: TLoc
   if n.kind == nkStringToCString:
-    genArgStringToCString(p, n, result, needsTmp)
+    result = genArgStringToCString(p, n, needsTmp)
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
     var n = if n.kind != nkHiddenAddr: n else: n[0]
-    openArrayLoc(p, param.typ, n, result)
+    result = openArrayLoc(p, param.typ, n)
   elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
-    if n.kind in {nkCharLit..nkNilLit}:
-      addAddrLoc(p.config, literalsNeedsTmp(p, a), result)
-    else:
-      addAddrLoc(p.config, withTmpIfNeeded(p, a, needsTmp), result)
+    result = addrLoc(p.config, withTmpIfNeeded(p, a, needsTmp))
   elif p.module.compileToCpp and param.typ.kind in {tyVar} and
       n.kind == nkHiddenAddr:
     initLocExprSingleUse(p, n[0], a)
@@ -305,77 +283,64 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Rope; need
     if callee.kind == nkSym and
         {sfImportc, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportc} and
         {lfHeader, lfNoDecl} * callee.sym.loc.flags != {}:
-      addAddrLoc(p.config, a, result)
+      result = addrLoc(p.config, a)
     else:
-      addRdLoc(a, result)
+      result = rdLoc(a)
   else:
     initLocExprSingleUse(p, n, a)
-    addRdLoc(withTmpIfNeeded(p, a, needsTmp), result)
-  #assert result != nil
+    result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
 
-proc genArgNoParam(p: BProc, n: PNode; result: var Rope; needsTmp = false) =
+proc genArgNoParam(p: BProc, n: PNode, needsTmp = false): Rope =
   var a: TLoc
   if n.kind == nkStringToCString:
-    genArgStringToCString(p, n, result, needsTmp)
+    result = genArgStringToCString(p, n, needsTmp)
   else:
     initLocExprSingleUse(p, n, a)
-    addRdLoc(withTmpIfNeeded(p, a, needsTmp), result)
+    result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
 
-import aliasanalysis
+from dfa import instrTargets, InstrTargetKind
 
 proc potentialAlias(n: PNode, potentialWrites: seq[PNode]): bool =
   for p in potentialWrites:
-    if p.aliases(n) != no or n.aliases(p) != no:
+    if instrTargets(p, n) != None:
       return true
 
 proc skipTrivialIndirections(n: PNode): PNode =
   result = n
   while true:
     case result.kind
-    of nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv:
+    of {nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv}:
       result = result[0]
-    of nkHiddenStdConv, nkHiddenSubConv:
+    of {nkHiddenStdConv, nkHiddenSubConv}:
       result = result[1]
     else: break
 
-proc getPotentialWrites(n: PNode; mutate: bool; result: var seq[PNode]) =
+proc getPotentialWrites(n: PNode, mutate = false): seq[PNode] =
   case n.kind:
-  of nkLiterals, nkIdent, nkFormalParams: discard
+  of nkLiterals, nkIdent: discard
   of nkSym:
     if mutate: result.add n
-  of nkAsgn, nkFastAsgn, nkSinkAsgn:
-    getPotentialWrites(n[0], true, result)
-    getPotentialWrites(n[1], mutate, result)
+  of nkAsgn, nkFastAsgn:
+    result.add getPotentialWrites(n[0], true)
+    result.add getPotentialWrites(n[1], mutate)
   of nkAddr, nkHiddenAddr:
-    getPotentialWrites(n[0], true, result)
-  of nkBracketExpr, nkDotExpr, nkCheckedFieldExpr:
-    getPotentialWrites(n[0], mutate, result)
-  of nkCallKinds:
-    case n.getMagic:
-    of mIncl, mExcl, mInc, mDec, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
-        mAddr, mNew, mNewFinalize, mWasMoved, mDestroy, mReset:
-      getPotentialWrites(n[1], true, result)
-      for i in 2..<n.len:
-        getPotentialWrites(n[i], mutate, result)
-    of mSwap:
-      for i in 1..<n.len:
-        getPotentialWrites(n[i], true, result)
-    else:
-      for i in 1..<n.len:
-        getPotentialWrites(n[i], mutate, result)
+    result.add getPotentialWrites(n[0], true)
+  of nkCallKinds: #TODO: Find out why in f += 1, f is a nkSym and not a nkHiddenAddr
+    for s in n.sons:
+      result.add getPotentialWrites(s, true)
   else:
-    for s in n:
-      getPotentialWrites(s, mutate, result)
+    for s in n.sons:
+      result.add getPotentialWrites(s, mutate)
 
-proc getPotentialReads(n: PNode; result: var seq[PNode]) =
+proc getPotentialReads(n: PNode): seq[PNode] =
   case n.kind:
-  of nkLiterals, nkIdent, nkFormalParams: discard
+  of nkLiterals, nkIdent: discard
   of nkSym: result.add n
   else:
-    for s in n:
-      getPotentialReads(s, result)
+    for s in n.sons:
+      result.add getPotentialReads(s)
 
-proc genParams(p: BProc, ri: PNode, typ: PType; result: var Rope) =
+proc genParams(p: BProc, ri: PNode, typ: PType): Rope =
   # We must generate temporaries in cases like #14396
   # to keep the strict Left-To-Right evaluation
   var needTmp = newSeq[bool](ri.len - 1)
@@ -384,32 +349,24 @@ proc genParams(p: BProc, ri: PNode, typ: PType; result: var Rope) =
     if ri[i].skipTrivialIndirections.kind == nkSym:
       needTmp[i - 1] = potentialAlias(ri[i], potentialWrites)
     else:
-      #if not ri[i].typ.isCompileTimeOnly:
-      var potentialReads: seq[PNode]
-      getPotentialReads(ri[i], potentialReads)
-      for n in potentialReads:
+      for n in getPotentialReads(ri[i]):
         if not needTmp[i - 1]:
           needTmp[i - 1] = potentialAlias(n, potentialWrites)
-      getPotentialWrites(ri[i], false, potentialWrites)
-    if ri[i].kind in {nkHiddenAddr, nkAddr}:
-      # Optimization: don't use a temp, if we would only take the address anyway
+      potentialWrites.add getPotentialWrites(ri[i])
+    if ri[i].kind == nkHiddenAddr:
+      # Optimization: don't use a temp, if we would only take the adress anyway
       needTmp[i - 1] = false
 
-  var oldLen = result.len
   for i in 1..<ri.len:
     if i < typ.len:
       assert(typ.n[i].kind == nkSym)
       let paramType = typ.n[i]
       if not paramType.typ.isCompileTimeOnly:
-        if oldLen != result.len:
-          result.add(", ")
-          oldLen = result.len
-        genArg(p, ri[i], paramType.sym, ri, result, needTmp[i-1])
+        if result != nil: result.add(~", ")
+        result.add(genArg(p, ri[i], paramType.sym, ri, needTmp[i-1]))
     else:
-      if oldLen != result.len:
-        result.add(", ")
-        oldLen = result.len
-      genArgNoParam(p, ri[i], result, needTmp[i-1])
+      if result != nil: result.add(~", ")
+      result.add(genArgNoParam(p, ri[i], needTmp[i-1]))
 
 proc addActualSuffixForHCR(res: var Rope, module: PSym, sym: PSym) =
   if sym.flags * {sfImportc, sfNonReloadable} == {} and sym.loc.k == locProc and
@@ -425,8 +382,7 @@ proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   assert(typ.kind == tyProc)
   assert(typ.len == typ.n.len)
 
-  var params = newRopeAppender()
-  genParams(p, ri, typ, params)
+  var params = genParams(p, ri, typ)
 
   var callee = rdLoc(op)
   if p.hcrOn and ri[0].kind == nkSym:
@@ -436,7 +392,7 @@ proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
 proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
 
   proc addComma(r: Rope): Rope =
-    if r.len == 0: r else: r & ", "
+    if r == nil: r else: r & ~", "
 
   const PatProc = "$1.ClE_0? $1.ClP_0($3$1.ClE_0):(($4)($1.ClP_0))($2)"
   const PatIter = "$1.ClP_0($3$1.ClE_0)" # we know the env exists
@@ -449,8 +405,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
   assert(typ.kind == tyProc)
   assert(typ.len == typ.n.len)
 
-  var pl = newRopeAppender()
-  genParams(p, ri, typ, pl)
+  var pl = genParams(p, ri, typ)
 
   template genCallPattern {.dirty.} =
     if tfIterator in typ.flags:
@@ -461,10 +416,10 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
   let rawProc = getClosureType(p.module, typ, clHalf)
   let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ):
-      if ri.len > 1: pl.add(", ")
+    if isInvalidReturnType(p.config, typ[0]):
+      if ri.len > 1: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not preventNrvo(p, d.lode, le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
           getTemp(p, typ[0], d, needsInit=true)
@@ -508,30 +463,24 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
     genCallPattern()
     if canRaise: raiseExit(p)
 
-proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope;
-                 argsCounter: var int) =
+proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   if i < typ.len:
     # 'var T' is 'T&' in C++. This means we ignore the request of
     # any nkHiddenAddr when it's a 'var T'.
     let paramType = typ.n[i]
     assert(paramType.kind == nkSym)
     if paramType.typ.isCompileTimeOnly:
-      discard
+      result = nil
     elif typ[i].kind in {tyVar} and ri[i].kind == nkHiddenAddr:
-      if argsCounter > 0: result.add ", "
-      genArgNoParam(p, ri[i][0], result)
-      inc argsCounter
+      result = genArgNoParam(p, ri[i][0])
     else:
-      if argsCounter > 0: result.add ", "
-      genArgNoParam(p, ri[i], result) #, typ.n[i].sym)
-      inc argsCounter
+      result = genArgNoParam(p, ri[i]) #, typ.n[i].sym)
   else:
     if tfVarargs notin typ.flags:
       localError(p.config, ri.info, "wrong argument count")
+      result = nil
     else:
-      if argsCounter > 0: result.add ", "
-      genArgNoParam(p, ri[i], result)
-      inc argsCounter
+      result = genArgNoParam(p, ri[i])
 
 discard """
 Dot call syntax in C++
@@ -588,7 +537,7 @@ proc skipAddrDeref(node: PNode): PNode =
   else:
     result = node
 
-proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope) =
+proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   # for better or worse c2nim translates the 'this' argument to a 'var T'.
   # However manual wrappers may also use 'ptr T'. In any case we support both
   # for convenience.
@@ -602,72 +551,75 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope) =
   if t.kind in {tyVar}:
     let x = if ri.kind == nkHiddenAddr: ri[0] else: ri
     if x.typ.kind == tyPtr:
-      genArgNoParam(p, x, result)
+      result = genArgNoParam(p, x)
       result.add("->")
     elif x.kind in {nkHiddenDeref, nkDerefExpr} and x[0].typ.kind == tyPtr:
-      genArgNoParam(p, x[0], result)
+      result = genArgNoParam(p, x[0])
       result.add("->")
     else:
-      genArgNoParam(p, x, result)
+      result = genArgNoParam(p, x)
       result.add(".")
   elif t.kind == tyPtr:
     if ri.kind in {nkAddr, nkHiddenAddr}:
-      genArgNoParam(p, ri[0], result)
+      result = genArgNoParam(p, ri[0])
       result.add(".")
     else:
-      genArgNoParam(p, ri, result)
+      result = genArgNoParam(p, ri)
       result.add("->")
   else:
     ri = skipAddrDeref(ri)
     if ri.kind in {nkAddr, nkHiddenAddr}: ri = ri[0]
-    genArgNoParam(p, ri, result) #, typ.n[i].sym)
+    result = genArgNoParam(p, ri) #, typ.n[i].sym)
     result.add(".")
 
-proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType; result: var Rope) =
+proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
   var i = 0
   var j = 1
   while i < pat.len:
     case pat[i]
     of '@':
-      var argsCounter = 0
+      var first = true
       for k in j..<ri.len:
-        genOtherArg(p, ri, k, typ, result, argsCounter)
+        let arg = genOtherArg(p, ri, k, typ)
+        if arg.len > 0:
+          if not first:
+            result.add(~", ")
+          first = false
+          result.add arg
       inc i
     of '#':
       if i+1 < pat.len and pat[i+1] in {'+', '@'}:
         let ri = ri[j]
         if ri.kind in nkCallKinds:
           let typ = skipTypes(ri[0].typ, abstractInst)
-          if pat[i+1] == '+': genArgNoParam(p, ri[0], result)
-          result.add("(")
+          if pat[i+1] == '+': result.add genArgNoParam(p, ri[0])
+          result.add(~"(")
           if 1 < ri.len:
-            var argsCounterB = 0
-            genOtherArg(p, ri, 1, typ, result, argsCounterB)
+            result.add genOtherArg(p, ri, 1, typ)
           for k in j+1..<ri.len:
-            var argsCounterB = 0
-            genOtherArg(p, ri, k, typ, result, argsCounterB)
-          result.add(")")
+            result.add(~", ")
+            result.add genOtherArg(p, ri, k, typ)
+          result.add(~")")
         else:
           localError(p.config, ri.info, "call expression expected for C++ pattern")
         inc i
       elif i+1 < pat.len and pat[i+1] == '.':
-        genThisArg(p, ri, j, typ, result)
+        result.add genThisArg(p, ri, j, typ)
         inc i
       elif i+1 < pat.len and pat[i+1] == '[':
         var arg = ri[j].skipAddrDeref
         while arg.kind in {nkAddr, nkHiddenAddr, nkObjDownConv}: arg = arg[0]
-        genArgNoParam(p, arg, result)
+        result.add genArgNoParam(p, arg)
         #result.add debugTree(arg, 0, 10)
       else:
-        var argsCounter = 0
-        genOtherArg(p, ri, j, typ, result, argsCounter)
+        result.add genOtherArg(p, ri, j, typ)
       inc j
       inc i
     of '\'':
       var idx, stars: int
       if scanCppGenericSlot(pat, i, idx, stars):
         var t = resolveStarsInCppType(typ, idx, stars)
-        if t == nil: result.add("void")
+        if t == nil: result.add(~"void")
         else: result.add(getTypeDesc(p.module, t))
     else:
       let start = i
@@ -685,11 +637,10 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   assert(typ.kind == tyProc)
   assert(typ.len == typ.n.len)
   # don't call '$' here for efficiency:
-  let pat = $ri[0].sym.loc.r
+  let pat = ri[0].sym.loc.r.data
   internalAssert p.config, pat.len > 0
   if pat.contains({'#', '(', '@', '\''}):
-    var pl = newRopeAppender()
-    genPatternCall(p, ri, pat, typ, pl)
+    var pl = genPatternCall(p, ri, pat, typ)
     # simpler version of 'fixupCall' that works with the pl+params combination:
     var typ = skipTypes(ri[0].typ, abstractInst)
     if typ[0] != nil:
@@ -708,79 +659,80 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
     else:
-      pl.add(";\n")
+      pl.add(~";$n")
       line(p, cpsStmts, pl)
   else:
-    var pl = newRopeAppender()
-    var argsCounter = 0
+    var pl: Rope = nil
+    #var param = typ.n[1].sym
     if 1 < ri.len:
-      genThisArg(p, ri, 1, typ, pl)
+      pl.add(genThisArg(p, ri, 1, typ))
     pl.add(op.r)
-    var params = newRopeAppender()
+    var params: Rope
     for i in 2..<ri.len:
+      if params != nil: params.add(~", ")
       assert(typ.len == typ.n.len)
-      genOtherArg(p, ri, i, typ, params, argsCounter)
+      params.add(genOtherArg(p, ri, i, typ))
     fixupCall(p, le, ri, d, pl, params)
 
 proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
   # generates a crappy ObjC call
   var op: TLoc
   initLocExpr(p, ri[0], op)
-  var pl = "["
+  var pl = ~"["
   # getUniqueType() is too expensive here:
   var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
   assert(typ.len == typ.n.len)
 
   # don't call '$' here for efficiency:
-  let pat = $ri[0].sym.loc.r
+  let pat = ri[0].sym.loc.r.data
   internalAssert p.config, pat.len > 0
   var start = 3
   if ' ' in pat:
     start = 1
     pl.add(op.r)
     if ri.len > 1:
-      pl.add(": ")
-      genArg(p, ri[1], typ.n[1].sym, ri, pl)
+      pl.add(~": ")
+      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
       start = 2
   else:
     if ri.len > 1:
-      genArg(p, ri[1], typ.n[1].sym, ri, pl)
-      pl.add(" ")
+      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
+      pl.add(~" ")
     pl.add(op.r)
     if ri.len > 2:
-      pl.add(": ")
-      genArg(p, ri[2], typ.n[2].sym, ri, pl)
+      pl.add(~": ")
+      pl.add(genArg(p, ri[2], typ.n[2].sym, ri))
   for i in start..<ri.len:
     assert(typ.len == typ.n.len)
     if i >= typ.len:
       internalError(p.config, ri.info, "varargs for objective C method?")
     assert(typ.n[i].kind == nkSym)
     var param = typ.n[i].sym
-    pl.add(" ")
+    pl.add(~" ")
     pl.add(param.name.s)
-    pl.add(": ")
-    genArg(p, ri[i], param, ri, pl)
+    pl.add(~": ")
+    pl.add(genArg(p, ri[i], param, ri))
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ):
-      if ri.len > 1: pl.add(" ")
+    if isInvalidReturnType(p.config, typ[0]):
+      if ri.len > 1: pl.add(~" ")
       # beware of 'result = p(result)'. We always allocate a temporary:
       if d.k in {locTemp, locNone}:
         # We already got a temp. Great, special case it:
         if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
-        pl.add("Result: ")
+        pl.add(~"Result: ")
         pl.add(addrLoc(p.config, d))
-        pl.add("];\n")
+        pl.add(~"];$n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
         getTemp(p, typ[0], tmp, needsInit=true)
         pl.add(addrLoc(p.config, tmp))
-        pl.add("];\n")
+        pl.add(~"];$n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
     else:
-      pl.add("]")
+      pl.add(~"]")
       if d.k == locNone: getTemp(p, typ[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
@@ -788,7 +740,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
       list.r = pl
       genAssignment(p, d, list, {}) # no need for deep copying
   else:
-    pl.add("];\n")
+    pl.add(~"];$n")
     line(p, cpsStmts, pl)
 
 proc notYetAlive(n: PNode): bool {.inline.} =
@@ -826,5 +778,6 @@ proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
     genNamedParamCall(p, ri, d)
   else:
     genPrefixCall(p, le, ri, d)
+  postStmtActions(p)
 
 proc genCall(p: BProc, e: PNode, d: var TLoc) = genAsgnCall(p, nil, e, d)

compiler/ccgliterals.nim

@@ -21,7 +21,7 @@ template detectVersion(field, corename) =
     if core == nil or core.kind != skConst:
       m.g.field = 1
     else:
-      m.g.field = toInt(ast.getInt(core.astdef))
+      m.g.field = toInt(ast.getInt(core.ast))
   result = m.g.field
 
 proc detectStrVersion(m: BModule): int =
@@ -32,87 +32,82 @@ proc detectSeqVersion(m: BModule): int =
 
 # ----- Version 1: GC'ed strings and seqs --------------------------------
 
-proc genStringLiteralDataOnlyV1(m: BModule, s: string; result: var Rope) =
-  cgsym(m, "TGenericSeq")
-  let tmp = getTempName(m)
-  result.add tmp
-  m.s[cfsStrData].addf("STRING_LITERAL($1, $2, $3);$n",
-       [tmp, makeCString(s), rope(s.len)])
+proc genStringLiteralDataOnlyV1(m: BModule, s: string): Rope =
+  discard cgsym(m, "TGenericSeq")
+  result = getTempName(m)
+  m.s[cfsData].addf("STRING_LITERAL($1, $2, $3);$n",
+       [result, makeCString(s), rope(s.len)])
 
-proc genStringLiteralV1(m: BModule; n: PNode; result: var Rope) =
+proc genStringLiteralV1(m: BModule; n: PNode): Rope =
   if s.isNil:
-    appcg(m, result, "((#NimStringDesc*) NIM_NIL)", [])
+    result = ropecg(m, "((#NimStringDesc*) NIM_NIL)", [])
   else:
     let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
     if id == m.labels:
       # string literal not found in the cache:
-      appcg(m, result, "((#NimStringDesc*) &", [])
-      genStringLiteralDataOnlyV1(m, n.strVal, result)
-      result.add ")"
+      result = ropecg(m, "((#NimStringDesc*) &$1)",
+                      [genStringLiteralDataOnlyV1(m, n.strVal)])
     else:
-      appcg(m, result, "((#NimStringDesc*) &$1$2)",
+      result = ropecg(m, "((#NimStringDesc*) &$1$2)",
                       [m.tmpBase, id])
 
 # ------ Version 2: destructor based strings and seqs -----------------------
 
 proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope; isConst: bool) =
-  m.s[cfsStrData].addf("static $4 struct {$n" &
+  m.s[cfsData].addf("static $4 struct {$n" &
        "  NI cap; NIM_CHAR data[$2+1];$n" &
        "} $1 = { $2 | NIM_STRLIT_FLAG, $3 };$n",
        [result, rope(s.len), makeCString(s),
        rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool; result: var Rope) =
+proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   if id == m.labels:
     let pureLit = getTempName(m)
     genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
-    let tmp = getTempName(m)
-    result.add tmp
-    cgsym(m, "NimStrPayload")
-    cgsym(m, "NimStringV2")
+    result = getTempName(m)
+    discard cgsym(m, "NimStrPayload")
+    discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    m.s[cfsStrData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [tmp, rope(n.strVal.len), pureLit, rope(if isConst: "const" else: "")])
+    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [result, rope(n.strVal.len), pureLit, rope(if isConst: "const" else: "")])
   else:
-    let tmp = getTempName(m)
-    result.add tmp
-    m.s[cfsStrData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [tmp, rope(n.strVal.len), m.tmpBase & rope(id),
+    result = getTempName(m)
+    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [result, rope(n.strVal.len), m.tmpBase & rope(id),
           rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool; result: var Rope) =
+proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   var pureLit: Rope
   if id == m.labels:
     pureLit = getTempName(m)
-    cgsym(m, "NimStrPayload")
-    cgsym(m, "NimStringV2")
+    discard cgsym(m, "NimStrPayload")
+    discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
     genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
   else:
     pureLit = m.tmpBase & rope(id)
-  result.addf "{$1, (NimStrPayload*)&$2}", [rope(n.strVal.len), pureLit]
+  result = "{$1, (NimStrPayload*)&$2}" % [rope(n.strVal.len), pureLit]
 
 # ------ Version selector ---------------------------------------------------
 
 proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo;
-                              isConst: bool; result: var Rope) =
+                              isConst: bool): Rope =
   case detectStrVersion(m)
-  of 0, 1: genStringLiteralDataOnlyV1(m, s, result)
+  of 0, 1: result = genStringLiteralDataOnlyV1(m, s)
   of 2:
-    let tmp = getTempName(m)
-    genStringLiteralDataOnlyV2(m, s, tmp, isConst)
-    result.add tmp
+    result = getTempName(m)
+    genStringLiteralDataOnlyV2(m, s, result, isConst)
   else:
     localError(m.config, info, "cannot determine how to produce code for string literal")
 
-proc genNilStringLiteral(m: BModule; info: TLineInfo; result: var Rope) =
-  appcg(m, result, "((#NimStringDesc*) NIM_NIL)", [])
+proc genNilStringLiteral(m: BModule; info: TLineInfo): Rope =
+  result = ropecg(m, "((#NimStringDesc*) NIM_NIL)", [])
 
-proc genStringLiteral(m: BModule; n: PNode; result: var Rope) =
+proc genStringLiteral(m: BModule; n: PNode): Rope =
   case detectStrVersion(m)
-  of 0, 1: genStringLiteralV1(m, n, result)
-  of 2: genStringLiteralV2(m, n, isConst = true, result)
+  of 0, 1: result = genStringLiteralV1(m, n)
+  of 2: result = genStringLiteralV2(m, n, isConst = true)
   else:
     localError(m.config, n.info, "cannot determine how to produce code for string literal")

compiler/ccgmerge.nim

@@ -19,11 +19,13 @@ import
 
 const
   CFileSectionNames: array[TCFileSection, string] = [
+    cfsMergeInfo: "",
     cfsHeaders: "NIM_merge_HEADERS",
     cfsFrameDefines: "NIM_merge_FRAME_DEFINES",
     cfsForwardTypes: "NIM_merge_FORWARD_TYPES",
     cfsTypes: "NIM_merge_TYPES",
     cfsSeqTypes: "NIM_merge_SEQ_TYPES",
+    cfsFieldInfo: "NIM_merge_FIELD_INFO",
     cfsTypeInfo: "NIM_merge_TYPE_INFO",
     cfsProcHeaders: "NIM_merge_PROC_HEADERS",
     cfsData: "NIM_merge_DATA",
@@ -32,8 +34,11 @@ const
     cfsInitProc: "NIM_merge_INIT_PROC",
     cfsDatInitProc: "NIM_merge_DATINIT_PROC",
     cfsTypeInit1: "NIM_merge_TYPE_INIT1",
+    cfsTypeInit2: "NIM_merge_TYPE_INIT2",
     cfsTypeInit3: "NIM_merge_TYPE_INIT3",
-    cfsDynLibInit: "NIM_merge_DYNLIB_INIT"
+    cfsDebugInit: "NIM_merge_DEBUG_INIT",
+    cfsDynLibInit: "NIM_merge_DYNLIB_INIT",
+    cfsDynLibDeinit: "NIM_merge_DYNLIB_DEINIT",
   ]
   CProcSectionNames: array[TCProcSection, string] = [
     cpsLocals: "NIM_merge_PROC_LOCALS",
@@ -175,6 +180,10 @@ proc readKey(L: var TBaseLexer, result: var string) =
   if L.buf[pos] != ':': doAssert(false, "ccgmerge: ':' expected")
   L.bufpos = pos + 1 # skip ':'
 
+proc newFakeType(id: int): PType =
+  new(result)
+  result.id = id
+
 proc readTypeCache(L: var TBaseLexer, result: var TypeCache) =
   if ^L.bufpos != '{': doAssert(false, "ccgmerge: '{' expected")
   inc L.bufpos
@@ -183,7 +192,10 @@ proc readTypeCache(L: var TBaseLexer, result: var TypeCache) =
     var key = decodeStr(L.buf, L.bufpos)
     if ^L.bufpos != ':': doAssert(false, "ccgmerge: ':' expected")
     inc L.bufpos
-    discard decodeStr(L.buf, L.bufpos)
+    var value = decodeStr(L.buf, L.bufpos)
+    # XXX implement me
+    when false:
+      idTablePut(result, newFakeType(key), value.rope)
   inc L.bufpos
 
 proc readIntSet(L: var TBaseLexer, result: var IntSet) =
@@ -213,6 +225,9 @@ proc processMergeInfo(L: var TBaseLexer, m: BModule) =
       m.flags = cast[set[CodegenFlag]](decodeVInt(L.buf, L.bufpos) != 0)
     else: doAssert(false, "ccgmerge: unknown key: " & k)
 
+when not defined(nimhygiene):
+  {.pragma: inject.}
+
 template withCFile(cfilename: AbsoluteFile, body: untyped) =
   var s = llStreamOpen(cfilename, fmRead)
   if s == nil: return

compiler/ccgreset.nim

@@ -24,7 +24,7 @@ proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
   of nkRecCase:
     if (n[0].kind != nkSym): internalError(p.config, n.info, "specializeResetN")
     let disc = n[0].sym
-    if disc.loc.r == "": fillObjectFields(p.module, typ)
+    if disc.loc.r == nil: fillObjectFields(p.module, typ)
     if disc.loc.t == nil:
       internalError(p.config, n.info, "specializeResetN()")
     lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
@@ -42,7 +42,7 @@ proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
   of nkSym:
     let field = n.sym
     if field.typ.kind == tyVoid: return
-    if field.loc.r == "": fillObjectFields(p.module, typ)
+    if field.loc.r == nil: fillObjectFields(p.module, typ)
     if field.loc.t == nil:
       internalError(p.config, n.info, "specializeResetN()")
     specializeResetT(p, "$1.$2" % [accessor, field.loc.r], field.loc.t)
@@ -85,22 +85,9 @@ proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
       lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
   of tyChar, tyBool, tyEnum, tyInt..tyUInt64:
     lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
-  of tyCstring, tyPointer, tyPtr, tyVar, tyLent:
+  of tyCString, tyPointer, tyPtr, tyVar, tyLent:
     lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
-  of tySet:
-    case mapSetType(p.config, typ)
-    of ctArray:
-      lineCg(p, cpsStmts, "#nimZeroMem($1, sizeof($2));$n",
-          [accessor, getTypeDesc(p.module, typ)])
-    of ctInt8, ctInt16, ctInt32, ctInt64:
-      lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
-    else:
-      doAssert false, "unexpected set type kind"
-  of {tyNone, tyEmpty, tyNil, tyUntyped, tyTyped, tyGenericInvocation,
-      tyGenericParam, tyOrdinal, tyRange, tyOpenArray, tyForward, tyVarargs,
-      tyUncheckedArray, tyProxy, tyBuiltInTypeClass, tyUserTypeClass,
-      tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot,
-      tyAnything, tyStatic, tyFromExpr, tyConcept, tyVoid, tyIterable}:
+  else:
     discard
 
 proc specializeReset(p: BProc, a: TLoc) =

compiler/ccgstmts.nim

@@ -15,38 +15,30 @@ const
   stringCaseThreshold = 8
     # above X strings a hash-switch for strings is generated
 
-proc registerTraverseProc(p: BProc, v: PSym) =
-  var traverseProc = ""
-  if p.config.selectedGC in {gcMarkAndSweep, gcHooks, gcRefc} and
+proc getTraverseProc(p: BProc, v: PSym): Rope =
+  if p.config.selectedGC in {gcMarkAndSweep, gcHooks, gcV2, gcRefc} and
       optOwnedRefs notin p.config.globalOptions and
       containsGarbageCollectedRef(v.loc.t):
     # we register a specialized marked proc here; this has the advantage
     # that it works out of the box for thread local storage then :-)
-    traverseProc = genTraverseProcForGlobal(p.module, v, v.info)
+    result = genTraverseProcForGlobal(p.module, v, v.info)
 
-  if traverseProc.len != 0 and not p.hcrOn:
-    if sfThread in v.flags:
-      appcg(p.module, p.module.preInitProc.procSec(cpsInit),
-        "$n\t#nimRegisterThreadLocalMarker($1);$n$n", [traverseProc])
-    else:
-      appcg(p.module, p.module.preInitProc.procSec(cpsInit),
-        "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc])
+proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) =
+  if sfThread in v.flags:
+    appcg(p.module, p.module.preInitProc.procSec(cpsInit),
+      "$n\t#nimRegisterThreadLocalMarker($1);$n$n", [traverseProc])
+  else:
+    appcg(p.module, p.module.preInitProc.procSec(cpsInit),
+      "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc])
 
 proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} =
-  if n.kind == nkEmpty:
-    result = false
-  elif n.kind in nkCallKinds and n[0] != nil and n[0].typ != nil and n[0].typ.skipTypes(abstractInst).kind == tyProc:
-    if isInvalidReturnType(conf, n[0].typ, true):
-      # var v = f()
-      # is transformed into: var v;  f(addr v)
-      # where 'f' **does not** initialize the result!
-      result = false
-    else:
-      result = true
-  elif isInvalidReturnType(conf, n.typ, false):
-    result = false
-  else:
-    result = true
+  if n.kind == nkEmpty: return false
+  if isInvalidReturnType(conf, n.typ):
+    # var v = f()
+    # is transformed into: var v;  f(addr v)
+    # where 'f' **does not** initialize the result!
+    return false
+  result = true
 
 proc inExceptBlockLen(p: BProc): int =
   for x in p.nestedTryStmts:
@@ -55,9 +47,7 @@ proc inExceptBlockLen(p: BProc): int =
 proc startBlockInternal(p: BProc): int {.discardable.} =
   inc(p.labels)
   result = p.blocks.len
-
-  p.blocks.add initBlock()
-
+  setLen(p.blocks, result + 1)
   p.blocks[result].id = p.labels
   p.blocks[result].nestedTryStmts = p.nestedTryStmts.len.int16
   p.blocks[result].nestedExceptStmts = p.inExceptBlockLen.int16
@@ -76,12 +66,12 @@ proc genVarTuple(p: BProc, n: PNode) =
   # if we have a something that's been captured, use the lowering instead:
   for i in 0..<n.len-2:
     if n[i].kind != nkSym:
-      genStmts(p, lowerTupleUnpacking(p.module.g.graph, n, p.module.idgen, p.prc))
+      genStmts(p, lowerTupleUnpacking(p.module.g.graph, n, p.prc))
       return
 
   # check only the first son
   var forHcr = treatGlobalDifferentlyForHCR(p.module, n[0].sym)
-  let hcrCond = if forHcr: getTempName(p.module) else: ""
+  let hcrCond = if forHcr: getTempName(p.module) else: nil
   var hcrGlobals: seq[tuple[loc: TLoc, tp: Rope]]
   # determine if the tuple is constructed at top-level scope or inside of a block (if/while/block)
   let isGlobalInBlock = forHcr and p.blocks.len > 2
@@ -100,10 +90,13 @@ proc genVarTuple(p: BProc, n: PNode) =
     let vn = n[i]
     let v = vn.sym
     if sfCompileTime in v.flags: continue
+    var traverseProc: Rope
     if sfGlobal in v.flags:
-      assignGlobalVar(p, vn, "")
+      assignGlobalVar(p, vn, nil)
       genObjectInit(p, cpsInit, v.typ, v.loc, constructObj)
-      registerTraverseProc(p, v)
+      traverseProc = getTraverseProc(p, v)
+      if traverseProc != nil and not p.hcrOn:
+        registerTraverseProc(p, v, traverseProc)
     else:
       assignLocalVar(p, vn)
       initLocalVar(p, v, immediateAsgn=isAssignedImmediately(p.config, n[^1]))
@@ -115,7 +108,7 @@ proc genVarTuple(p: BProc, n: PNode) =
       field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(p.module, t.n[i].sym)]
     putLocIntoDest(p, v.loc, field)
     if forHcr or isGlobalInBlock:
-      hcrGlobals.add((loc: v.loc, tp: "NULL"))
+      hcrGlobals.add((loc: v.loc, tp: if traverseProc == nil: ~"NULL" else: traverseProc))
 
   if forHcr:
     # end the block where the tuple gets initialized
@@ -145,12 +138,12 @@ proc loadInto(p: BProc, le, ri: PNode, a: var TLoc) {.inline.} =
     a.flags.incl(lfEnforceDeref)
     expr(p, ri, a)
 
-proc assignLabel(b: var TBlock; result: var Rope) {.inline.} =
+proc assignLabel(b: var TBlock): Rope {.inline.} =
   b.label = "LA" & b.id.rope
-  result.add b.label
+  result = b.label
 
-proc blockBody(b: var TBlock; result: var Rope) =
-  result.add b.sections[cpsLocals]
+proc blockBody(b: var TBlock): Rope =
+  result = b.sections[cpsLocals]
   if b.frameLen > 0:
     result.addf("FR_.len+=$1;$n", [b.frameLen.rope])
   result.add(b.sections[cpsInit])
@@ -159,7 +152,7 @@ proc blockBody(b: var TBlock; result: var Rope) =
 proc endBlock(p: BProc, blockEnd: Rope) =
   let topBlock = p.blocks.len-1
   # the block is merged into the parent block
-  p.blocks[topBlock].blockBody(p.blocks[topBlock-1].sections[cpsStmts])
+  p.blocks[topBlock-1].sections[cpsStmts].add(p.blocks[topBlock].blockBody)
   setLen(p.blocks, topBlock)
   # this is done after the block is popped so $n is
   # properly indented when pretty printing is enabled
@@ -171,7 +164,7 @@ proc endBlock(p: BProc) =
   var blockEnd: Rope
   if frameLen > 0:
     blockEnd.addf("FR_.len-=$1;$n", [frameLen.rope])
-  if p.blocks[topBlock].label.len != 0:
+  if p.blocks[topBlock].label != nil:
     blockEnd.addf("} $1: ;$n", [p.blocks[topBlock].label])
   else:
     blockEnd.addf("}$n", [])
@@ -279,15 +272,17 @@ proc genGotoVar(p: BProc; value: PNode) =
   else:
     lineF(p, cpsStmts, "goto NIMSTATE_$#;$n", [value.intVal.rope])
 
-proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; result: var Rope)
+proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType): Rope
 
-proc potentialValueInit(p: BProc; v: PSym; value: PNode; result: var Rope) =
+proc potentialValueInit(p: BProc; v: PSym; value: PNode): Rope =
   if lfDynamicLib in v.loc.flags or sfThread in v.flags or p.hcrOn:
-    discard "nothing to do"
+    result = nil
   elif sfGlobal in v.flags and value != nil and isDeepConstExpr(value, p.module.compileToCpp) and
       p.withinLoop == 0 and not containsGarbageCollectedRef(v.typ):
     #echo "New code produced for ", v.name.s, " ", p.config $ value.info
-    genBracedInit(p, value, isConst = false, v.typ, result)
+    result = genBracedInit(p, value, isConst = false, v.typ)
+  else:
+    result = nil
 
 proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
   if sfGoto in v.flags:
@@ -295,8 +290,8 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     genGotoVar(p, value)
     return
   var targetProc = p
-  var valueAsRope = ""
-  potentialValueInit(p, v, value, valueAsRope)
+  var traverseProc: Rope
+  let valueAsRope = potentialValueInit(p, v, value)
   if sfGlobal in v.flags:
     if v.flags * {sfImportc, sfExportc} == {sfImportc} and
         value.kind == nkEmpty and
@@ -312,7 +307,7 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     # That's why we are doing the construction inside the preInitProc.
     # genObjectInit relies on the C runtime's guarantees that
     # global variables will be initialized to zero.
-    if valueAsRope.len == 0:
+    if valueAsRope == nil:
       var loc = v.loc
 
       # When the native TLS is unavailable, a global thread-local variable needs
@@ -326,7 +321,9 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     # if sfImportc notin v.flags: constructLoc(p.module.preInitProc, v.loc)
     if sfExportc in v.flags and p.module.g.generatedHeader != nil:
       genVarPrototype(p.module.g.generatedHeader, vn)
-    registerTraverseProc(p, v)
+    traverseProc = getTraverseProc(p, v)
+    if traverseProc != nil and not p.hcrOn:
+      registerTraverseProc(p, v, traverseProc)
   else:
     let imm = isAssignedImmediately(p.config, value)
     if imm and p.module.compileToCpp and p.splitDecls == 0 and
@@ -339,14 +336,14 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
       var tmp: TLoc
       if value.kind in nkCallKinds and value[0].kind == nkSym and
            sfConstructor in value[0].sym.flags:
-        var params = newRopeAppender()
-        var argsCounter = 0
+        var params: Rope
         let typ = skipTypes(value[0].typ, abstractInst)
         assert(typ.kind == tyProc)
         for i in 1..<value.len:
+          if params != nil: params.add(~", ")
           assert(typ.len == typ.n.len)
-          genOtherArg(p, value, i, typ, params, argsCounter)
-        if params.len == 0:
+          params.add(genOtherArg(p, value, i, typ))
+        if params == nil:
           lineF(p, cpsStmts, "$#;$n", [decl])
         else:
           lineF(p, cpsStmts, "$#($#);$n", [decl, params])
@@ -357,7 +354,7 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     assignLocalVar(p, vn)
     initLocalVar(p, v, imm)
 
-  let traverseProc = "NULL"
+  if traverseProc == nil: traverseProc = ~"NULL"
   # If the var is in a block (control flow like if/while or a block) in global scope just
   # register the so called "global" so it can be used later on. There is no need to close
   # and reopen of if (nim_hcr_do_init_) blocks because we are in one already anyway.
@@ -377,7 +374,7 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     lineCg(targetProc, cpsStmts, "if (hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1))$N",
            [v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc])
     startBlock(targetProc)
-  if value.kind != nkEmpty and valueAsRope.len == 0:
+  if value.kind != nkEmpty and valueAsRope == nil:
     genLineDir(targetProc, vn)
     loadInto(targetProc, vn, value, v.loc)
   if forHcr:
@@ -401,7 +398,7 @@ proc genClosureVar(p: BProc, a: PNode) =
   genLineDir(p, a)
   if immediateAsgn:
     loadInto(p, a[0], a[2], v)
-  elif sfNoInit notin a[0][1].sym.flags:
+  else:
     constructLoc(p, v)
 
 proc genVarStmt(p: BProc, n: PNode) =
@@ -556,9 +553,7 @@ proc genComputedGoto(p: BProc; n: PNode) =
         return
 
       let val = getOrdValue(it[j])
-      var lit = newRopeAppender()
-      intLiteral(toInt64(val)+id+1, lit)
-      lineF(p, cpsStmts, "TMP$#_:$n", [lit])
+      lineF(p, cpsStmts, "TMP$#_:$n", [intLiteral(toInt64(val)+id+1)])
 
     genStmts(p, it.lastSon)
 
@@ -612,9 +607,8 @@ proc genWhileStmt(p: BProc, t: PNode) =
       p.blocks[p.breakIdx].isLoop = true
       initLocExpr(p, t[0], a)
       if (t[0].kind != nkIntLit) or (t[0].intVal == 0):
-        lineF(p, cpsStmts, "if (!$1) goto ", [rdLoc(a)])
-        assignLabel(p.blocks[p.breakIdx], p.s(cpsStmts))
-        lineF(p, cpsStmts, ";$n", [])
+        let label = assignLabel(p.blocks[p.breakIdx])
+        lineF(p, cpsStmts, "if (!$1) goto $2;$n", [rdLoc(a), label])
       genStmts(p, loopBody)
 
       if optProfiler in p.options:
@@ -659,19 +653,12 @@ proc genParForStmt(p: BProc, t: PNode) =
     initLocExpr(p, call[2], rangeB)
 
     # $n at the beginning because of #9710
-    if call.len == 4: # procName(a, b, annotation)
-      if call[0].sym.name.s == "||":  # `||`(a, b, annotation)
-        lineF(p, cpsStmts, "$n#pragma omp $4$n" &
-                            "for ($1 = $2; $1 <= $3; ++$1)",
-                            [forLoopVar.loc.rdLoc,
-                            rangeA.rdLoc, rangeB.rdLoc,
-                            call[3].getStr.rope])
-      else:
-        lineF(p, cpsStmts, "$n#pragma $4$n" &
-                    "for ($1 = $2; $1 <= $3; ++$1)",
-                    [forLoopVar.loc.rdLoc,
-                    rangeA.rdLoc, rangeB.rdLoc,
-                    call[3].getStr.rope])
+    if call.len == 4: # `||`(a, b, annotation)
+      lineF(p, cpsStmts, "$n#pragma omp $4$n" &
+                          "for ($1 = $2; $1 <= $3; ++$1)",
+                          [forLoopVar.loc.rdLoc,
+                          rangeA.rdLoc, rangeB.rdLoc,
+                          call[3].getStr.rope])
     else: # `||`(a, b, step, annotation)
       var step: TLoc
       initLocExpr(p, call[3], step)
@@ -701,12 +688,12 @@ proc genBreakStmt(p: BProc, t: PNode) =
     while idx >= 0 and not p.blocks[idx].isLoop: dec idx
     if idx < 0 or not p.blocks[idx].isLoop:
       internalError(p.config, t.info, "no loop to break")
-  p.blocks[idx].label = "LA" & p.blocks[idx].id.rope
+  let label = assignLabel(p.blocks[idx])
   blockLeaveActions(p,
     p.nestedTryStmts.len - p.blocks[idx].nestedTryStmts,
     p.inExceptBlockLen - p.blocks[idx].nestedExceptStmts)
   genLineDir(p, t)
-  lineF(p, cpsStmts, "goto $1;$n", [p.blocks[idx].label])
+  lineF(p, cpsStmts, "goto $1;$n", [label])
 
 proc raiseExit(p: BProc) =
   assert p.config.exc == excGoto
@@ -728,19 +715,21 @@ proc finallyActions(p: BProc) =
     if finallyBlock != nil:
       genSimpleBlock(p, finallyBlock[0])
 
-proc raiseInstr(p: BProc; result: var Rope) =
+proc raiseInstr(p: BProc): Rope =
   if p.config.exc == excGoto:
     let L = p.nestedTryStmts.len
     if L == 0:
       p.flags.incl beforeRetNeeded
       # easy case, simply goto 'ret':
-      result.add ropecg(p.module, "goto BeforeRet_;$n", [])
+      result = ropecg(p.module, "goto BeforeRet_;$n", [])
     else:
       # raise inside an 'except' must go to the finally block,
       # raise outside an 'except' block must go to the 'except' list.
-      result.add ropecg(p.module, "goto LA$1_;$n",
+      result = ropecg(p.module, "goto LA$1_;$n",
         [p.nestedTryStmts[L-1].label])
       # + ord(p.nestedTryStmts[L-1].inExcept)])
+  else:
+    result = nil
 
 proc genRaiseStmt(p: BProc, t: PNode) =
   if t[0].kind != nkEmpty:
@@ -769,10 +758,12 @@ proc genRaiseStmt(p: BProc, t: PNode) =
     genLineDir(p, t)
     # reraise the last exception:
     if p.config.exc == excCpp:
-      line(p, cpsStmts, "throw;\n")
+      line(p, cpsStmts, ~"throw;$n")
     else:
       linefmt(p, cpsStmts, "#reraiseException();$n", [])
-  raiseInstr(p, p.s(cpsStmts))
+  let gotoInstr = raiseInstr(p)
+  if gotoInstr != nil:
+    line(p, cpsStmts, gotoInstr)
 
 template genCaseGenericBranch(p: BProc, b: PNode, e: TLoc,
                           rangeFormat, eqFormat: FormatStr, labl: TLabel) =
@@ -832,27 +823,17 @@ template genCaseGeneric(p: BProc, t: PNode, d: var TLoc,
   fixLabel(p, lend)
 
 proc genCaseStringBranch(p: BProc, b: PNode, e: TLoc, labl: TLabel,
-                         stringKind: TTypeKind,
                          branches: var openArray[Rope]) =
   var x: TLoc
   for i in 0..<b.len - 1:
     assert(b[i].kind != nkRange)
     initLocExpr(p, b[i], x)
-    var j: int
-    case b[i].kind
-    of nkStrLit..nkTripleStrLit:
-      j = int(hashString(p.config, b[i].strVal) and high(branches))
-    of nkNilLit: j = 0
-    else:
-      assert false, "invalid string case branch node kind"
-    if stringKind == tyCstring:
-      appcg(p.module, branches[j], "if (#eqCstrings($1, $2)) goto $3;$n",
-         [rdLoc(e), rdLoc(x), labl])
-    else:
-      appcg(p.module, branches[j], "if (#eqStrings($1, $2)) goto $3;$n",
+    assert(b[i].kind in {nkStrLit..nkTripleStrLit})
+    var j = int(hashString(p.config, b[i].strVal) and high(branches))
+    appcg(p.module, branches[j], "if (#eqStrings($1, $2)) goto $3;$n",
          [rdLoc(e), rdLoc(x), labl])
 
-proc genStringCase(p: BProc, t: PNode, stringKind: TTypeKind, d: var TLoc) =
+proc genStringCase(p: BProc, t: PNode, d: var TLoc) =
   # count how many constant strings there are in the case:
   var strings = 0
   for i in 1..<t.len:
@@ -868,23 +849,17 @@ proc genStringCase(p: BProc, t: PNode, stringKind: TTypeKind, d: var TLoc) =
       inc(p.labels)
       if t[i].kind == nkOfBranch:
         genCaseStringBranch(p, t[i], a, "LA" & rope(p.labels) & "_",
-                            stringKind, branches)
+                            branches)
       else:
         # else statement: nothing to do yet
         # but we reserved a label, which we use later
         discard
-    if stringKind == tyCstring:
-      linefmt(p, cpsStmts, "switch (#hashCstring($1) & $2) {$n",
-              [rdLoc(a), bitMask])
-    else:
-      linefmt(p, cpsStmts, "switch (#hashString($1) & $2) {$n",
-              [rdLoc(a), bitMask])
+    linefmt(p, cpsStmts, "switch (#hashString($1) & $2) {$n",
+            [rdLoc(a), bitMask])
     for j in 0..high(branches):
-      if branches[j] != "":
-        var lit = newRopeAppender()
-        intLiteral(j, lit)
+      if branches[j] != nil:
         lineF(p, cpsStmts, "case $1: $n$2break;$n",
-             [lit, branches[j]])
+             [intLiteral(j), branches[j]])
     lineF(p, cpsStmts, "}$n", []) # else statement:
     if t[^1].kind != nkOfBranch:
       lineF(p, cpsStmts, "goto LA$1_;$n", [rope(p.labels)])
@@ -892,10 +867,7 @@ proc genStringCase(p: BProc, t: PNode, stringKind: TTypeKind, d: var TLoc) =
     var lend = genCaseSecondPass(p, t, d, labId, t.len-1)
     fixLabel(p, lend)
   else:
-    if stringKind == tyCstring:
-      genCaseGeneric(p, t, d, "", "if (#eqCstrings($1, $2)) goto $3;$n")
-    else:
-      genCaseGeneric(p, t, d, "", "if (#eqStrings($1, $2)) goto $3;$n")
+    genCaseGeneric(p, t, d, "", "if (#eqStrings($1, $2)) goto $3;$n")
 
 proc branchHasTooBigRange(b: PNode): bool =
   for it in b:
@@ -918,22 +890,16 @@ proc genCaseRange(p: BProc, branch: PNode) =
   for j in 0..<branch.len-1:
     if branch[j].kind == nkRange:
       if hasSwitchRange in CC[p.config.cCompiler].props:
-        var litA = newRopeAppender()
-        var litB = newRopeAppender()
-        genLiteral(p, branch[j][0], litA)
-        genLiteral(p, branch[j][1], litB)
-        lineF(p, cpsStmts, "case $1 ... $2:$n", [litA, litB])
+        lineF(p, cpsStmts, "case $1 ... $2:$n", [
+            genLiteral(p, branch[j][0]),
+            genLiteral(p, branch[j][1])])
       else:
         var v = copyNode(branch[j][0])
         while v.intVal <= branch[j][1].intVal:
-          var litA = newRopeAppender()
-          genLiteral(p, v, litA)
-          lineF(p, cpsStmts, "case $1:$n", [litA])
+          lineF(p, cpsStmts, "case $1:$n", [genLiteral(p, v)])
           inc(v.intVal)
     else:
-      var litA = newRopeAppender()
-      genLiteral(p, branch[j], litA)
-      lineF(p, cpsStmts, "case $1:$n", [litA])
+      lineF(p, cpsStmts, "case $1:$n", [genLiteral(p, branch[j])])
 
 proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) =
   # analyse 'case' statement:
@@ -945,7 +911,7 @@ proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) =
   var lend = if splitPoint > 0: genIfForCaseUntil(p, n, d,
                     rangeFormat = "if ($1 >= $2 && $1 <= $3) goto $4;$n",
                     eqFormat = "if ($1 == $2) goto $3;$n",
-                    splitPoint, a) else: ""
+                    splitPoint, a) else: nil
 
   # generate switch part (might be empty):
   if splitPoint+1 < n.len:
@@ -966,7 +932,7 @@ proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) =
     if (hasAssume in CC[p.config.cCompiler].props) and not hasDefault:
       lineF(p, cpsStmts, "default: __assume(0);$n", [])
     lineF(p, cpsStmts, "}$n", [])
-  if lend != "": fixLabel(p, lend)
+  if lend != nil: fixLabel(p, lend)
 
 proc genCase(p: BProc, t: PNode, d: var TLoc) =
   genLineDir(p, t)
@@ -974,9 +940,7 @@ proc genCase(p: BProc, t: PNode, d: var TLoc) =
     getTemp(p, t.typ, d)
   case skipTypes(t[0].typ, abstractVarRange).kind
   of tyString:
-    genStringCase(p, t, tyString, d)
-  of tyCstring:
-    genStringCase(p, t, tyCstring, d)
+    genStringCase(p, t, d)
   of tyFloat..tyFloat128:
     genCaseGeneric(p, t, d, "if ($1 >= $2 && $1 <= $3) goto $4;$n",
                             "if ($1 == $2) goto $3;$n")
@@ -1067,7 +1031,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
       linefmt(p, cpsStmts, "#popCurrentException();$n", [])
       endBlock(p)
     else:
-      var orExpr = newRopeAppender()
+      var orExpr = Rope(nil)
       var exvar = PNode(nil)
       for j in 0..<t[i].len - 1:
         var typeNode = t[i][j]
@@ -1078,24 +1042,22 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
         if isImportedException(typeNode.typ, p.config):
           hasImportedCppExceptions = true
         else:
-          if orExpr.len != 0: orExpr.add("||")
-          let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
-          if optTinyRtti in p.config.globalOptions:
-            let checkFor = $getObjDepth(typeNode.typ)
-            appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(typeNode.typ)))])
+          if orExpr != nil: orExpr.add("||")
+          let checkFor = if optTinyRtti in p.config.globalOptions:
+            genTypeInfo2Name(p.module, typeNode.typ)
           else:
-            let checkFor = genTypeInfoV1(p.module, typeNode.typ, typeNode.info)
-            appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
+            genTypeInfoV1(p.module, typeNode.typ, typeNode.info)
+          let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
+          appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
 
-      if orExpr.len != 0:
+      if orExpr != nil:
         if hasIf:
           startBlock(p, "else if ($1) {$n", [orExpr])
         else:
           startBlock(p, "if ($1) {$n", [orExpr])
           hasIf = true
         if exvar != nil:
-          fillLocalName(p, exvar.sym)
-          fillLoc(exvar.sym.loc, locTemp, exvar, OnStack)
+          fillLoc(exvar.sym.loc, locTemp, exvar, mangleLocalName(p, exvar.sym), OnStack)
           linefmt(p, cpsStmts, "$1 $2 = T$3_;$n", [getTypeDesc(p.module, exvar.sym.typ),
             rdLoc(exvar.sym.loc), rope(etmp+1)])
         # we handled the error:
@@ -1136,8 +1098,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
             typeNode = t[i][j][1]
             if isImportedException(typeNode.typ, p.config):
               let exvar = t[i][j][2] # ex1 in `except ExceptType as ex1:`
-              fillLocalName(p, exvar.sym)
-              fillLoc(exvar.sym.loc, locTemp, exvar, OnStack)
+              fillLoc(exvar.sym.loc, locTemp, exvar, mangleLocalName(p, exvar.sym), OnStack)
               startBlock(p, "catch ($1& $2) {$n", getTypeDesc(p.module, typeNode.typ), rdLoc(exvar.sym.loc))
               genExceptBranchBody(t[i][^1])  # exception handler body will duplicated for every type
               endBlock(p)
@@ -1190,7 +1151,7 @@ proc genTryCppOld(p: BProc, t: PNode, d: var TLoc) =
   if not isEmptyType(t.typ) and d.k == locNone:
     getTemp(p, t.typ, d)
   genLineDir(p, t)
-  cgsym(p.module, "popCurrentExceptionEx")
+  discard cgsym(p.module, "popCurrentExceptionEx")
   let fin = if t[^1].kind == nkFinally: t[^1] else: nil
   p.nestedTryStmts.add((fin, false, 0.Natural))
   startBlock(p, "try {$n")
@@ -1216,8 +1177,7 @@ proc genTryCppOld(p: BProc, t: PNode, d: var TLoc) =
       for j in 0..<t[i].len-1:
         if t[i][j].isInfixAs():
           let exvar = t[i][j][2] # ex1 in `except ExceptType as ex1:`
-          fillLocalName(p, exvar.sym)
-          fillLoc(exvar.sym.loc, locTemp, exvar, OnUnknown)
+          fillLoc(exvar.sym.loc, locTemp, exvar, mangleLocalName(p, exvar.sym), OnUnknown)
           startBlock(p, "catch ($1& $2) {$n", getTypeDesc(p.module, t[i][j][1].typ), rdLoc(exvar.sym.loc))
         else:
           startBlock(p, "catch ($1&) {$n", getTypeDesc(p.module, t[i][j].typ))
@@ -1232,7 +1192,7 @@ proc genTryCppOld(p: BProc, t: PNode, d: var TLoc) =
       # finally requires catch all presence
       startBlock(p, "catch (...) {$n")
       genStmts(p, t[^1][0])
-      line(p, cpsStmts, "throw;\n")
+      line(p, cpsStmts, ~"throw;$n")
       endBlock(p)
 
     genSimpleBlock(p, t[^1][0])
@@ -1294,17 +1254,16 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
       linefmt(p, cpsStmts, "*nimErr_ = NIM_FALSE;$n", [])
       expr(p, t[i][0], d)
     else:
-      var orExpr = newRopeAppender()
+      var orExpr: Rope = nil
       for j in 0..<t[i].len - 1:
         assert(t[i][j].kind == nkType)
-        if orExpr.len != 0: orExpr.add("||")
-        let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
-        if optTinyRtti in p.config.globalOptions:
-          let checkFor = $getObjDepth(t[i][j].typ)
-          appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor,  $genDisplayElem(MD5Digest(hashType(t[i][j].typ)))])
+        if orExpr != nil: orExpr.add("||")
+        let checkFor = if optTinyRtti in p.config.globalOptions:
+          genTypeInfo2Name(p.module, t[i][j].typ)
         else:
-          let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
-          appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
+          genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
+        let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
+        appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
 
       if i > 1: line(p, cpsStmts, "else ")
       startBlock(p, "if ($1) {$n", [orExpr])
@@ -1380,7 +1339,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
   else:
     p.flags.incl noSafePoints
   genLineDir(p, t)
-  cgsym(p.module, "Exception")
+  discard cgsym(p.module, "Exception")
   var safePoint: Rope
   if not quirkyExceptions:
     safePoint = getTempName(p.module)
@@ -1390,24 +1349,8 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
       linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
     elif isDefined(p.config, "nimSigSetjmp"):
       linefmt(p, cpsStmts, "$1.status = sigsetjmp($1.context, 0);$n", [safePoint])
-    elif isDefined(p.config, "nimBuiltinSetjmp"):
-      linefmt(p, cpsStmts, "$1.status = __builtin_setjmp($1.context);$n", [safePoint])
     elif isDefined(p.config, "nimRawSetjmp"):
-      if isDefined(p.config, "mswindows"):
-        if isDefined(p.config, "vcc") or isDefined(p.config, "clangcl"):
-          # For the vcc compiler, use `setjmp()` with one argument.
-          # See https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setjmp?view=msvc-170
-          linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
-        else:
-          # The Windows `_setjmp()` takes two arguments, with the second being an
-          # undocumented buffer used by the SEH mechanism for stack unwinding.
-          # Mingw-w64 has been trying to get it right for years, but it's still
-          # prone to stack corruption during unwinding, so we disable that by setting
-          # it to NULL.
-          # More details: https://github.com/status-im/nimbus-eth2/issues/3121
-          linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint])
-      else:
-        linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint])
+      linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint])
     else:
       linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
     lineCg(p, cpsStmts, "if ($1.status == 0) {$n", [safePoint])
@@ -1439,17 +1382,16 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
       linefmt(p, cpsStmts, "#popCurrentException();$n", [])
       endBlock(p)
     else:
-      var orExpr = newRopeAppender()
+      var orExpr: Rope = nil
       for j in 0..<t[i].len - 1:
         assert(t[i][j].kind == nkType)
-        if orExpr.len != 0: orExpr.add("||")
-        let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
-        if optTinyRtti in p.config.globalOptions:
-          let checkFor = $getObjDepth(t[i][j].typ)
-          appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor,  $genDisplayElem(MD5Digest(hashType(t[i][j].typ)))])
+        if orExpr != nil: orExpr.add("||")
+        let checkFor = if optTinyRtti in p.config.globalOptions:
+          genTypeInfo2Name(p.module, t[i][j].typ)
         else:
-          let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
-          appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
+          genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
+        let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
+        appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
 
       if i > 1: line(p, cpsStmts, "else ")
       startBlock(p, "if ($1) {$n", [orExpr])
@@ -1475,7 +1417,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
   if not quirkyExceptions:
     linefmt(p, cpsStmts, "if ($1.status != 0) #reraiseException();$n", [safePoint])
 
-proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false; result: var Rope) =
+proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
   var res = ""
   for it in t.sons:
     case it.kind
@@ -1491,8 +1433,13 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false; result: var Rope) =
         res.add($getTypeDesc(p.module, sym.typ))
       else:
         discard getTypeDesc(p.module, skipTypes(sym.typ, abstractPtrs))
-        fillBackendName(p.module, sym)
-        res.add($sym.loc.r)
+        var r = sym.loc.r
+        if r == nil:
+          # if no name has already been given,
+          # it doesn't matter much:
+          r = mangleName(p.module, sym)
+          sym.loc.r = r       # but be consequent!
+        res.add($r)
     of nkTypeOfExpr:
       res.add($getTypeDesc(p.module, it.typ))
     else:
@@ -1517,13 +1464,12 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false; result: var Rope) =
           result.add("\\n\"\n")
   else:
     res.add("\L")
-    result.add res.rope
+    result = res.rope
 
 proc genAsmStmt(p: BProc, t: PNode) =
   assert(t.kind == nkAsmStmt)
   genLineDir(p, t)
-  var s = newRopeAppender()
-  genAsmOrEmitStmt(p, t, isAsmStmt=true, s)
+  var s = genAsmOrEmitStmt(p, t, isAsmStmt=true)
   # see bug #2362, "top level asm statements" seem to be a mis-feature
   # but even if we don't do this, the example in #2362 cannot possibly
   # work:
@@ -1531,8 +1477,7 @@ proc genAsmStmt(p: BProc, t: PNode) =
     # top level asm statement?
     p.module.s[cfsProcHeaders].add runtimeFormat(CC[p.config.cCompiler].asmStmtFrmt, [s])
   else:
-    addIndent p, p.s(cpsStmts)
-    p.s(cpsStmts).add runtimeFormat(CC[p.config.cCompiler].asmStmtFrmt, [s])
+    p.s(cpsStmts).add indentLine(p, runtimeFormat(CC[p.config.cCompiler].asmStmtFrmt, [s]))
 
 proc determineSection(n: PNode): TCFileSection =
   result = cfsProcHeaders
@@ -1543,8 +1488,7 @@ proc determineSection(n: PNode): TCFileSection =
     elif sec.startsWith("/*INCLUDESECTION*/"): result = cfsHeaders
 
 proc genEmit(p: BProc, t: PNode) =
-  var s = newRopeAppender()
-  genAsmOrEmitStmt(p, t[1], false, s)
+  var s = genAsmOrEmitStmt(p, t[1])
   if p.prc == nil:
     # top level emit pragma?
     let section = determineSection(t[1])
@@ -1558,6 +1502,11 @@ proc genPragma(p: BProc, n: PNode) =
   for it in n.sons:
     case whichPragma(it)
     of wEmit: genEmit(p, it)
+    of wInjectStmt:
+      var p = newProc(nil, p.module)
+      p.options.excl {optLineTrace, optStackTrace}
+      genStmts(p, it[1])
+      p.module.injectStmt = p.s(cpsStmts)
     else: discard
 
 
@@ -1569,28 +1518,25 @@ proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
   if not containsOrIncl(p.module.declaredThings, field.id):
     appcg(p.module, cfsVars, "extern $1",
           [discriminatorTableDecl(p.module, t, field)])
-  var lit = newRopeAppender()
-  intLiteral(toInt64(lengthOrd(p.config, field.typ))+1, lit)
   lineCg(p, cpsStmts,
         "#FieldDiscriminantCheck((NI)(NU)($1), (NI)(NU)($2), $3, $4);$n",
         [rdLoc(a), rdLoc(tmp), discriminatorTableName(p.module, t, field),
-         lit])
+         intLiteral(toInt64(lengthOrd(p.config, field.typ))+1)])
 
-when false:
-  proc genCaseObjDiscMapping(p: BProc, e: PNode, t: PType, field: PSym; d: var TLoc) =
-    const ObjDiscMappingProcSlot = -5
-    var theProc: PSym = nil
-    for idx, p in items(t.methods):
-      if idx == ObjDiscMappingProcSlot:
-        theProc = p
-        break
-    if theProc == nil:
-      theProc = genCaseObjDiscMapping(t, field, e.info, p.module.g.graph, p.module.idgen)
-      t.methods.add((ObjDiscMappingProcSlot, theProc))
-    var call = newNodeIT(nkCall, e.info, getSysType(p.module.g.graph, e.info, tyUInt8))
-    call.add newSymNode(theProc)
-    call.add e
-    expr(p, call, d)
+proc genCaseObjDiscMapping(p: BProc, e: PNode, t: PType, field: PSym; d: var TLoc) =
+  const ObjDiscMappingProcSlot = -5
+  var theProc: PSym = nil
+  for idx, p in items(t.methods):
+    if idx == ObjDiscMappingProcSlot:
+      theProc = p
+      break
+  if theProc == nil:
+    theProc = genCaseObjDiscMapping(t, field, e.info, p.module.g.graph)
+    t.methods.add((ObjDiscMappingProcSlot, theProc))
+  var call = newNodeIT(nkCall, e.info, getSysType(p.module.g.graph, e.info, tyUInt8))
+  call.add newSymNode(theProc)
+  call.add e
+  expr(p, call, d)
 
 proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   var a, tmp: TLoc
@@ -1599,7 +1545,7 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   initLocExpr(p, e[0], a)
   getTemp(p, a.t, tmp)
   expr(p, e[1], tmp)
-  if optTinyRtti notin p.config.globalOptions and p.inUncheckedAssignSection == 0:
+  if optTinyRtti notin p.config.globalOptions:
     let field = dotExpr[1].sym
     genDiscriminantCheck(p, a, tmp, dotExpr[0].typ, field)
     message(p.config, e.info, warnCaseTransition)

compiler/ccgthreadvars.nim

@@ -7,8 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
-## Thread var support for architectures that lack native support for
-## thread local storage.
+## Thread var support for crappy architectures that lack native support for
+## thread local storage. (**Thank you Mac OS X!**)
 
 # included from cgen.nim
 
@@ -35,22 +35,18 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
     if isExtern: m.s[cfsVars].add("extern ")
     elif lfExportLib in s.loc.flags: m.s[cfsVars].add("N_LIB_EXPORT_VAR ")
     else: m.s[cfsVars].add("N_LIB_PRIVATE ")
-    if optThreads in m.config.globalOptions:
-      let sym = s.typ.sym
-      if sym != nil and sfCppNonPod in sym.flags:
-        m.s[cfsVars].add("NIM_THREAD_LOCAL ")
-      else: m.s[cfsVars].add("NIM_THREADVAR ")
+    if optThreads in m.config.globalOptions: m.s[cfsVars].add("NIM_THREADVAR ")
     m.s[cfsVars].add(getTypeDesc(m, s.loc.t))
     m.s[cfsVars].addf(" $1;$n", [s.loc.r])
 
 proc generateThreadLocalStorage(m: BModule) =
-  if m.g.nimtv != "" and (usesThreadVars in m.flags or sfMainModule in m.module.flags):
+  if m.g.nimtv != nil and (usesThreadVars in m.flags or sfMainModule in m.module.flags):
     for t in items(m.g.nimtvDeps): discard getTypeDesc(m, t)
     finishTypeDescriptions(m)
     m.s[cfsSeqTypes].addf("typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
 
 proc generateThreadVarsSize(m: BModule) =
-  if m.g.nimtv != "":
+  if m.g.nimtv != nil:
     let externc = if m.config.backend == backendCpp or
                        sfCompileToCpp in m.module.flags: "extern \"C\" "
                   else: ""

compiler/ccgtrav.nim

@@ -34,7 +34,7 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
     if (n[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
     var p = c.p
     let disc = n[0].sym
-    if disc.loc.r == "": fillObjectFields(c.p.module, typ)
+    if disc.loc.r == nil: fillObjectFields(c.p.module, typ)
     if disc.loc.t == nil:
       internalError(c.p.config, n.info, "genTraverseProc()")
     lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
@@ -51,7 +51,7 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
   of nkSym:
     let field = n.sym
     if field.typ.kind == tyVoid: return
-    if field.loc.r == "": fillObjectFields(c.p.module, typ)
+    if field.loc.r == nil: fillObjectFields(c.p.module, typ)
     if field.loc.t == nil:
       internalError(c.p.config, n.info, "genTraverseProc()")
     genTraverseProc(c, "$1.$2" % [accessor, field.loc.r], field.loc.t)
@@ -76,8 +76,7 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType) =
     let arraySize = lengthOrd(c.p.config, typ[0])
     var i: TLoc
     getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
-    var oldCode = p.s(cpsStmts)
-    freeze oldCode
+    let oldCode = p.s(cpsStmts)
     linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
             [i.r, arraySize])
     let oldLen = p.s(cpsStmts).len
@@ -121,8 +120,7 @@ proc genTraverseProcSeq(c: TTraversalClosure, accessor: Rope, typ: PType) =
   assert typ.kind == tySequence
   var i: TLoc
   getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
-  var oldCode = p.s(cpsStmts)
-  freeze oldCode
+  let oldCode = p.s(cpsStmts)
   var a: TLoc
   a.r = accessor
 

compiler/ccgutils.nim

@@ -10,11 +10,8 @@
 # This module declares some helpers for the C code generator.
 
 import
-  ast, types, hashes, strutils, msgs, wordrecg,
-  platform, trees, options, cgendata
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
+  ast, hashes, strutils, msgs, wordrecg,
+  platform, trees, options
 
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
   case n.kind
@@ -109,39 +106,3 @@ proc mangle*(name: string): string =
       requiresUnderscore = true
   if requiresUnderscore:
     result.add "_"
-
-proc mapSetType(conf: ConfigRef; typ: PType): TCTypeKind =
-  case int(getSize(conf, typ))
-  of 1: result = ctInt8
-  of 2: result = ctInt16
-  of 4: result = ctInt32
-  of 8: result = ctInt64
-  else: result = ctArray
-
-proc ccgIntroducedPtr*(conf: ConfigRef; s: PSym, retType: PType): bool =
-  var pt = skipTypes(s.typ, typedescInst)
-  assert skResult != s.kind
-
-  if tfByRef in pt.flags: return true
-  elif tfByCopy in pt.flags: return false
-  case pt.kind
-  of tyObject:
-    if s.typ.sym != nil and sfForward in s.typ.sym.flags:
-      # forwarded objects are *always* passed by pointers for consistency!
-      result = true
-    elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
-      result = true           # requested anyway
-    elif (tfFinal in pt.flags) and (pt[0] == nil):
-      result = false          # no need, because no subtyping possible
-    else:
-      result = true           # ordinary objects are always passed by reference,
-                              # otherwise casting doesn't work
-  of tyTuple:
-    result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options)
-  else:
-    result = false
-  # first parameter and return type is 'lent T'? --> use pass by pointer
-  if s.position == 0 and retType != nil and retType.kind == tyLent:
-    result = not (pt.kind in {tyVar, tyArray, tyOpenArray, tyVarargs, tyRef, tyPtr, tyPointer} or
-      pt.kind == tySet and mapSetType(conf, pt) == ctArray)
-

compiler/cgendata.nim

@@ -16,6 +16,7 @@ import
 type
   TLabel* = Rope              # for the C generator a label is just a rope
   TCFileSection* = enum       # the sections a generated C file consists of
+    cfsMergeInfo,             # section containing merge information
     cfsHeaders,               # section for C include file headers
     cfsFrameDefines           # section for nim frame macros
     cfsForwardTypes,          # section for C forward typedefs
@@ -23,17 +24,21 @@ type
     cfsSeqTypes,              # section for sequence types only
                               # this is needed for strange type generation
                               # reasons
+    cfsFieldInfo,             # section for field information
     cfsTypeInfo,              # section for type information (ag ABI checks)
     cfsProcHeaders,           # section for C procs prototypes
-    cfsStrData,               # section for constant string literals
     cfsData,                  # section for C constant data
     cfsVars,                  # section for C variable declarations
     cfsProcs,                 # section for C procs that are not inline
     cfsInitProc,              # section for the C init proc
     cfsDatInitProc,           # section for the C datInit proc
     cfsTypeInit1,             # section 1 for declarations of type information
+    cfsTypeInit2,             # section 2 for init of type information
     cfsTypeInit3,             # section 3 for init of type information
+    cfsDebugInit,             # section for init of debug information
     cfsDynLibInit,            # section for init of dynamic library binding
+    cfsDynLibDeinit           # section for deinitialization of dynamic
+                              # libraries
   TCTypeKind* = enum          # describes the type kind of a C type
     ctVoid, ctChar, ctBool,
     ctInt, ctInt8, ctInt16, ctInt32, ctInt64,
@@ -94,11 +99,10 @@ type
     withinTryWithExcept*: int # required for goto based exception handling
     withinBlockLeaveActions*: int # complex to explain
     sigConflicts*: CountTable[string]
-    inUncheckedAssignSection*: int
 
   TTypeSeq* = seq[PType]
   TypeCache* = Table[SigHash, Rope]
-  TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: int32]]
+  TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: PSym]]
 
   CodegenFlag* = enum
     preventStackTrace,  # true if stack traces need to be prevented
@@ -108,8 +112,7 @@ type
     isHeaderFile,       # C source file is the header file
     includesStringh,    # C source file already includes ``<string.h>``
     objHasKidsValid     # whether we can rely on tfObjHasKids
-    useAliveDataFromDce # use the `alive: IntSet` field instead of
-                        # computing alive data on our own.
+
 
   BModuleList* = ref object of RootObj
     mainModProcs*, mainModInit*, otherModsInit*, mainDatInit*: Rope
@@ -151,7 +154,6 @@ type
     forwTypeCache*: TypeCache # cache for forward declarations of types
     declaredThings*: IntSet   # things we have declared in this .c file
     declaredProtos*: IntSet   # prototypes we have declared in this .c file
-    alive*: IntSet            # symbol IDs of alive data as computed by `dce.nim`
     headerFiles*: seq[string] # needed headers to include
     typeInfoMarker*: TypeCache # needed for generating type information
     typeInfoMarkerV2*: TypeCache
@@ -166,6 +168,7 @@ type
     labels*: Natural          # for generating unique module-scope names
     extensionLoaders*: array['0'..'9', Rope] # special procs for the
                                              # OpenGL wrapper
+    injectStmt*: Rope
     sigConflicts*: CountTable[SigHash]
     g*: BModuleList
     ndi*: NdiFile
@@ -185,24 +188,19 @@ proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # top level proc sections
   result = p.blocks[0].sections[s]
 
-proc initBlock*(): TBlock =
-  result = TBlock()
-  for i in low(result.sections)..high(result.sections):
-    result.sections[i] = newRopeAppender()
-
 proc newProc*(prc: PSym, module: BModule): BProc =
   new(result)
   result.prc = prc
   result.module = module
   result.options = if prc != nil: prc.options
                    else: module.config.options
-  result.blocks = @[initBlock()]
+  newSeq(result.blocks, 1)
   result.nestedTryStmts = @[]
   result.finallySafePoints = @[]
   result.sigConflicts = initCountTable[string]()
 
 proc newModuleList*(g: ModuleGraph): BModuleList =
-  BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: int32]](),
+  BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: PSym]](),
     config: g.config, graph: g, nimtvDeclared: initIntSet())
 
 iterator cgenModules*(g: BModuleList): BModule =

compiler/cgmeth.nim

@@ -11,11 +11,7 @@
 
 import
   intsets, options, ast, msgs, idents, renderer, types, magicsys,
-  sempass2, modulegraphs, lineinfos
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
+  sempass2, strutils, modulegraphs, lineinfos
 
 proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
   var dest = skipTypes(d, abstractPtrs)
@@ -50,7 +46,6 @@ proc methodCall*(n: PNode; conf: ConfigRef): PNode =
   # replace ordinary method by dispatcher method:
   let disp = getDispatcher(result[0].sym)
   if disp != nil:
-    result[0].typ = disp.typ
     result[0].sym = disp
     # change the arguments to up/downcasts to fit the dispatcher's parameters:
     for i in 1..<result.len:
@@ -72,7 +67,7 @@ proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
     while true:
       aa = skipTypes(aa, {tyGenericInst, tyAlias})
       bb = skipTypes(bb, {tyGenericInst, tyAlias})
-      if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef, tyLent, tySink}:
+      if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef, tyLent}:
         aa = aa.lastSon
         bb = bb.lastSon
       else:
@@ -112,22 +107,19 @@ proc attachDispatcher(s: PSym, dispatcher: PNode) =
       s.ast[resultPos] = newNodeI(nkEmpty, s.info)
     s.ast[dispatcherPos] = dispatcher
 
-proc createDispatcher(s: PSym; g: ModuleGraph; idgen: IdGenerator): PSym =
-  var disp = copySym(s, nextSymId(idgen))
+proc createDispatcher(s: PSym): PSym =
+  var disp = copySym(s)
   incl(disp.flags, sfDispatcher)
   excl(disp.flags, sfExported)
-  let old = disp.typ
-  disp.typ = copyType(disp.typ, nextTypeId(idgen), disp.typ.owner)
-  copyTypeProps(g, idgen.module, disp.typ, old)
-
+  disp.typ = copyType(disp.typ, disp.typ.owner, false)
   # we can't inline the dispatcher itself (for now):
   if disp.typ.callConv == ccInline: disp.typ.callConv = ccNimCall
   disp.ast = copyTree(s.ast)
   disp.ast[bodyPos] = newNodeI(nkEmpty, s.info)
-  disp.loc.r = ""
+  disp.loc.r = nil
   if s.typ[0] != nil:
     if disp.ast.len > resultPos:
-      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, nextSymId(idgen))
+      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym)
     else:
       # We've encountered a method prototype without a filled-in
       # resultPos slot. We put a placeholder in there that will
@@ -148,7 +140,24 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
      disp.ast[resultPos].kind == nkEmpty:
     disp.ast[resultPos] = copyTree(meth.ast[resultPos])
 
-proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
+  # The following code works only with lock levels, so we disable
+  # it when they're not available.
+  when declared(TLockLevel):
+    proc `<`(a, b: TLockLevel): bool {.borrow.}
+    proc `==`(a, b: TLockLevel): bool {.borrow.}
+    if disp.typ.lockLevel == UnspecifiedLockLevel:
+      disp.typ.lockLevel = meth.typ.lockLevel
+    elif meth.typ.lockLevel != UnspecifiedLockLevel and
+         meth.typ.lockLevel != disp.typ.lockLevel:
+      message(conf, meth.info, warnLockLevel,
+        "method has lock level $1, but another method has $2" %
+        [$meth.typ.lockLevel, $disp.typ.lockLevel])
+      # XXX The following code silences a duplicate warning in
+      # checkMethodeffects() in sempass2.nim for now.
+      if disp.typ.lockLevel < meth.typ.lockLevel:
+        disp.typ.lockLevel = meth.typ.lockLevel
+
+proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
   var witness: PSym
   for i in 0..<g.methods.len:
     let disp = g.methods[i].dispatcher
@@ -168,8 +177,10 @@ proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
     of Invalid:
       if witness.isNil: witness = g.methods[i].methods[0]
   # create a new dispatcher:
-  g.methods.add((methods: @[s], dispatcher: createDispatcher(s, g, idgen)))
+  g.methods.add((methods: @[s], dispatcher: createDispatcher(s)))
   #echo "adding ", s.info
+  #if fromCache:
+  #  internalError(s.info, "no method dispatcher found")
   if witness != nil:
     localError(g.config, s.info, "invalid declaration order; cannot attach '" & s.name.s &
                        "' to method defined here: " & g.config$witness.info)

compiler/closureiters.nim

@@ -121,10 +121,7 @@
 #   yield 2
 #   if :unrollFinally: # This node is created by `newEndFinallyNode`
 #     if :curExc.isNil:
-#       if nearestFinally == 0:
-#         return :tmpResult
-#       else:
-#         :state = nearestFinally # bubble up
+#       return :tmpResult
 #     else:
 #       closureIterSetupExc(nil)
 #       raise
@@ -136,9 +133,6 @@ import
   renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos,
   tables, options
 
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
 type
   Ctx = object
     g: ModuleGraph
@@ -159,7 +153,6 @@ type
     curExcHandlingState: int # Negative for except, positive for finally
     nearestFinally: int # Index of the nearest finally block. For try/except it
                     # is their finally. For finally it is parent finally. Otherwise -1
-    idgen: IdGenerator
 
 const
   nkSkip = {nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
@@ -183,7 +176,7 @@ proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
   ctx.newStateAssgn(newIntTypeNode(stateNo, ctx.g.getSysType(TLineInfo(), tyInt)))
 
 proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
-  result = newSym(skVar, getIdent(ctx.g.cache, name), nextSymId(ctx.idgen), ctx.fn, ctx.fn.info)
+  result = newSym(skVar, getIdent(ctx.g.cache, name), ctx.fn, ctx.fn.info)
   result.typ = typ
   assert(not typ.isNil)
 
@@ -196,7 +189,7 @@ proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
   else:
     let envParam = getEnvParam(ctx.fn)
     # let obj = envParam.typ.lastSon
-    result = addUniqueField(envParam.typ.lastSon, result, ctx.g.cache, ctx.idgen)
+    result = addUniqueField(envParam.typ.lastSon, result, ctx.g.cache)
 
 proc newEnvVarAccess(ctx: Ctx, s: PSym): PNode =
   if ctx.stateVarSym.isNil:
@@ -420,11 +413,8 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
 
 
 proc newEnvVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode =
-  if isEmptyType(v.typ):
-    result = v
-  else:
-    result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
-    result.info = v.info
+  result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
+  result.info = v.info
 
 proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) =
   if input.kind == nkStmtListExpr:
@@ -614,12 +604,6 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
             internalError(ctx.g.config, "lowerStmtListExpr(nkCaseStmt): " & $branch.kind)
         result.add(n)
         result.add(ctx.newEnvVarAccess(tmp))
-      elif n[0].kind == nkStmtListExpr:
-        result = newNodeI(nkStmtList, n.info)
-        let (st, ex) = exprToStmtList(n[0])
-        result.add(st)
-        n[0] = ex
-        result.add(n)
 
   of nkCallKinds, nkChckRange, nkChckRangeF, nkChckRange64:
     var ns = false
@@ -718,7 +702,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       n[^1] = ex
       result.add(n)
 
-  of nkAsgn, nkFastAsgn, nkSinkAsgn:
+  of nkAsgn, nkFastAsgn:
     var ns = false
     for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
@@ -810,10 +794,7 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   # Generate the following code:
   #   if :unrollFinally:
   #       if :curExc.isNil:
-  #         if nearestFinally == 0:
-  #           return :tmpResult
-  #         else:
-  #           :state = nearestFinally # bubble up
+  #         return :tmpResult
   #       else:
   #         raise
   let curExc = ctx.newCurExcAccess()
@@ -822,20 +803,11 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   let cmp = newTree(nkCall, newSymNode(ctx.g.getSysMagic(info, "==", mEqRef), info), curExc, nilnode)
   cmp.typ = ctx.g.getSysType(info, tyBool)
 
-  let retStmt =
-    if ctx.nearestFinally == 0:
-      # last finally, we can return
-      let retValue = if ctx.fn.typ[0].isNil:
-                   ctx.g.emptyNode
-                 else:
-                   newTree(nkFastAsgn,
-                           newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen), info),
-                           ctx.newTmpResultAccess())
-      newTree(nkReturnStmt, retValue)
-    else:
-      # bubble up to next finally
-      newTree(nkGotoState, ctx.g.newIntLit(info, ctx.nearestFinally))
+  let asgn = newTree(nkFastAsgn,
+    newSymNode(getClosureIterResult(ctx.g, ctx.fn), info),
+    ctx.newTmpResultAccess())
 
+  let retStmt = newTree(nkReturnStmt, asgn)
   let branch = newTree(nkElifBranch, cmp, retStmt)
 
   let nullifyExc = newTree(nkCall, newSymNode(ctx.g.getCompilerProc("closureIterSetupExc")), nilnode)
@@ -868,7 +840,7 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
     if n[0].kind != nkEmpty:
       let asgnTmpResult = newNodeI(nkAsgn, n.info)
       asgnTmpResult.add(ctx.newTmpResultAccess())
-      let x = if n[0].kind in {nkAsgn, nkFastAsgn, nkSinkAsgn}: n[0][1] else: n[0]
+      let x = if n[0].kind in {nkAsgn, nkFastAsgn}: n[0][1] else: n[0]
       asgnTmpResult.add(x)
       result.add(asgnTmpResult)
 
@@ -879,13 +851,6 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
 
   of nkSkip:
     discard
-  of nkTryStmt:
-    if n.hasYields:
-      # the inner try will handle these transformations
-      discard
-    else:
-      for i in 0..<n.len:
-        n[i] = ctx.transformReturnsInTry(n[i])
   else:
     for i in 0..<n.len:
       n[i] = ctx.transformReturnsInTry(n[i])
@@ -1079,7 +1044,7 @@ proc transformStateAssignments(ctx: var Ctx, n: PNode): PNode =
       if n[0][0].kind != nkEmpty:
         var a = newNodeI(nkAsgn, n[0][0].info)
         var retVal = n[0][0] #liftCapturedVars(n[0], owner, d, c)
-        a.add newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen))
+        a.add newSymNode(getClosureIterResult(ctx.g, ctx.fn))
         a.add retVal
         retStmt.add(a)
       else:
@@ -1151,11 +1116,11 @@ proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
     for i in 0..<n.len:
       n[i] = ctx.skipThroughEmptyStates(n[i])
 
-proc newArrayType(g: ModuleGraph; n: int, t: PType; idgen: IdGenerator; owner: PSym): PType =
-  result = newType(tyArray, nextTypeId(idgen), owner)
+proc newArrayType(g: ModuleGraph; n: int, t: PType, owner: PSym): PType =
+  result = newType(tyArray, owner)
 
-  let rng = newType(tyRange, nextTypeId(idgen), owner)
-  rng.n = newTree(nkRange, g.newIntLit(owner.info, 0), g.newIntLit(owner.info, n - 1))
+  let rng = newType(tyRange, owner)
+  rng.n = newTree(nkRange, g.newIntLit(owner.info, 0), g.newIntLit(owner.info, n))
   rng.rawAddSon(t)
 
   result.rawAddSon(rng)
@@ -1163,7 +1128,7 @@ proc newArrayType(g: ModuleGraph; n: int, t: PType; idgen: IdGenerator; owner: P
 
 proc createExceptionTable(ctx: var Ctx): PNode {.inline.} =
   result = newNodeI(nkBracket, ctx.fn.info)
-  result.typ = ctx.g.newArrayType(ctx.exceptionTable.len, ctx.g.getSysType(ctx.fn.info, tyInt16), ctx.idgen, ctx.fn)
+  result.typ = ctx.g.newArrayType(ctx.exceptionTable.len, ctx.g.getSysType(ctx.fn.info, tyInt16), ctx.fn)
 
   for i in ctx.exceptionTable:
     let elem = newIntNode(nkIntLit, i)
@@ -1324,12 +1289,10 @@ type
     finallys: seq[PNode]
     config: ConfigRef
     blocks: seq[(PNode, int)]
-    idgen: IdGenerator
   FreshVarsContext = object
     tab: Table[int, PSym]
     config: ConfigRef
     info: TLineInfo
-    idgen: IdGenerator
 
 proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
   case n.kind
@@ -1348,7 +1311,7 @@ proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
         let idefs = copyNode(it)
         for v in 0..it.len-3:
           if it[v].kind == nkSym:
-            let x = copySym(it[v].sym, nextSymId(c.idgen))
+            let x = copySym(it[v].sym)
             c.tab[it[v].sym.id] = x
             idefs.add newSymNode(x)
           else:
@@ -1372,24 +1335,20 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode =
   # detect: 'finally: raises X' which is currently not supported. We produce
   # an error for this case for now. All this will be done properly with Yuriy's
   # patch.
-
   result = n
   case n.kind
   of nkTryStmt:
     let f = n.lastSon
-    var didAddSomething = false
     if f.kind == nkFinally:
       c.finallys.add f.lastSon
-      didAddSomething = true
 
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
 
-    if didAddSomething:
+    if f.kind == nkFinally:
       discard c.finallys.pop()
 
   of nkWhileStmt, nkBlockStmt:
-    if n.hasYields == false: return n
     c.blocks.add((n, c.finallys.len))
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
@@ -1412,29 +1371,27 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode =
       if fin >= 0:
         result = newNodeI(nkStmtList, n.info)
         for i in countdown(c.finallys.high, fin):
-          var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info, idgen: c.idgen)
-          result.add freshVars(copyTree(c.finallys[i]), vars)
-          c.idgen = vars.idgen
+          var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info)
+          result.add freshVars(preprocess(c, c.finallys[i]), vars)
         result.add n
   of nkSkip: discard
   else:
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
 
-proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n: PNode): PNode =
+proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   var ctx: Ctx
   ctx.g = g
   ctx.fn = fn
-  ctx.idgen = idgen
 
   if getEnvParam(fn).isNil:
     # Lambda lifting was not done yet. Use temporary :state sym, which will
     # be handled specially by lambda lifting. Local temp vars (if needed)
     # should follow the same logic.
-    ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), nextSymId(idgen), fn, fn.info)
-    ctx.stateVarSym.typ = g.createClosureIterStateType(fn, idgen)
-  ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), nextSymId(idgen), fn, fn.info)
-  var pc = PreprocessContext(finallys: @[], config: g.config, idgen: idgen)
+    ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), fn, fn.info)
+    ctx.stateVarSym.typ = g.createClosureIterStateType(fn)
+  ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), fn, fn.info)
+  var pc = PreprocessContext(finallys: @[], config: g.config)
   var n = preprocess(pc, n.toStmtList)
   #echo "transformed into ", n
   #var n = n.toStmtList

compiler/cmdlinehelper.nim

@@ -13,6 +13,8 @@ import
   options, idents, nimconf, extccomp, commands, msgs,
   lineinfos, modulegraphs, condsyms, os, pathutils, parseopt
 
+from strutils import normalize
+
 proc prependCurDir*(f: AbsoluteFile): AbsoluteFile =
   when defined(unix):
     if os.isAbsolute(f.string): result = f
@@ -39,40 +41,42 @@ proc initDefinesProg*(self: NimProg, conf: ConfigRef, name: string) =
 
 proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   self.processCmdLine(passCmd1, "", conf)
-  if conf.projectIsCmd and conf.projectName in ["-", ""]:
-    handleCmdInput(conf)
-  elif self.supportsStdinFile and conf.projectName == "-":
+  if self.supportsStdinFile and conf.projectName == "-":
     handleStdinInput(conf)
   elif conf.projectName != "":
-    setFromProjectName(conf, conf.projectName)
+    try:
+      conf.projectFull = canonicalizePath(conf, AbsoluteFile conf.projectName)
+    except OSError:
+      conf.projectFull = AbsoluteFile conf.projectName
+    let p = splitFile(conf.projectFull)
+    let dir = if p.dir.isEmpty: AbsoluteDir getCurrentDir() else: p.dir
+    conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile dir)
+    conf.projectName = p.name
   else:
     conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir())
 
-proc loadConfigsAndProcessCmdLine*(self: NimProg, cache: IdentCache; conf: ConfigRef;
+proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef;
                                    graph: ModuleGraph): bool =
   if self.suggestMode:
-    conf.setCmd cmdIdeTools
-  if conf.cmd == cmdNimscript:
-    incl(conf.globalOptions, optWasNimscript)
-  loadConfigs(DefaultConfig, cache, conf, graph.idgen) # load all config files
+    conf.command = "nimsuggest"
+  loadConfigs(DefaultConfig, cache, conf) # load all config files
 
   if not self.suggestMode:
     let scriptFile = conf.projectFull.changeFileExt("nims")
     # 'nim foo.nims' means to just run the NimScript file and do nothing more:
     if fileExists(scriptFile) and scriptFile == conf.projectFull:
-      if conf.cmd == cmdNone: conf.setCmd cmdNimscript
-      if conf.cmd == cmdNimscript: return false
+      if conf.command == "":
+        conf.command = "e"
+        return false
+      elif conf.command.normalize == "e":
+        return false
+
   # now process command line arguments again, because some options in the
   # command line can overwrite the config file's settings
-  if conf.backend != backendJs: # bug #19059
-    extccomp.initVars(conf)
+  extccomp.initVars(conf)
   self.processCmdLine(passCmd2, "", conf)
-  if conf.cmd == cmdNone:
+  if conf.command == "":
     rawMessage(conf, errGenerated, "command missing")
 
   graph.suggestMode = self.suggestMode
   return true
-
-proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef; graph: ModuleGraph): bool =
-  ## Alias for loadConfigsAndProcessCmdLine, here for backwards compatibility
-  loadConfigsAndProcessCmdLine(self, cache, conf, graph)

compiler/commands.nim

@@ -9,8 +9,9 @@
 
 # This module handles the parsing of command line arguments.
 
+
 # We do this here before the 'import' statement so 'defined' does not get
-# confused with 'TGCMode.gcMarkAndSweep' etc.
+# confused with 'TGCMode.gcGenerational' etc.
 template bootSwitch(name, expr, userString) =
   # Helper to build boot constants, for debugging you can 'echo' the else part.
   const name = if expr: " " & userString else: ""
@@ -21,21 +22,23 @@ bootSwitch(usedDanger, defined(danger), "-d:danger")
 bootSwitch(useLinenoise, defined(nimUseLinenoise) or defined(useLinenoise), "-d:nimUseLinenoise")
 bootSwitch(usedBoehm, defined(boehmgc), "--gc:boehm")
 bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
+bootSwitch(usedGenerational, defined(gcgenerational), "--gc:generational")
 bootSwitch(usedGoGC, defined(gogc), "--gc:go")
 bootSwitch(usedNoGC, defined(nogc), "--gc:none")
 
-import std/[setutils, os, strutils, parseutils, parseopt, sequtils, strtabs]
 import
-  msgs, options, nversion, condsyms, extccomp, platform,
-  wordrecg, nimblecmd, lineinfos, pathutils, pathnorm
+  os, msgs, options, nversion, condsyms, strutils, extccomp, platform,
+  wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos,
+  pathutils, strtabs
 
-from ast import setUseIc, eqTypeFlags, tfGcSafe, tfNoSideEffect
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
+from incremental import nimIncremental
+from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect
 
 # but some have deps to imported modules. Yay.
 bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
+bootSwitch(usedNativeStacktrace,
+  defined(nativeStackTrace) and nativeStackTraceSupported,
+  "-d:nativeStackTrace")
 bootSwitch(usedFFI, hasFFI, "-d:nimHasLibFFI")
 
 type
@@ -95,13 +98,12 @@ proc writeVersionInfo(conf: ConfigRef; pass: TCmdLinePass) =
                {msgStdout})
 
     const gitHash {.strdefine.} = gorge("git log -n 1 --format=%H").strip
-      # xxx move this logic to std/private/gitutils
     when gitHash.len == 40:
       msgWriteln(conf, "git hash: " & gitHash, {msgStdout})
 
     msgWriteln(conf, "active boot switches:" & usedRelease & usedDanger &
-      usedTinyC & useLinenoise &
-      usedFFI & usedBoehm & usedMarkAndSweep & usedGoGC & usedNoGC,
+      usedTinyC & useLinenoise & usedNativeStacktrace &
+      usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC,
                {msgStdout})
     msgQuit(0)
 
@@ -116,7 +118,7 @@ const
   errInvalidCmdLineOption = "invalid command line option: '$1'"
   errOnOrOffExpectedButXFound = "'on' or 'off' expected, but '$1' found"
   errOnOffOrListExpectedButXFound = "'on', 'off' or 'list' expected, but '$1' found"
-  errOffHintsError = "'off', 'hint', 'error' or 'usages' expected, but '$1' found"
+  errOffHintsError = "'off', 'hint' or 'error' expected, but '$1' found"
 
 proc invalidCmdLineOption(conf: ConfigRef; pass: TCmdLinePass, switch: string, info: TLineInfo) =
   if switch == " ": localError(conf, info, errInvalidCmdLineOption % "-")
@@ -140,15 +142,6 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
   elif switch[i] == '[': arg = substr(switch, i)
   else: invalidCmdLineOption(conf, pass, switch, info)
 
-template switchOn(arg: string): bool =
-  # xxx use `switchOn` wherever appropriate
-  case arg.normalize
-  of "", "on": true
-  of "off": false
-  else:
-    localError(conf, info, errOnOrOffExpectedButXFound % arg)
-    false
-
 proc processOnOffSwitch(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass,
                         info: TLineInfo) =
   case arg.normalize
@@ -184,7 +177,7 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
                          info: TLineInfo; orig: string; conf: ConfigRef) =
   var id = ""  # arg = key or [key] or key:val or [key]:val;  with val=on|off
   var i = 0
-  var notes: set[TMsgKind]
+  var n = hintMin
   var isBracket = false
   if i < arg.len and arg[i] == '[':
     isBracket = true
@@ -199,38 +192,36 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
   if i == arg.len: discard
   elif i < arg.len and (arg[i] in {':', '='}): inc(i)
   else: invalidCmdLineOption(conf, pass, orig, info)
+  if state == wHint:
+    let x = findStr(hintMin..hintMax, id, errUnknown)
+    if x != errUnknown: n = TNoteKind(x)
+    else: localError(conf, info, "unknown hint: " & id)
+  else:
+    let x = findStr(warnMin..warnMax, id, errUnknown)
+    if x != errUnknown: n = TNoteKind(x)
+    else: localError(conf, info, "unknown warning: " & id)
 
-  let isSomeHint = state in {wHint, wHintAsError}
-  template findNote(noteMin, noteMax, name) =
-    # unfortunately, hintUser and warningUser clash, otherwise implementation would simplify a bit
-    let x = findStr(noteMin, noteMax, id, errUnknown)
-    if x != errUnknown: notes = {TNoteKind(x)}
-    else: localError(conf, info, "unknown $#: $#" % [name, id])
-  case id.normalize
-  of "all": # other note groups would be easy to support via additional cases
-    notes = if isSomeHint: {hintMin..hintMax} else: {warnMin..warnMax}
-  elif isSomeHint: findNote(hintMin, hintMax, "hint")
-  else: findNote(warnMin, warnMax, "warning")
   var val = substr(arg, i).normalize
   if val == "": val = "on"
   if val notin ["on", "off"]:
-    # xxx in future work we should also allow users to have control over `foreignPackageNotes`
-    # so that they can enable `hints|warnings|warningAsErrors` for all the code they depend on.
     localError(conf, info, errOnOrOffExpectedButXFound % arg)
-  else:
-    let isOn = val == "on"
-    if isOn and id.normalize == "all":
-      localError(conf, info, "only 'all:off' is supported")
-    for n in notes:
-      if n notin conf.cmdlineNotes or pass == passCmd1:
-        if pass == passCmd1: incl(conf.cmdlineNotes, n)
-        incl(conf.modifiedyNotes, n)
-        if state in {wWarningAsError, wHintAsError}:
-          conf.warningAsErrors[n] = isOn # xxx rename warningAsErrors to noteAsErrors
-        else:
-          conf.notes[n] = isOn
-          conf.mainPackageNotes[n] = isOn
-        if not isOn: excl(conf.foreignPackageNotes, n)
+  elif n notin conf.cmdlineNotes or pass == passCmd1:
+    if pass == passCmd1: incl(conf.cmdlineNotes, n)
+    incl(conf.modifiedyNotes, n)
+    case val
+    of "on":
+      if state == wWarningAsError:
+        incl(conf.warningAsErrors, n)
+      else:
+        incl(conf.notes, n)
+        incl(conf.mainPackageNotes, n)
+    of "off":
+      if state == wWarningAsError:
+        excl(conf.warningAsErrors, n)
+      else:
+        excl(conf.notes, n)
+        excl(conf.mainPackageNotes, n)
+        excl(conf.foreignPackageNotes, n)
 
 proc processCompile(conf: ConfigRef; filename: string) =
   var found = findFile(conf, filename)
@@ -241,21 +232,17 @@ const
   errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found"
-  errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found"
-
-template warningOptionNoop(switch: string) =
-  warningDeprecated(conf, info, "'$#' is deprecated, now a noop" % switch)
-
-template deprecatedAlias(oldName, newName: string) =
-  warningDeprecated(conf, info, "'$#' is a deprecated alias for '$#'" % [oldName, newName])
+  errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found"
 
 proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo): bool =
   case switch.normalize
-  of "gc", "mm":
+  of "gc":
     case arg.normalize
     of "boehm": result = conf.selectedGC == gcBoehm
     of "refc": result = conf.selectedGC == gcRefc
+    of "v2": result = false
     of "markandsweep": result = conf.selectedGC == gcMarkAndSweep
+    of "generational": result = false
     of "destructors", "arc": result = conf.selectedGC == gcArc
     of "orc": result = conf.selectedGC == gcOrc
     of "hooks": result = conf.selectedGC == gcHooks
@@ -311,13 +298,12 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
     result = conf.options * {optNaNCheck, optInfCheck} == {optNaNCheck, optInfCheck}
   of "infchecks": result = contains(conf.options, optInfCheck)
   of "nanchecks": result = contains(conf.options, optNaNCheck)
+  of "nilchecks": result = false # not a thing
   of "objchecks": result = contains(conf.options, optObjCheck)
   of "fieldchecks": result = contains(conf.options, optFieldCheck)
   of "rangechecks": result = contains(conf.options, optRangeCheck)
   of "boundchecks": result = contains(conf.options, optBoundsCheck)
-  of "refchecks":
-    warningDeprecated(conf, info, "refchecks is deprecated!")
-    result = contains(conf.options, optRefCheck)
+  of "refchecks": result = contains(conf.options, optRefCheck)
   of "overflowchecks": result = contains(conf.options, optOverflowCheck)
   of "staticboundchecks": result = contains(conf.options, optStaticBoundsCheck)
   of "stylechecks": result = contains(conf.options, optStyleCheck)
@@ -326,16 +312,13 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "run", "r": result = contains(conf.globalOptions, optRun)
   of "symbolfiles": result = conf.symbolFiles != disabledSf
   of "genscript": result = contains(conf.globalOptions, optGenScript)
-  of "gencdeps": result = contains(conf.globalOptions, optGenCDeps)
   of "threads": result = contains(conf.globalOptions, optThreads)
+  of "taintmode": result = contains(conf.globalOptions, optTaintMode)
   of "tlsemulation": result = contains(conf.globalOptions, optTlsEmulation)
   of "implicitstatic": result = contains(conf.options, optImplicitStatic)
-  of "patterns", "trmacros":
-    if switch.normalize == "patterns": deprecatedAlias(switch, "trmacros")
-    result = contains(conf.options, optTrMacros)
+  of "patterns", "trmacros": result = contains(conf.options, optTrMacros)
   of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace)
-  of "nilseqs", "nilchecks", "taintmode": warningOptionNoop(switch)
-  of "panics": result = contains(conf.globalOptions, optPanics)
+  of "nilseqs": result = contains(conf.options, optNilSeqs)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc processPath(conf: ConfigRef; path: string, info: TLineInfo,
@@ -369,242 +352,70 @@ proc processCfgPath(conf: ConfigRef; path: string, info: TLineInfo): AbsoluteDir
 const
   errInvalidNumber = "$1 is not a valid number"
 
-proc makeAbsolute(s: string): AbsoluteFile =
-  if isAbsolute(s):
-    AbsoluteFile pathnorm.normalizePath(s)
-  else:
-    AbsoluteFile pathnorm.normalizePath(os.getCurrentDir() / s)
-
-proc setTrackingInfo(conf: ConfigRef; dirty, file, line, column: string,
-                     info: TLineInfo) =
-  ## set tracking info, common code for track, trackDirty, & ideTrack
-  var ln, col: int
-  if parseUtils.parseInt(line, ln) <= 0:
-    localError(conf, info, errInvalidNumber % line)
-  if parseUtils.parseInt(column, col) <= 0:
-    localError(conf, info, errInvalidNumber % column)
-
-  let a = makeAbsolute(file)
-  if dirty == "":
-    conf.m.trackPos = newLineInfo(conf, a, ln, col)
-  else:
-    let dirtyOriginalIdx = fileInfoIdx(conf, a)
-    if dirtyOriginalIdx.int32 >= 0:
-      msgs.setDirtyFile(conf, dirtyOriginalIdx, makeAbsolute(dirty))
-    conf.m.trackPos = newLineInfo(dirtyOriginalIdx, ln, col)
-
 proc trackDirty(conf: ConfigRef; arg: string, info: TLineInfo) =
   var a = arg.split(',')
   if a.len != 4: localError(conf, info,
                             "DIRTY_BUFFER,ORIGINAL_FILE,LINE,COLUMN expected")
-  setTrackingInfo(conf, a[0], a[1], a[2], a[3], info)
+  var line, column: int
+  if parseUtils.parseInt(a[2], line) <= 0:
+    localError(conf, info, errInvalidNumber % a[1])
+  if parseUtils.parseInt(a[3], column) <= 0:
+    localError(conf, info, errInvalidNumber % a[2])
+
+  let dirtyOriginalIdx = fileInfoIdx(conf, AbsoluteFile a[1])
+  if dirtyOriginalIdx.int32 >= 0:
+    msgs.setDirtyFile(conf, dirtyOriginalIdx, AbsoluteFile a[0])
+
+  conf.m.trackPos = newLineInfo(dirtyOriginalIdx, line, column)
 
 proc track(conf: ConfigRef; arg: string, info: TLineInfo) =
   var a = arg.split(',')
   if a.len != 3: localError(conf, info, "FILE,LINE,COLUMN expected")
-  setTrackingInfo(conf, "", a[0], a[1], a[2], info)
-
-proc trackIde(conf: ConfigRef; cmd: IdeCmd, arg: string, info: TLineInfo) =
-  ## set the tracking info related to an ide cmd, supports optional dirty file
-  var a = arg.split(',')
-  case a.len
-  of 4:
-    setTrackingInfo(conf, a[0], a[1], a[2], a[3], info)
-  of 3:
-    setTrackingInfo(conf, "", a[0], a[1], a[2], info)
-  else:
-    localError(conf, info, "[DIRTY_BUFFER,]ORIGINAL_FILE,LINE,COLUMN expected")
-  conf.ideCmd = cmd
+  var line, column: int
+  if parseUtils.parseInt(a[1], line) <= 0:
+    localError(conf, info, errInvalidNumber % a[1])
+  if parseUtils.parseInt(a[2], column) <= 0:
+    localError(conf, info, errInvalidNumber % a[2])
+  conf.m.trackPos = newLineInfo(conf, AbsoluteFile a[0], line, column)
 
 proc dynlibOverride(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
   if pass in {passCmd2, passPP}:
     expectArg(conf, switch, arg, pass, info)
     options.inclDynlibOverride(conf, arg)
 
-template handleStdinOrCmdInput =
+proc handleStdinInput*(conf: ConfigRef) =
+  conf.projectName = "stdinfile"
   conf.projectFull = conf.projectName.AbsoluteFile
   conf.projectPath = AbsoluteDir getCurrentDir()
+  conf.projectIsStdin = true
   if conf.outDir.isEmpty:
     conf.outDir = getNimcacheDir(conf)
 
-proc handleStdinInput*(conf: ConfigRef) =
-  conf.projectName = "stdinfile"
-  conf.projectIsStdin = true
-  handleStdinOrCmdInput()
-
-proc handleCmdInput*(conf: ConfigRef) =
-  conf.projectName = "cmdfile"
-  handleStdinOrCmdInput()
-
-proc parseCommand*(command: string): Command =
-  case command.normalize
-  of "c", "cc", "compile", "compiletoc": cmdCompileToC
-  of "cpp", "compiletocpp": cmdCompileToCpp
-  of "objc", "compiletooc": cmdCompileToOC
-  of "js", "compiletojs": cmdCompileToJS
-  of "r": cmdCrun
-  of "run": cmdTcc
-  of "check": cmdCheck
-  of "e": cmdNimscript
-  of "doc0": cmdDoc0
-  of "doc2", "doc": cmdDoc
-  of "doc2tex": cmdDoc2tex
-  of "rst2html": cmdRst2html
-  of "md2tex": cmdMd2tex
-  of "md2html": cmdMd2html
-  of "rst2tex": cmdRst2tex
-  of "jsondoc0": cmdJsondoc0
-  of "jsondoc2", "jsondoc": cmdJsondoc
-  of "ctags": cmdCtags
-  of "buildindex": cmdBuildindex
-  of "gendepend": cmdGendepend
-  of "dump": cmdDump
-  of "parse": cmdParse
-  of "rod": cmdRod
-  of "secret": cmdInteractive
-  of "nop", "help": cmdNop
-  of "jsonscript": cmdJsonscript
-  else: cmdUnknown
-
-proc setCmd*(conf: ConfigRef, cmd: Command) =
-  ## sets cmd, backend so subsequent flags can query it (e.g. so --gc:arc can be ignored for backendJs)
-  # Note that `--backend` can override the backend, so the logic here must remain reversible.
-  conf.cmd = cmd
-  case cmd
-  of cmdCompileToC, cmdCrun, cmdTcc: conf.backend = backendC
-  of cmdCompileToCpp: conf.backend = backendCpp
-  of cmdCompileToOC: conf.backend = backendObjc
-  of cmdCompileToJS: conf.backend = backendJs
-  else: discard
-
-proc setCommandEarly*(conf: ConfigRef, command: string) =
-  conf.command = command
-  setCmd(conf, command.parseCommand)
-  # command early customizations
-  # must be handled here to honor subsequent `--hint:x:on|off`
-  case conf.cmd
-  of cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex:
-      # xxx see whether to add others: cmdGendepend, etc.
-    conf.foreignPackageNotes = {hintSuccessX}
-  else:
-    conf.foreignPackageNotes = foreignPackageNotesDefault
-
-proc specialDefine(conf: ConfigRef, key: string; pass: TCmdLinePass) =
+proc specialDefine(conf: ConfigRef, key: string) =
   # Keep this syncronized with the default config/nim.cfg!
   if cmpIgnoreStyle(key, "nimQuirky") == 0:
     conf.exc = excQuirky
   elif cmpIgnoreStyle(key, "release") == 0 or cmpIgnoreStyle(key, "danger") == 0:
-    if pass in {passCmd1, passPP}:
-      conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize}
-      conf.globalOptions.excl {optExcessiveStackTrace, optCDebug}
-      conf.options.incl optOptimizeSpeed
+    conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize}
+    conf.globalOptions.excl {optExcessiveStackTrace, optCDebug}
+    conf.options.incl optOptimizeSpeed
   if cmpIgnoreStyle(key, "danger") == 0 or cmpIgnoreStyle(key, "quick") == 0:
-    if pass in {passCmd1, passPP}:
-      conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck,
-        optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir}
-      conf.globalOptions.excl {optCDebug}
-
-proc initOrcDefines*(conf: ConfigRef) =
-  conf.selectedGC = gcOrc
-  defineSymbol(conf.symbols, "gcorc")
-  defineSymbol(conf.symbols, "gcdestructors")
-  incl conf.globalOptions, optSeqDestructors
-  incl conf.globalOptions, optTinyRtti
-  defineSymbol(conf.symbols, "nimSeqsV2")
-  defineSymbol(conf.symbols, "nimV2")
-  if conf.exc == excNone and conf.backend != backendCpp:
-    conf.exc = excGoto
-
-proc registerArcOrc(pass: TCmdLinePass, conf: ConfigRef, isOrc: bool) =
-  if isOrc:
-    conf.selectedGC = gcOrc
-    defineSymbol(conf.symbols, "gcorc")
-  else:
-    conf.selectedGC = gcArc
-    defineSymbol(conf.symbols, "gcarc")
-
-  defineSymbol(conf.symbols, "gcdestructors")
-  incl conf.globalOptions, optSeqDestructors
-  incl conf.globalOptions, optTinyRtti
-  if pass in {passCmd2, passPP}:
-    defineSymbol(conf.symbols, "nimSeqsV2")
-    defineSymbol(conf.symbols, "nimV2")
-  if conf.exc == excNone and conf.backend != backendCpp:
-    conf.exc = excGoto
-
-proc unregisterArcOrc(conf: ConfigRef) =
-  undefSymbol(conf.symbols, "gcdestructors")
-  undefSymbol(conf.symbols, "gcarc")
-  undefSymbol(conf.symbols, "gcorc")
-  undefSymbol(conf.symbols, "nimSeqsV2")
-  undefSymbol(conf.symbols, "nimV2")
-  excl conf.globalOptions, optSeqDestructors
-  excl conf.globalOptions, optTinyRtti
-
-proc processMemoryManagementOption(switch, arg: string, pass: TCmdLinePass,
-                info: TLineInfo; conf: ConfigRef) =
-  if conf.backend == backendJs: return # for: bug #16033
-  expectArg(conf, switch, arg, pass, info)
-  if pass in {passCmd2, passPP}:
-    case arg.normalize
-    of "boehm":
-      unregisterArcOrc(conf)
-      conf.selectedGC = gcBoehm
-      defineSymbol(conf.symbols, "boehmgc")
-      incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS
-    of "refc":
-      unregisterArcOrc(conf)
-      defineSymbol(conf.symbols, "gcrefc")
-      conf.selectedGC = gcRefc
-    of "markandsweep":
-      unregisterArcOrc(conf)
-      conf.selectedGC = gcMarkAndSweep
-      defineSymbol(conf.symbols, "gcmarkandsweep")
-    of "destructors", "arc":
-      registerArcOrc(pass, conf, false)
-    of "orc":
-      registerArcOrc(pass, conf, true)
-    of "hooks":
-      conf.selectedGC = gcHooks
-      defineSymbol(conf.symbols, "gchooks")
-      incl conf.globalOptions, optSeqDestructors
-      processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
-      if pass in {passCmd2, passPP}:
-        defineSymbol(conf.symbols, "nimSeqsV2")
-    of "go":
-      unregisterArcOrc(conf)
-      conf.selectedGC = gcGo
-      defineSymbol(conf.symbols, "gogc")
-    of "none":
-      unregisterArcOrc(conf)
-      conf.selectedGC = gcNone
-      defineSymbol(conf.symbols, "nogc")
-    of "stack", "regions":
-      unregisterArcOrc(conf)
-      conf.selectedGC = gcRegions
-      defineSymbol(conf.symbols, "gcregions")
-    else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
+    conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck,
+      optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir}
+    conf.globalOptions.excl {optCDebug}
 
 proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
                     conf: ConfigRef) =
   var
     key, val: string
   case switch.normalize
-  of "eval":
-    expectArg(conf, switch, arg, pass, info)
-    conf.projectIsCmd = true
-    conf.cmdInput = arg # can be empty (a nim file with empty content is valid too)
-    if conf.cmd == cmdNone:
-      conf.command = "e"
-      conf.setCmd cmdNimscript # better than `cmdCrun` as a default
-      conf.implicitCmd = true
   of "path", "p":
     expectArg(conf, switch, arg, pass, info)
     for path in nimbleSubs(conf, arg):
       addPath(conf, if pass == passPP: processCfgPath(conf, path, info)
                     else: processPath(conf, path, info), info)
   of "nimblepath", "babelpath":
-    if switch.normalize == "babelpath": deprecatedAlias(switch, "nimblepath")
+    # keep the old name for compat
     if pass in {passCmd2, passPP} and optNoNimblePath notin conf.globalOptions:
       expectArg(conf, switch, arg, pass, info)
       var path = processPath(conf, arg, info, notRelativeToProj=true)
@@ -613,7 +424,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         path = nimbleDir / RelativeDir"pkgs"
       nimblePath(conf, path, info)
   of "nonimblepath", "nobabelpath":
-    if switch.normalize == "nobabelpath": deprecatedAlias(switch, "nonimblepath")
     expectNoArg(conf, switch, arg, pass, info)
     disableNimblePath(conf)
   of "clearnimblepath":
@@ -626,10 +436,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     conf.lazyPaths.keepItIf(it != path)
   of "nimcache":
     expectArg(conf, switch, arg, pass, info)
-    var arg = arg
-    # refs bug #18674, otherwise `--os:windows` messes up with `--nimcache` set
-    # in config nims files, e.g. via: `import os; switch("nimcache", "/tmp/somedir")`
-    if conf.target.targetOS == osWindows and DirSep == '/': arg = arg.replace('\\', '/')
     conf.nimcacheDir = processPath(conf, arg, info, notRelativeToProj=true)
   of "out", "o":
     expectArg(conf, switch, arg, pass, info)
@@ -651,18 +457,23 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
     conf.backend = backend
   of "doccmd": conf.docCmd = arg
+  of "mainmodule", "m":
+    discard "allow for backwards compatibility, but don't do anything"
   of "define", "d":
     expectArg(conf, switch, arg, pass, info)
     if {':', '='} in arg:
       splitSwitch(conf, arg, key, val, pass, info)
-      specialDefine(conf, key, pass)
+      specialDefine(conf, key)
       defineSymbol(conf.symbols, key, val)
     else:
-      specialDefine(conf, arg, pass)
+      specialDefine(conf, arg)
       defineSymbol(conf.symbols, arg)
   of "undef", "u":
     expectArg(conf, switch, arg, pass, info)
     undefSymbol(conf.symbols, arg)
+  of "symbol":
+    expectArg(conf, switch, arg, pass, info)
+    # deprecated, do nothing
   of "compile":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}: processCompile(conf, arg)
@@ -685,21 +496,68 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "project":
     processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info)
   of "gc":
-    warningDeprecated(conf, info, "`gc:option` is deprecated; use `mm:option` instead")
-    processMemoryManagementOption(switch, arg, pass, info, conf)
-  of "mm":
-    processMemoryManagementOption(switch, arg, pass, info, conf)
+    if conf.backend == backendJs or conf.command == "js":
+      # for: bug #16033
+      # This might still be imperfect, in rarse corner cases
+      # (where command is reset in nimscript, maybe).
+      return
+    expectArg(conf, switch, arg, pass, info)
+    if pass in {passCmd2, passPP}:
+      case arg.normalize
+      of "boehm":
+        conf.selectedGC = gcBoehm
+        defineSymbol(conf.symbols, "boehmgc")
+        incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS
+      of "refc":
+        conf.selectedGC = gcRefc
+      of "v2":
+        message(conf, info, warnDeprecated, "--gc:v2 is deprecated; using default gc")
+      of "markandsweep":
+        conf.selectedGC = gcMarkAndSweep
+        defineSymbol(conf.symbols, "gcmarkandsweep")
+      of "destructors", "arc":
+        conf.selectedGC = gcArc
+        defineSymbol(conf.symbols, "gcdestructors")
+        defineSymbol(conf.symbols, "gcarc")
+        incl conf.globalOptions, optSeqDestructors
+        incl conf.globalOptions, optTinyRtti
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+          defineSymbol(conf.symbols, "nimV2")
+      of "orc":
+        conf.selectedGC = gcOrc
+        defineSymbol(conf.symbols, "gcdestructors")
+        defineSymbol(conf.symbols, "gcorc")
+        incl conf.globalOptions, optSeqDestructors
+        incl conf.globalOptions, optTinyRtti
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+          defineSymbol(conf.symbols, "nimV2")
+      of "hooks":
+        conf.selectedGC = gcHooks
+        defineSymbol(conf.symbols, "gchooks")
+        incl conf.globalOptions, optSeqDestructors
+        processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+      of "go":
+        conf.selectedGC = gcGo
+        defineSymbol(conf.symbols, "gogc")
+      of "none":
+        conf.selectedGC = gcNone
+        defineSymbol(conf.symbols, "nogc")
+      of "stack", "regions":
+        conf.selectedGC = gcRegions
+        defineSymbol(conf.symbols, "gcregions")
+      else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "warnings", "w":
     if processOnOffSwitchOrList(conf, {optWarns}, arg, pass, info): listWarnings(conf)
   of "warning": processSpecificNote(arg, wWarning, pass, info, switch, conf)
   of "hint": processSpecificNote(arg, wHint, pass, info, switch, conf)
   of "warningaserror": processSpecificNote(arg, wWarningAsError, pass, info, switch, conf)
-  of "hintaserror": processSpecificNote(arg, wHintAsError, pass, info, switch, conf)
   of "hints":
     if processOnOffSwitchOrList(conf, {optHints}, arg, pass, info): listHints(conf)
-  of "threadanalysis":
-    if conf.backend == backendJs: discard
-    else: processOnOffSwitchG(conf, {optThreadAnalysis}, arg, pass, info)
+  of "threadanalysis": processOnOffSwitchG(conf, {optThreadAnalysis}, arg, pass, info)
   of "stacktrace": processOnOffSwitch(conf, {optStackTrace}, arg, pass, info)
   of "stacktracemsgs": processOnOffSwitch(conf, {optStackTraceMsgs}, arg, pass, info)
   of "excessivestacktrace": processOnOffSwitchG(conf, {optExcessiveStackTrace}, arg, pass, info)
@@ -738,32 +596,32 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     else:
       undefSymbol(conf.symbols, "hotcodereloading")
       undefSymbol(conf.symbols, "useNimRtl")
+  of "nilseqs": processOnOffSwitch(conf, {optNilSeqs}, arg, pass, info)
   of "checks", "x": processOnOffSwitch(conf, ChecksOptions, arg, pass, info)
   of "floatchecks":
     processOnOffSwitch(conf, {optNaNCheck, optInfCheck}, arg, pass, info)
   of "infchecks": processOnOffSwitch(conf, {optInfCheck}, arg, pass, info)
   of "nanchecks": processOnOffSwitch(conf, {optNaNCheck}, arg, pass, info)
+  of "nilchecks": discard "no such thing anymore"
   of "objchecks": processOnOffSwitch(conf, {optObjCheck}, arg, pass, info)
   of "fieldchecks": processOnOffSwitch(conf, {optFieldCheck}, arg, pass, info)
   of "rangechecks": processOnOffSwitch(conf, {optRangeCheck}, arg, pass, info)
   of "boundchecks": processOnOffSwitch(conf, {optBoundsCheck}, arg, pass, info)
-  of "refchecks":
-    warningDeprecated(conf, info, "refchecks is deprecated!")
-    processOnOffSwitch(conf, {optRefCheck}, arg, pass, info)
+  of "refchecks": processOnOffSwitch(conf, {optRefCheck}, arg, pass, info)
   of "overflowchecks": processOnOffSwitch(conf, {optOverflowCheck}, arg, pass, info)
   of "staticboundchecks": processOnOffSwitch(conf, {optStaticBoundsCheck}, arg, pass, info)
   of "stylechecks": processOnOffSwitch(conf, {optStyleCheck}, arg, pass, info)
   of "linedir": processOnOffSwitch(conf, {optLineDir}, arg, pass, info)
   of "assertions", "a": processOnOffSwitch(conf, {optAssert}, arg, pass, info)
+  of "deadcodeelim": discard # deprecated, dead code elim always on
   of "threads":
-    if conf.backend == backendJs or conf.cmd == cmdNimscript: discard
-    else: processOnOffSwitchG(conf, {optThreads}, arg, pass, info)
+    processOnOffSwitchG(conf, {optThreads}, arg, pass, info)
     #if optThreads in conf.globalOptions: conf.setNote(warnGcUnsafe)
   of "tlsemulation": processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info)
+  of "taintmode": processOnOffSwitchG(conf, {optTaintMode}, arg, pass, info)
   of "implicitstatic":
     processOnOffSwitch(conf, {optImplicitStatic}, arg, pass, info)
   of "patterns", "trmacros":
-    if switch.normalize == "patterns": deprecatedAlias(switch, "trmacros")
     processOnOffSwitch(conf, {optTrMacros}, arg, pass, info)
   of "opt":
     expectArg(conf, switch, arg, pass, info)
@@ -815,7 +673,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "clib":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
-      conf.cLinkedLibs.add arg
+      conf.cLinkedLibs.add processPath(conf, arg, info).string
   of "header":
     if conf != nil: conf.headerFile = arg
     incl(conf.globalOptions, optGenIndex)
@@ -890,19 +748,19 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "help", "h":
     expectNoArg(conf, switch, arg, pass, info)
     helpOnError(conf, pass)
-  of "symbolfiles", "incremental", "ic":
-    if switch.normalize == "symbolfiles": deprecatedAlias(switch, "incremental")
-      # xxx maybe also ic, since not in help?
-    if pass in {passCmd2, passPP}:
-      case arg.normalize
-      of "on": conf.symbolFiles = v2Sf
-      of "off": conf.symbolFiles = disabledSf
-      of "writeonly": conf.symbolFiles = writeOnlySf
-      of "readonly": conf.symbolFiles = readOnlySf
-      of "v2": conf.symbolFiles = v2Sf
-      of "stress": conf.symbolFiles = stressTest
-      else: localError(conf, info, "invalid option for --incremental: " & arg)
-    setUseIc(conf.symbolFiles != disabledSf)
+  of "symbolfiles": discard "ignore for backwards compat"
+  of "incremental":
+    when not nimIncremental:
+      localError(conf, info, "the compiler was not built with " &
+        "incremental compilation features; bootstrap with " &
+        "-d:nimIncremental to enable")
+    case arg.normalize
+    of "on": conf.symbolFiles = v2Sf
+    of "off": conf.symbolFiles = disabledSf
+    of "writeonly": conf.symbolFiles = writeOnlySf
+    of "readonly": conf.symbolFiles = readOnlySf
+    of "v2": conf.symbolFiles = v2Sf
+    else: localError(conf, info, "invalid option for --incremental: " & arg)
   of "skipcfg":
     processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info)
   of "skipprojcfg":
@@ -912,11 +770,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "skipparentcfg":
     processOnOffSwitchG(conf, {optSkipParentConfigFiles}, arg, pass, info)
   of "genscript", "gendeps":
-    if switch.normalize == "gendeps": deprecatedAlias(switch, "genscript")
     processOnOffSwitchG(conf, {optGenScript}, arg, pass, info)
     processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info)
-  of "gencdeps":
-    processOnOffSwitchG(conf, {optGenCDeps}, arg, pass, info)
   of "colors": processOnOffSwitchG(conf, {optUseColors}, arg, pass, info)
   of "lib":
     expectArg(conf, switch, arg, pass, info)
@@ -926,9 +781,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     splitSwitch(conf, arg, key, val, pass, info)
     os.putEnv(key, val)
   of "cc":
-    if conf.backend != backendJs: # bug #19330
-      expectArg(conf, switch, arg, pass, info)
-      setCC(conf, arg, info)
+    expectArg(conf, switch, arg, pass, info)
+    setCC(conf, arg, info)
   of "track":
     expectArg(conf, switch, arg, pass, info)
     track(conf, arg, info)
@@ -939,50 +793,28 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectNoArg(conf, switch, arg, pass, info)
     conf.ideCmd = ideSug
   of "def":
+    expectNoArg(conf, switch, arg, pass, info)
+    conf.ideCmd = ideDef
+  of "eval":
     expectArg(conf, switch, arg, pass, info)
-    trackIde(conf, ideDef, arg, info)
+    conf.evalExpr = arg
   of "context":
     expectNoArg(conf, switch, arg, pass, info)
     conf.ideCmd = ideCon
   of "usages":
-    expectArg(conf, switch, arg, pass, info)
-    trackIde(conf, ideUse, arg, info)
-  of "defusages":
-    expectArg(conf, switch, arg, pass, info)
-    trackIde(conf, ideDus, arg, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    conf.ideCmd = ideUse
   of "stdout":
     processOnOffSwitchG(conf, {optStdout}, arg, pass, info)
-  of "filenames":
-    case arg.normalize
-    of "abs": conf.filenameOption = foAbs
-    of "canonical": conf.filenameOption = foCanonical
-    of "legacyrelproj": conf.filenameOption = foLegacyRelProj
-    else: localError(conf, info, "expected: abs|canonical|legacyRelProj, got: $1" % arg)
-  of "processing":
-    incl(conf.notes, hintProcessing)
-    incl(conf.mainPackageNotes, hintProcessing)
-    case arg.normalize
-    of "dots": conf.hintProcessingDots = true
-    of "filenames": conf.hintProcessingDots = false
-    of "off":
-      excl(conf.notes, hintProcessing)
-      excl(conf.mainPackageNotes, hintProcessing)
-    else: localError(conf, info, "expected: dots|filenames|off, got: $1" % arg)
-  of "unitsep":
-    conf.unitSep = if switchOn(arg): "\31" else: ""
   of "listfullpaths":
-    # xxx in future work, use `warningDeprecated`
-    conf.filenameOption = if switchOn(arg): foAbs else: foCanonical
-  of "spellsuggest":
-    if arg.len == 0: conf.spellSuggestMax = spellSuggestSecretSauce
-    elif arg == "auto": conf.spellSuggestMax = spellSuggestSecretSauce
-    else: conf.spellSuggestMax = parseInt(arg)
-  of "declaredlocs":
-    processOnOffSwitchG(conf, {optDeclaredLocs}, arg, pass, info)
+    processOnOffSwitchG(conf, {optListFullPaths}, arg, pass, info)
   of "dynliboverride":
     dynlibOverride(conf, switch, arg, pass, info)
   of "dynliboverrideall":
     processOnOffSwitchG(conf, {optDynlibOverrideAll}, arg, pass, info)
+  of "cs":
+    # only supported for compatibility. Does nothing.
+    expectArg(conf, switch, arg, pass, info)
   of "experimental":
     if arg.len == 0:
       conf.features.incl oldExperimentalFeatures
@@ -1012,7 +844,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if conf != nil:
       conf.cppDefine(arg)
   of "newruntime":
-    warningDeprecated(conf, info, "newruntime is deprecated, use arc/orc instead!")
     expectNoArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
       doAssert(conf != nil)
@@ -1060,6 +891,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "1.0":
       defineSymbol(conf.symbols, "NimMajor", "1")
       defineSymbol(conf.symbols, "NimMinor", "0")
+      # always be compatible with 1.0.100:
+      defineSymbol(conf.symbols, "NimPatch", "100")
       # old behaviors go here:
       defineSymbol(conf.symbols, "nimOldRelativePathBehavior")
       undefSymbol(conf.symbols, "nimDoesntTrackDefects")
@@ -1068,15 +901,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "1.2":
       defineSymbol(conf.symbols, "NimMajor", "1")
       defineSymbol(conf.symbols, "NimMinor", "2")
+      # always be compatible with 1.2.100:
+      defineSymbol(conf.symbols, "NimPatch", "100")
       conf.globalOptions.incl optNimV12Emulation
-    of "1.6":
-      defineSymbol(conf.symbols, "NimMajor", "1")
-      defineSymbol(conf.symbols, "NimMinor", "6")
-      conf.globalOptions.incl optNimV16Emulation
     else:
-      localError(conf, info, "unknown Nim version; currently supported values are: `1.0`, `1.2`")
-    # always be compatible with 1.x.100:
-    defineSymbol(conf.symbols, "NimPatch", "100")
+      localError(conf, info, "unknown Nim version; currently supported values are: {1.0}")
   of "benchmarkvm":
     processOnOffSwitchG(conf, {optBenchmarkVM}, arg, pass, info)
   of "profilevm":
@@ -1090,15 +919,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitchG(conf, {optPanics}, arg, pass, info)
     if optPanics in conf.globalOptions:
       defineSymbol(conf.symbols, "nimPanics")
-  of "sourcemap": # xxx document in --fullhelp
+  of "sourcemap":
     conf.globalOptions.incl optSourcemap
     conf.options.incl optLineDir
   of "deepcopy":
     processOnOffSwitchG(conf, {optEnableDeepCopy}, arg, pass, info)
   of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -)
     handleStdinInput(conf)
-  of "nilseqs", "nilchecks", "symbol", "taintmode", "cs", "deadcodeelim": warningOptionNoop(switch)
-  of "nimmainprefix": conf.nimMainPrefix = arg
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)
@@ -1108,6 +935,7 @@ proc processCommand*(switch: string, pass: TCmdLinePass; config: ConfigRef) =
   splitSwitch(config, switch, cmd, arg, pass, gCmdLineInfo)
   processSwitch(cmd, arg, pass, gCmdLineInfo, config)
 
+
 proc processSwitch*(pass: TCmdLinePass; p: OptParser; config: ConfigRef) =
   # hint[X]:off is parsed as (p.key = "hint[X]", p.val = "off")
   # we transform it to (key = hint, val = [X]:off)
@@ -1121,22 +949,19 @@ proc processSwitch*(pass: TCmdLinePass; p: OptParser; config: ConfigRef) =
 
 proc processArgument*(pass: TCmdLinePass; p: OptParser;
                       argsCount: var int; config: ConfigRef): bool =
-  if argsCount == 0 and config.implicitCmd:
-    argsCount.inc
   if argsCount == 0:
     # nim filename.nims  is the same as "nim e filename.nims":
     if p.key.endsWith(".nims"):
-      config.setCmd cmdNimscript
+      config.command = "e"
       incl(config.globalOptions, optWasNimscript)
       config.projectName = unixToNativePath(p.key)
       config.arguments = cmdLineRest(p)
       result = true
-    elif pass != passCmd2: setCommandEarly(config, p.key)
+    elif pass != passCmd2:
+      config.command = p.key
   else:
     if pass == passCmd1: config.commandArgs.add p.key
     if argsCount == 1:
-      if p.key.endsWith(".nims"):
-        incl(config.globalOptions, optWasNimscript)
       # support UNIX style filenames everywhere for portable build scripts:
       if config.projectName.len == 0:
         config.projectName = unixToNativePath(p.key)

compiler/concepts.nim

@@ -1,344 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2020 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## New styled concepts for Nim. See https://github.com/nim-lang/RFCs/issues/168
-## for details. Note this is a first implementation and only the "Concept matching"
-## section has been implemented.
-
-import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types, intsets
-
-from magicsys import addSonSkipIntLit
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
-const
-  logBindings = false
-
-## Code dealing with Concept declarations
-## --------------------------------------
-
-proc declareSelf(c: PContext; info: TLineInfo) =
-  ## Adds the magical 'Self' symbols to the current scope.
-  let ow = getCurrOwner(c)
-  let s = newSym(skType, getIdent(c.cache, "Self"), nextSymId(c.idgen), ow, info)
-  s.typ = newType(tyTypeDesc, nextTypeId(c.idgen), ow)
-  s.typ.flags.incl {tfUnresolved, tfPacked}
-  s.typ.add newType(tyEmpty, nextTypeId(c.idgen), ow)
-  addDecl(c, s, info)
-
-proc isSelf*(t: PType): bool {.inline.} =
-  ## Is this the magical 'Self' type?
-  t.kind == tyTypeDesc and tfPacked in t.flags
-
-proc makeTypeDesc*(c: PContext, typ: PType): PType =
-  if typ.kind == tyTypeDesc and not isSelf(typ):
-    result = typ
-  else:
-    result = newTypeS(tyTypeDesc, c)
-    incl result.flags, tfCheckedForDestructor
-    result.addSonSkipIntLit(typ, c.idgen)
-
-proc semConceptDecl(c: PContext; n: PNode): PNode =
-  ## Recursive helper for semantic checking for the concept declaration.
-  ## Currently we only support (possibly empty) lists of statements
-  ## containing 'proc' declarations and the like.
-  case n.kind
-  of nkStmtList, nkStmtListExpr:
-    result = shallowCopy(n)
-    for i in 0..<n.len:
-      result[i] = semConceptDecl(c, n[i])
-  of nkProcDef..nkIteratorDef, nkFuncDef:
-    result = c.semExpr(c, n, {efWantStmt})
-  of nkTypeClassTy:
-    result = shallowCopy(n)
-    for i in 0..<n.len-1:
-      result[i] = n[i]
-    result[^1] = semConceptDecl(c, n[^1])
-  of nkCommentStmt:
-    discard
-  else:
-    localError(c.config, n.info, "unexpected construct in the new-styled concept: " & renderTree(n))
-    result = n
-
-proc semConceptDeclaration*(c: PContext; n: PNode): PNode =
-  ## Semantic checking for the concept declaration. Runs
-  ## when we process the concept itself, not its matching process.
-  assert n.kind == nkTypeClassTy
-  inc c.inConceptDecl
-  openScope(c)
-  declareSelf(c, n.info)
-  result = semConceptDecl(c, n)
-  rawCloseScope(c)
-  dec c.inConceptDecl
-
-## Concept matching
-## ----------------
-
-type
-  MatchCon = object ## Context we pass around during concept matching.
-    inferred: seq[(PType, PType)] ## we need a seq here so that we can easily undo inferences \
-      ## that turned out to be wrong.
-    marker: IntSet ## Some protection against wild runaway recursions.
-    potentialImplementation: PType ## the concrete type that might match the concept we try to match.
-    magic: TMagic  ## mArrGet and mArrPut is wrong in system.nim and
-                   ## cannot be fixed that easily.
-                   ## Thus we special case it here.
-
-proc existingBinding(m: MatchCon; key: PType): PType =
-  ## checks if we bound the type variable 'key' already to some
-  ## concrete type.
-  for i in 0..<m.inferred.len:
-    if m.inferred[i][0] == key: return m.inferred[i][1]
-  return nil
-
-proc conceptMatchNode(c: PContext; n: PNode; m: var MatchCon): bool
-
-proc matchType(c: PContext; f, a: PType; m: var MatchCon): bool =
-  ## The heart of the concept matching process. 'f' is the formal parameter of some
-  ## routine inside the concept that we're looking for. 'a' is the formal parameter
-  ## of a routine that might match.
-  const
-    ignorableForArgType = {tyVar, tySink, tyLent, tyOwned, tyGenericInst, tyAlias, tyInferred}
-  case f.kind
-  of tyAlias:
-    result = matchType(c, f.lastSon, a, m)
-  of tyTypeDesc:
-    if isSelf(f):
-      #let oldLen = m.inferred.len
-      result = matchType(c, a, m.potentialImplementation, m)
-      #echo "self is? ", result, " ", a.kind, " ", a, " ", m.potentialImplementation, " ", m.potentialImplementation.kind
-      #m.inferred.setLen oldLen
-      #echo "A for ", result, " to ", typeToString(a), " to ", typeToString(m.potentialImplementation)
-    else:
-      if a.kind == tyTypeDesc and f.len == a.len:
-        for i in 0..<a.len:
-          if not matchType(c, f[i], a[i], m): return false
-        return true
-
-  of tyGenericInvocation:
-    if a.kind == tyGenericInst and a[0].kind == tyGenericBody:
-      if sameType(f[0], a[0]) and f.len == a.len-1:
-        for i in 1 ..< f.len:
-          if not matchType(c, f[i], a[i], m): return false
-        return true
-  of tyGenericParam:
-    let ak = a.skipTypes({tyVar, tySink, tyLent, tyOwned})
-    if ak.kind in {tyTypeDesc, tyStatic} and not isSelf(ak):
-      result = false
-    else:
-      let old = existingBinding(m, f)
-      if old == nil:
-        if f.len > 0 and f[0].kind != tyNone:
-          # also check the generic's constraints:
-          let oldLen = m.inferred.len
-          result = matchType(c, f[0], a, m)
-          m.inferred.setLen oldLen
-          if result:
-            when logBindings: echo "A adding ", f, " ", ak
-            m.inferred.add((f, ak))
-        elif m.magic == mArrGet and ak.kind in {tyArray, tyOpenArray, tySequence, tyVarargs, tyCstring, tyString}:
-          when logBindings: echo "B adding ", f, " ", lastSon ak
-          m.inferred.add((f, lastSon ak))
-          result = true
-        else:
-          when logBindings: echo "C adding ", f, " ", ak
-          m.inferred.add((f, ak))
-          #echo "binding ", typeToString(ak), " to ", typeToString(f)
-          result = true
-      elif not m.marker.containsOrIncl(old.id):
-        result = matchType(c, old, ak, m)
-        if m.magic == mArrPut and ak.kind == tyGenericParam:
-          result = true
-    #echo "B for ", result, " to ", typeToString(a), " to ", typeToString(m.potentialImplementation)
-
-  of tyVar, tySink, tyLent, tyOwned:
-    # modifiers in the concept must be there in the actual implementation
-    # too but not vice versa.
-    if a.kind == f.kind:
-      result = matchType(c, f.sons[0], a.sons[0], m)
-    elif m.magic == mArrPut:
-      result = matchType(c, f.sons[0], a, m)
-    else:
-      result = false
-  of tyEnum, tyObject, tyDistinct:
-    result = sameType(f, a)
-  of tyEmpty, tyString, tyCstring, tyPointer, tyNil, tyUntyped, tyTyped, tyVoid:
-    result = a.skipTypes(ignorableForArgType).kind == f.kind
-  of tyBool, tyChar, tyInt..tyUInt64:
-    let ak = a.skipTypes(ignorableForArgType)
-    result = ak.kind == f.kind or ak.kind == tyOrdinal or
-       (ak.kind == tyGenericParam and ak.len > 0 and ak[0].kind == tyOrdinal)
-  of tyConcept:
-    let oldLen = m.inferred.len
-    let oldPotentialImplementation = m.potentialImplementation
-    m.potentialImplementation = a
-    result = conceptMatchNode(c, f.n.lastSon, m)
-    m.potentialImplementation = oldPotentialImplementation
-    if not result:
-      m.inferred.setLen oldLen
-  of tyArray, tyTuple, tyVarargs, tyOpenArray, tyRange, tySequence, tyRef, tyPtr,
-     tyGenericInst:
-    let ak = a.skipTypes(ignorableForArgType - {f.kind})
-    if ak.kind == f.kind and f.len == ak.len:
-      for i in 0..<ak.len:
-        if not matchType(c, f[i], ak[i], m): return false
-      return true
-  of tyOr:
-    let oldLen = m.inferred.len
-    if a.kind == tyOr:
-      # say the concept requires 'int|float|string' if the potentialImplementation
-      # says 'int|string' that is good enough.
-      var covered = 0
-      for i in 0..<f.len:
-        for j in 0..<a.len:
-          let oldLenB = m.inferred.len
-          let r = matchType(c, f[i], a[j], m)
-          if r:
-            inc covered
-            break
-          m.inferred.setLen oldLenB
-
-      result = covered >= a.len
-      if not result:
-        m.inferred.setLen oldLen
-    else:
-      for i in 0..<f.len:
-        result = matchType(c, f[i], a, m)
-        if result: break # and remember the binding!
-        m.inferred.setLen oldLen
-  of tyNot:
-    if a.kind == tyNot:
-      result = matchType(c, f[0], a[0], m)
-    else:
-      let oldLen = m.inferred.len
-      result = not matchType(c, f[0], a, m)
-      m.inferred.setLen oldLen
-  of tyAnything:
-    result = true
-  of tyOrdinal:
-    result = isOrdinalType(a, allowEnumWithHoles = false) or a.kind == tyGenericParam
-  else:
-    result = false
-
-proc matchReturnType(c: PContext; f, a: PType; m: var MatchCon): bool =
-  ## Like 'matchType' but with extra logic dealing with proc return types
-  ## which can be nil or the 'void' type.
-  if f.isEmptyType:
-    result = a.isEmptyType
-  elif a == nil:
-    result = false
-  else:
-    result = matchType(c, f, a, m)
-
-proc matchSym(c: PContext; candidate: PSym, n: PNode; m: var MatchCon): bool =
-  ## Checks if 'candidate' matches 'n' from the concept body. 'n' is a nkProcDef
-  ## or similar.
-
-  # watch out: only add bindings after a completely successful match.
-  let oldLen = m.inferred.len
-
-  let can = candidate.typ.n
-  let con = n[0].sym.typ.n
-
-  if can.len < con.len:
-    # too few arguments, cannot be a match:
-    return false
-
-  let common = min(can.len, con.len)
-  for i in 1 ..< common:
-    if not matchType(c, con[i].typ, can[i].typ, m):
-      m.inferred.setLen oldLen
-      return false
-
-  if not matchReturnType(c, n[0].sym.typ.sons[0], candidate.typ.sons[0], m):
-    m.inferred.setLen oldLen
-    return false
-
-  # all other parameters have to be optional parameters:
-  for i in common ..< can.len:
-    assert can[i].kind == nkSym
-    if can[i].sym.ast == nil:
-      # has too many arguments one of which is not optional:
-      m.inferred.setLen oldLen
-      return false
-
-  return true
-
-proc matchSyms(c: PContext, n: PNode; kinds: set[TSymKind]; m: var MatchCon): bool =
-  ## Walk the current scope, extract candidates which the same name as 'n[namePos]',
-  ## 'n' is the nkProcDef or similar from the concept that we try to match.
-  let candidates = searchInScopesFilterBy(c, n[namePos].sym.name, kinds)
-  for candidate in candidates:
-    #echo "considering ", typeToString(candidate.typ), " ", candidate.magic
-    m.magic = candidate.magic
-    if matchSym(c, candidate, n, m): return true
-  result = false
-
-proc conceptMatchNode(c: PContext; n: PNode; m: var MatchCon): bool =
-  ## Traverse the concept's AST ('n') and see if every declaration inside 'n'
-  ## can be matched with the current scope.
-  case n.kind
-  of nkStmtList, nkStmtListExpr:
-    for i in 0..<n.len:
-      if not conceptMatchNode(c, n[i], m):
-        return false
-    return true
-  of nkProcDef, nkFuncDef:
-    # procs match any of: proc, template, macro, func, method, converter.
-    # The others are more specific.
-    # XXX: Enforce .noSideEffect for 'nkFuncDef'? But then what are the use cases...
-    const filter = {skProc, skTemplate, skMacro, skFunc, skMethod, skConverter}
-    result = matchSyms(c, n, filter, m)
-  of nkTemplateDef:
-    result = matchSyms(c, n, {skTemplate}, m)
-  of nkMacroDef:
-    result = matchSyms(c, n, {skMacro}, m)
-  of nkConverterDef:
-    result = matchSyms(c, n, {skConverter}, m)
-  of nkMethodDef:
-    result = matchSyms(c, n, {skMethod}, m)
-  of nkIteratorDef:
-    result = matchSyms(c, n, {skIterator}, m)
-  else:
-    # error was reported earlier.
-    result = false
-
-proc conceptMatch*(c: PContext; concpt, arg: PType; bindings: var TIdTable; invocation: PType): bool =
-  ## Entry point from sigmatch. 'concpt' is the concept we try to match (here still a PType but
-  ## we extract its AST via 'concpt.n.lastSon'). 'arg' is the type that might fullfill the
-  ## concept's requirements. If so, we return true and fill the 'bindings' with pairs of
-  ## (typeVar, instance) pairs. ('typeVar' is usually simply written as a generic 'T'.)
-  ## 'invocation' can be nil for atomic concepts. For non-atomic concepts, it contains the
-  ## `C[S, T]` parent type that we look for. We need this because we need to store bindings
-  ## for 'S' and 'T' inside 'bindings' on a successful match. It is very important that
-  ## we do not add any bindings at all on an unsuccessful match!
-  var m = MatchCon(inferred: @[], potentialImplementation: arg)
-  result = conceptMatchNode(c, concpt.n.lastSon, m)
-  if result:
-    for (a, b) in m.inferred:
-      if b.kind == tyGenericParam:
-        var dest = b
-        while true:
-          dest = existingBinding(m, dest)
-          if dest == nil or dest.kind != tyGenericParam: break
-        if dest != nil:
-          bindings.idTablePut(a, dest)
-          when logBindings: echo "A bind ", a, " ", dest
-      else:
-        bindings.idTablePut(a, b)
-        when logBindings: echo "B bind ", a, " ", b
-    # we have a match, so bind 'arg' itself to 'concpt':
-    bindings.idTablePut(concpt, arg)
-    # invocation != nil means we have a non-atomic concept:
-    if invocation != nil and arg.kind == tyGenericInst and invocation.len == arg.len-1:
-      # bind even more generic parameters
-      assert invocation.kind == tyGenericInvocation
-      for i in 1 ..< invocation.len:
-        bindings.idTablePut(invocation[i], arg[i])

compiler/condsyms.nim

@@ -34,58 +34,56 @@ proc countDefinedSymbols*(symbols: StringTableRef): int =
 proc initDefines*(symbols: StringTableRef) =
   # for bootstrapping purposes and old code:
   template defineSymbol(s) = symbols.defineSymbol(s)
-  defineSymbol("nimhygiene") # deadcode
-  defineSymbol("niminheritable") # deadcode
-  defineSymbol("nimmixin") # deadcode
-  defineSymbol("nimeffects") # deadcode
-  defineSymbol("nimbabel") # deadcode
-  defineSymbol("nimcomputedgoto") # deadcode
-  defineSymbol("nimunion") # deadcode
-  defineSymbol("nimnewshared") # deadcode
-  defineSymbol("nimNewTypedesc") # deadcode
-  defineSymbol("nimrequiresnimframe") # deadcode
-  defineSymbol("nimparsebiggestfloatmagic") # deadcode
-  defineSymbol("nimalias") # deadcode
-  defineSymbol("nimlocks") # deadcode
-  defineSymbol("nimnode") # deadcode pending `nimnode` reference in opengl package
-    # refs https://github.com/nim-lang/opengl/pull/79
-  defineSymbol("nimvarargstyped") # deadcode
-  defineSymbol("nimtypedescfixed") # deadcode
-  defineSymbol("nimKnowsNimvm") # deadcode
-  defineSymbol("nimArrIdx") # deadcode
-  defineSymbol("nimHasalignOf") # deadcode
-  defineSymbol("nimDistros") # deadcode
-  defineSymbol("nimHasCppDefine") # deadcode
-  defineSymbol("nimGenericInOutFlags") # deadcode
-  when false: defineSymbol("nimHasOpt") # deadcode
-  defineSymbol("nimNoArrayToCstringConversion") # deadcode
-  defineSymbol("nimHasRunnableExamples") # deadcode
-  defineSymbol("nimNewDot") # deadcode
-  defineSymbol("nimHasNilChecks") # deadcode
-  defineSymbol("nimSymKind") # deadcode
-  defineSymbol("nimVmEqIdent") # deadcode
-  defineSymbol("nimNoNil") # deadcode
-  defineSymbol("nimNoZeroTerminator") # deadcode
-  defineSymbol("nimNotNil") # deadcode
-  defineSymbol("nimVmExportFixed") # deadcode
-  defineSymbol("nimHasSymOwnerInMacro") # deadcode
-  defineSymbol("nimNewRuntime") # deadcode
-  defineSymbol("nimIncrSeqV3") # deadcode
-  defineSymbol("nimAshr") # deadcode
-  defineSymbol("nimNoNilSeqs") # deadcode
-  defineSymbol("nimNoNilSeqs2") # deadcode
-  defineSymbol("nimHasUserErrors") # deadcode
-  defineSymbol("nimUncheckedArrayTyp") # deadcode
-  defineSymbol("nimHasTypeof") # deadcode
-  defineSymbol("nimErrorProcCanHaveBody") # deadcode
-  defineSymbol("nimHasInstantiationOfInMacro") # deadcode
-  defineSymbol("nimHasHotCodeReloading") # deadcode
-  defineSymbol("nimHasNilSeqs") # deadcode
-  defineSymbol("nimHasSignatureHashInMacro") # deadcode
-  defineSymbol("nimHasDefault") # deadcode
-  defineSymbol("nimMacrosSizealignof") # deadcode
-
-  # > 0.20.0
+  defineSymbol("nimhygiene")
+  defineSymbol("niminheritable")
+  defineSymbol("nimmixin")
+  defineSymbol("nimeffects")
+  defineSymbol("nimbabel")
+  defineSymbol("nimcomputedgoto")
+  defineSymbol("nimunion")
+  defineSymbol("nimnewshared")
+  defineSymbol("nimNewTypedesc")
+  defineSymbol("nimrequiresnimframe")
+  defineSymbol("nimparsebiggestfloatmagic")
+  defineSymbol("nimalias")
+  defineSymbol("nimlocks")
+  defineSymbol("nimnode")
+  defineSymbol("nimvarargstyped")
+  defineSymbol("nimtypedescfixed")
+  defineSymbol("nimKnowsNimvm")
+  defineSymbol("nimArrIdx")
+  defineSymbol("nimHasalignOf")
+  defineSymbol("nimDistros")
+  defineSymbol("nimHasCppDefine")
+  defineSymbol("nimGenericInOutFlags")
+  when false: defineSymbol("nimHasOpt")
+  defineSymbol("nimNoArrayToCstringConversion")
+  defineSymbol("nimNewRoof")
+  defineSymbol("nimHasRunnableExamples")
+  defineSymbol("nimNewDot")
+  defineSymbol("nimHasNilChecks")
+  defineSymbol("nimSymKind")
+  defineSymbol("nimVmEqIdent")
+  defineSymbol("nimNoNil")
+  defineSymbol("nimNoZeroTerminator")
+  defineSymbol("nimNotNil")
+  defineSymbol("nimVmExportFixed")
+  defineSymbol("nimHasSymOwnerInMacro")
+  defineSymbol("nimNewRuntime")
+  defineSymbol("nimIncrSeqV3")
+  defineSymbol("nimAshr")
+  defineSymbol("nimNoNilSeqs")
+  defineSymbol("nimNoNilSeqs2")
+  defineSymbol("nimHasUserErrors")
+  defineSymbol("nimUncheckedArrayTyp")
+  defineSymbol("nimHasTypeof")
+  defineSymbol("nimErrorProcCanHaveBody")
+  defineSymbol("nimHasInstantiationOfInMacro")
+  defineSymbol("nimHasHotCodeReloading")
+  defineSymbol("nimHasNilSeqs")
+  defineSymbol("nimHasSignatureHashInMacro")
+  defineSymbol("nimHasDefault")
+  defineSymbol("nimMacrosSizealignof")
   defineSymbol("nimNoZeroExtendMagic")
   defineSymbol("nimMacrosGetNodeId")
   for f in Feature:
@@ -125,27 +123,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasStacktracesModule")
   defineSymbol("nimHasEffectTraitsModule")
   defineSymbol("nimHasCastPragmaBlocks")
-  defineSymbol("nimHasDeclaredLocs")
   defineSymbol("nimHasJsBigIntBackend")
-  defineSymbol("nimHasWarningAsError")
-  defineSymbol("nimHasHintAsError")
-  defineSymbol("nimHasSpellSuggest")
-  defineSymbol("nimHasCustomLiterals")
-  defineSymbol("nimHasUnifiedTuple")
-  defineSymbol("nimHasIterable")
-  defineSymbol("nimHasTypeofVoid")
-  defineSymbol("nimHasDragonBox")
-  defineSymbol("nimHasHintAll")
-  defineSymbol("nimHasTrace")
-  defineSymbol("nimHasEffectsOf")
-
-  defineSymbol("nimHasEnforceNoRaises")
-  defineSymbol("nimHasTopDownInference")
-  defineSymbol("nimHasTemplateRedefinitionPragma")
-  defineSymbol("nimHasCstringCase")
-  defineSymbol("nimHasCallsitePragma")
-  defineSymbol("nimHasAmbiguousEnumHint")
-
-  defineSymbol("nimHasOutParams")
-  defineSymbol("nimHasSystemRaisesDefect")
-  defineSymbol("nimHasWarnUnnamedBreak")

compiler/debugutils.nim

@@ -1,72 +0,0 @@
-##[
-Utilities to help with debugging nim compiler.
-
-Experimental API, subject to change.
-]##
-
-#[
-## example
-useful debugging flags:
---stacktrace -d:debug -d:nimDebugUtils
- nim c -o:bin/nim_temp --stacktrace -d:debug -d:nimDebugUtils compiler/nim
-
-## future work
-* expose and improve astalgo.debug, replacing it by std/prettyprints,
-  refs https://github.com/nim-lang/RFCs/issues/385
-]#
-
-import options
-import std/wrapnils
-export wrapnils
-  # allows using things like: `?.n.sym.typ.len`
-
-import std/stackframes
-export stackframes
-  # allows using things like: `setFrameMsg c.config$n.info & " " & $n.kind`
-  # which doesn't log, but augments stacktrace with side channel information
-
-var conf0: ConfigRef
-
-proc onNewConfigRef*(conf: ConfigRef) {.inline.} =
-  ## Caches `conf`, which can be retrieved with `getConfigRef`.
-  ## This avoids having to forward `conf` all the way down the call chain to
-  ## procs that need it during a debugging session.
-  conf0 = conf
-
-proc getConfigRef*(): ConfigRef =
-  ## nil, if -d:nimDebugUtils wasn't specified
-  result = conf0
-
-proc isCompilerDebug*(): bool =
-  ##[
-  Provides a simple way for user code to enable/disable logging in the compiler
-  in a granular way. This can then be used in the compiler as follows:
-  ```nim
-  if isCompilerDebug():
-    echo ?.n.sym.typ.len
-  ```
-  ]##
-  runnableExamples:
-    proc main =
-      echo 2
-      {.define(nimCompilerDebug).}
-      echo 3.5 # code section in which `isCompilerDebug` will be true
-      {.undef(nimCompilerDebug).}
-      echo 'x'
-  conf0.isDefined("nimCompilerDebug")
-
-proc enteringDebugSection*() {.exportc, dynlib.} =
-  ## Provides a way for native debuggers to enable breakpoints, watchpoints, etc
-  ## when code of interest is being compiled.
-  ## 
-  ## Set your debugger to break on entering `nimCompilerIsEnteringDebugSection`
-  ## and then execute a desired command.
-  discard
-
-proc exitingDebugSection*() {.exportc, dynlib.} =
-  ## Provides a way for native debuggers to disable breakpoints, watchpoints, etc
-  ## when code of interest is no longer being compiled.
-  ## 
-  ## Set your debugger to break on entering `exitingDebugSection`
-  ## and then execute a desired command.
-  discard

compiler/depends.nim

@@ -9,17 +9,10 @@
 
 # This module implements a dependency file generator.
 
-import options, ast, ropes, passes, pathutils, msgs, lineinfos
-
-import modulegraphs
-
-import std/[os, parseutils]
-import strutils except addf
-import std/private/globs
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
+import
+  options, ast, ropes, idents, passes, modulepaths, pathutils
 
+from modulegraphs import ModuleGraph, PPassContext
 
 type
   TGen = object of PPassContext
@@ -32,53 +25,9 @@ type
     dotGraph: Rope
 
 proc addDependencyAux(b: Backend; importing, imported: string) =
-  b.dotGraph.addf("\"$1\" -> \"$2\";$n", [rope(importing), rope(imported)])
+  b.dotGraph.addf("$1 -> \"$2\";$n", [rope(importing), rope(imported)])
   # s1 -> s2_4[label="[0-9]"];
 
-proc toNimblePath(s: string, isStdlib: bool): string =
-  const stdPrefix = "std/"
-  const pkgPrefix = "pkg/"
-  if isStdlib:
-    let sub = "lib/"
-    var start = s.find(sub)
-    if start < 0:
-      doAssert false
-    else:
-      start += sub.len
-      let base = s[start..^1]
-
-      if base.startsWith("system") or base.startsWith("std"):
-        result = base
-      else:
-        for dir in stdlibDirs:
-          if base.startsWith(dir):
-            return stdPrefix & base.splitFile.name
-
-        result = stdPrefix & base
-  else:
-    var sub = getEnv("NIMBLE_DIR")
-    if sub.len == 0:
-      sub = ".nimble/pkgs/"
-    else:
-      sub.add "/pkgs/"
-    var start = s.find(sub)
-    if start < 0:
-      result = s
-    else:
-      start += sub.len
-      start += skipUntil(s, '/', start)
-      start += 1
-      result = pkgPrefix & s[start..^1]
-
-proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) =
-  doAssert n.kind == nkSym, $n.kind
-
-  let path = splitFile(toProjPath(g.config, n.sym.position.FileIndex))
-  let modulePath = splitFile(toProjPath(g.config, g.module.position.FileIndex))
-  let parent = nativeToUnixPath(modulePath.dir / modulePath.name).toNimblePath(belongsToStdlib(g.graph, g.module))
-  let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym))
-  addDependencyAux(b, parent, child)
-
 proc addDotDependency(c: PPassContext, n: PNode): PNode =
   result = n
   let g = PGen(c)
@@ -86,9 +35,11 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode =
   case n.kind
   of nkImportStmt:
     for i in 0..<n.len:
-      addDependency(c, g, b, n[i])
+      var imported = getModuleName(g.config, n[i])
+      addDependencyAux(b, g.module.name.s, imported)
   of nkFromStmt, nkImportExceptStmt:
-    addDependency(c, g, b, n[0])
+    var imported = getModuleName(g.config, n[0])
+    addDependencyAux(b, g.module.name.s, imported)
   of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr:
     for i in 0..<n.len: discard addDotDependency(c, n[i])
   else:
@@ -103,14 +54,14 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) =
 when not defined(nimHasSinkInference):
   {.pragma: nosinks.}
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext {.nosinks.} =
   var g: PGen
   new(g)
   g.module = module
   g.config = graph.config
   g.graph = graph
   if graph.backend == nil:
-    graph.backend = Backend(dotGraph: "")
+    graph.backend = Backend(dotGraph: nil)
   result = g
 
 const gendependPass* = makePass(open = myOpen, process = addDotDependency)

compiler/dfa.nim

@@ -9,33 +9,38 @@
 
 ## Data flow analysis for Nim.
 ## We transform the AST into a linear list of instructions first to
-## make this easier to handle: There are only 3 different branching
+## make this easier to handle: There are only 2 different branching
 ## instructions: 'goto X' is an unconditional goto, 'fork X'
 ## is a conditional goto (either the next instruction or 'X' can be
-## taken), 'loop X' is the only jump that jumps back.
-##
-## Exhaustive case statements are translated
+## taken). Exhaustive case statements are translated
 ## so that the last branch is transformed into an 'else' branch.
 ## ``return`` and ``break`` are all covered by 'goto'.
 ##
+## Control flow through exception handling:
+## Contrary to popular belief, exception handling doesn't cause
+## many problems for this DFA representation, ``raise`` is a statement
+## that ``goes to`` the outer ``finally`` or ``except`` if there is one,
+## otherwise it is the same as ``return``. Every call is treated as
+## a call that can potentially ``raise``. However, without a surrounding
+## ``try`` we don't emit these ``fork ReturnLabel`` instructions in order
+## to speed up the dataflow analysis passes.
+##
 ## The data structures and algorithms used here are inspired by
 ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
 ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
 
-import ast, intsets, lineinfos, renderer, aliasanalysis
-import std/private/asciitables
+import ast, types, intsets, lineinfos, renderer, asciitables
 
-when defined(nimPreviewSlimSystem):
-  import std/assertions
+from patterns import sameTrees
 
 type
   InstrKind* = enum
-    goto, loop, fork, def, use
+    goto, fork, def, use
   Instr* = object
+    n*: PNode # contains the def/use location.
     case kind*: InstrKind
-    of goto, fork, loop: dest*: int
-    of def, use:
-      n*: PNode # contains the def/use location.
+    of goto, fork: dest*: int
+    else: discard
 
   ControlFlowGraph* = seq[Instr]
 
@@ -52,10 +57,9 @@ type
 
   Con = object
     code: ControlFlowGraph
-    inTryStmt, interestingInstructions: int
+    inTryStmt: int
     blocks: seq[TBlock]
     owner: PSym
-    root: PSym
 
 proc codeListing(c: ControlFlowGraph, start = 0; last = -1): string =
   # for debugging purposes
@@ -63,7 +67,7 @@ proc codeListing(c: ControlFlowGraph, start = 0; last = -1): string =
   var jumpTargets = initIntSet()
   let last = if last < 0: c.len-1 else: min(last, c.len-1)
   for i in start..last:
-    if c[i].kind in {goto, fork, loop}:
+    if c[i].kind in {goto, fork}:
       jumpTargets.incl(i+c[i].dest)
   var i = start
   while i <= last:
@@ -74,12 +78,12 @@ proc codeListing(c: ControlFlowGraph, start = 0; last = -1): string =
     case c[i].kind
     of def, use:
       result.add renderTree(c[i].n)
-      result.add("\t#")
-      result.add($c[i].n.info.line)
-      result.add("\n")
-    of goto, fork, loop:
+    of goto, fork:
       result.add "L"
       result.addInt c[i].dest+i
+    result.add("\t#")
+    result.add($c[i].n.info.line)
+    result.add("\n")
     inc i
   if i in jumpTargets: result.add("L" & $i & ": End\n")
 
@@ -87,13 +91,181 @@ proc echoCfg*(c: ControlFlowGraph; start = 0; last = -1) {.deprecated.} =
   ## echos the ControlFlowGraph for debugging purposes.
   echo codeListing(c, start, last).alignTable
 
-proc forkI(c: var Con): TPosition =
+proc forkI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
-  c.code.add Instr(kind: fork, dest: 0)
+  c.code.add Instr(n: n, kind: fork, dest: 0)
 
-proc gotoI(c: var Con): TPosition =
+proc gotoI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
-  c.code.add Instr(kind: goto, dest: 0)
+  c.code.add Instr(n: n, kind: goto, dest: 0)
+
+#[
+
+Join is no more
+===============
+Instead of generating join instructions we adapt our traversal of the CFG.
+
+When encountering a fork we split into two paths, we follow the path
+starting at "pc + 1" until it encounters the joinpoint: "pc + forkInstr.dest".
+If we encounter gotos that would jump further than the current joinpoint,
+as can happen with gotos generated by unstructured controlflow such as break, raise or return,
+we simply suspend following the current path, and follow the other path until the new joinpoint
+which is simply the instruction pointer returned to us by the now suspended path.
+If the path we are following now, also encounters a goto that exceeds the joinpoint
+we repeat the process; suspending the current path and evaluating the other one with a new joinpoint.
+If we eventually reach a common joinpoint we join the two paths.
+This new "ping-pong" approach has the obvious advantage of not requiring join instructions, as such
+cutting down on the CFG size but is also mandatory for correctly handling complicated cases
+of unstructured controlflow.
+
+
+Design of join
+==============
+
+block:
+  if cond: break
+  def(x)
+
+use(x)
+
+Generates:
+
+L0: fork lab1
+  join L0  # patched.
+  goto Louter
+lab1:
+  def x
+  join L0
+Louter:
+  use x
+
+
+block outer:
+  while a:
+    while b:
+      if foo:
+        if bar:
+          break outer # --> we need to 'join' every pushed 'fork' here
+
+
+This works and then our abstract interpretation needs to deal with 'fork'
+differently. It really causes a split in execution. Two threads are
+"spawned" and both need to reach the 'join L' instruction. Afterwards
+the abstract interpretations are joined and execution resumes single
+threaded.
+
+
+Abstract Interpretation
+-----------------------
+
+proc interpret(pc, state, comesFrom): state =
+  result = state
+  # we need an explicit 'create' instruction (an explicit heap), in order
+  # to deal with 'var x = create(); var y = x; var z = y; destroy(z)'
+  while true:
+    case pc
+    of fork:
+      let a = interpret(pc+1, result, pc)
+      let b = interpret(forkTarget, result, pc)
+      result = a ++ b # ++ is a union operation
+      inc pc
+    of join:
+      if joinTarget == comesFrom: return result
+      else: inc pc
+    of use X:
+      if not result.contains(x):
+        error "variable not initialized " & x
+      inc pc
+    of def X:
+      if not result.contains(x):
+        result.incl X
+      else:
+        error "overwrite of variable causes memory leak " & x
+      inc pc
+    of destroy X:
+      result.excl X
+
+This is correct but still can lead to false positives:
+
+proc p(cond: bool) =
+  if cond:
+    new(x)
+  otherThings()
+  if cond:
+    destroy x
+
+Is not a leak. We should find a way to model *data* flow, not just
+control flow. One solution is to rewrite the 'if' without a fork
+instruction. The unstructured aspect can now be easily dealt with
+the 'goto' and 'join' instructions.
+
+proc p(cond: bool) =
+  L0: fork Lend
+    new(x)
+    # do not 'join' here!
+
+  Lend:
+    otherThings()
+    join L0  # SKIP THIS FOR new(x) SOMEHOW
+  destroy x
+  join L0 # but here.
+
+
+
+But if we follow 'goto Louter' we will never come to the join point.
+We restore the bindings after popping pc from the stack then there
+"no" problem?!
+
+
+while cond:
+  prelude()
+  if not condB: break
+  postlude()
+
+--->
+var setFlag = true
+while cond and not setFlag:
+  prelude()
+  if not condB:
+    setFlag = true   # BUT: Dependency
+  if not setFlag:    # HERE
+    postlude()
+
+--->
+var setFlag = true
+while cond and not setFlag:
+  prelude()
+  if not condB:
+    postlude()
+    setFlag = true
+
+
+-------------------------------------------------
+
+while cond:
+  prelude()
+  if more:
+    if not condB: break
+    stuffHere()
+  postlude()
+
+-->
+var setFlag = true
+while cond and not setFlag:
+  prelude()
+  if more:
+    if not condB:
+      setFlag = false
+    else:
+      stuffHere()
+      postlude()
+  else:
+    postlude()
+
+This is getting complicated. Instead we keep the whole 'join' idea but
+duplicate the 'join' instructions on breaks and return exits!
+
+]#
 
 proc genLabel(c: Con): TPosition = TPosition(c.code.len)
 
@@ -101,8 +273,8 @@ template checkedDistance(dist): int =
   doAssert low(int) div 2 + 1 < dist and dist < high(int) div 2
   dist
 
-proc jmpBack(c: var Con, p = TPosition(0)) =
-  c.code.add Instr(kind: loop, dest: checkedDistance(p.int - c.code.len))
+proc jmpBack(c: var Con, n: PNode, p = TPosition(0)) =
+  c.code.add Instr(n: n, kind: goto, dest: checkedDistance(p.int - c.code.len))
 
 proc patch(c: var Con, p: TPosition) =
   # patch with current index
@@ -112,12 +284,12 @@ proc gen(c: var Con; n: PNode)
 
 proc popBlock(c: var Con; oldLen: int) =
   var exits: seq[TPosition]
-  exits.add c.gotoI()
+  exits.add c.gotoI(newNode(nkEmpty))
   for f in c.blocks[oldLen].breakFixups:
     c.patch(f[0])
     for finale in f[1]:
       c.gen(finale)
-    exits.add c.gotoI()
+    exits.add c.gotoI(newNode(nkEmpty))
   for e in exits:
     c.patch e
   c.blocks.setLen(oldLen)
@@ -132,29 +304,86 @@ proc isTrue(n: PNode): bool =
   n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or
     n.kind == nkIntLit and n.intVal != 0
 
-template forkT(body) =
-  let lab1 = c.forkI()
+when true:
+  proc genWhile(c: var Con; n: PNode) =
+    # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
+    # through the loop. We need to prove this is correct for our
+    # purposes. But Herb Sutter claims it is. (Proof by authority.)
+    #[
+    while cond:
+      body
+
+    Becomes:
+
+    block:
+      if cond:
+        body
+        if cond:
+          body
+          if cond:
+            body
+
+    We still need to ensure 'break' resolves properly, so an AST to AST
+    translation is impossible.
+
+    So the code to generate is:
+
+      cond
+      fork L4  # F1
+      body
+      cond
+      fork L5  # F2
+      body
+      cond
+      fork L6  # F3
+      body
+    L6:
+      join F3
+    L5:
+      join F2
+    L4:
+      join F1
+    ]#
+    if isTrue(n[0]):
+      # 'while true' is an idiom in Nim and so we produce
+      # better code for it:
+      withBlock(nil):
+        for i in 0..2:
+          c.gen(n[1])
+    else:
+      withBlock(nil):
+        var endings: array[3, TPosition]
+        for i in 0..2:
+          c.gen(n[0])
+          endings[i] = c.forkI(n)
+          c.gen(n[1])
+        for i in countdown(endings.high, 0):
+          c.patch(endings[i])
+
+else:
+  proc genWhile(c: var Con; n: PNode) =
+    # lab1:
+    #   cond, tmp
+    #   fork tmp, lab2
+    #   body
+    #   jmp lab1
+    # lab2:
+    let lab1 = c.genLabel
+    withBlock(nil):
+      if isTrue(n[0]):
+        c.gen(n[1])
+        c.jmpBack(n, lab1)
+      else:
+        c.gen(n[0])
+        forkT(n):
+          c.gen(n[1])
+          c.jmpBack(n, lab1)
+
+template forkT(n, body) =
+  let lab1 = c.forkI(n)
   body
   c.patch(lab1)
 
-proc genWhile(c: var Con; n: PNode) =
-  # lab1:
-  #   cond, tmp
-  #   fork tmp, lab2
-  #   body
-  #   jmp lab1
-  # lab2:
-  let lab1 = c.genLabel
-  withBlock(nil):
-    if isTrue(n[0]):
-      c.gen(n[1])
-      c.jmpBack(lab1)
-    else:
-      c.gen(n[0])
-      forkT:
-        c.gen(n[1])
-        c.jmpBack(lab1)
-
 proc genIf(c: var Con, n: PNode) =
   #[
 
@@ -193,22 +422,15 @@ proc genIf(c: var Con, n: PNode) =
 
   ]#
   var endings: seq[TPosition] = @[]
-  let oldInteresting = c.interestingInstructions
-  let oldLen = c.code.len
-
   for i in 0..<n.len:
     let it = n[i]
     c.gen(it[0])
     if it.len == 2:
-      forkT:
-        c.gen(it.lastSon)
-        endings.add c.gotoI()
-
-  if oldInteresting == c.interestingInstructions:
-    setLen c.code, oldLen
-  else:
-    for i in countdown(endings.high, 0):
-      c.patch(endings[i])
+      forkT(it[1]):
+        c.gen(it[1])
+        endings.add c.gotoI(it[1])
+  for i in countdown(endings.high, 0):
+    c.patch(endings[i])
 
 proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, a
@@ -217,7 +439,7 @@ proc genAndOr(c: var Con; n: PNode) =
   # lab1:
   #   join F1
   c.gen(n[1])
-  forkT:
+  forkT(n):
     c.gen(n[2])
 
 proc genCase(c: var Con; n: PNode) =
@@ -232,34 +454,29 @@ proc genCase(c: var Con; n: PNode) =
   #    elsePart
   #  Lend:
   let isExhaustive = skipTypes(n[0].typ,
-    abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString, tyCstring}
+    abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
 
   var endings: seq[TPosition] = @[]
   c.gen(n[0])
-  let oldInteresting = c.interestingInstructions
-  let oldLen = c.code.len
   for i in 1..<n.len:
     let it = n[i]
     if it.len == 1 or (i == n.len-1 and isExhaustive):
       # treat the last branch as 'else' if this is an exhaustive case statement.
       c.gen(it.lastSon)
     else:
-      forkT:
+      forkT(it.lastSon):
         c.gen(it.lastSon)
-        endings.add c.gotoI()
-
-  if oldInteresting == c.interestingInstructions:
-    setLen c.code, oldLen
-  else:
-    for i in countdown(endings.high, 0):
-      c.patch(endings[i])
+        endings.add c.gotoI(it.lastSon)
+  for i in countdown(endings.high, 0):
+    let endPos = endings[i]
+    c.patch(endPos)
 
 proc genBlock(c: var Con; n: PNode) =
   withBlock(n[0].sym):
     c.gen(n[1])
 
 proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) =
-  let lab1 = c.gotoI()
+  let lab1 = c.gotoI(n)
   if c.blocks[i].isTryBlock:
     c.blocks[i].raiseFixups.add lab1
   else:
@@ -272,7 +489,6 @@ proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) =
     c.blocks[i].breakFixups.add (lab1, trailingFinales)
 
 proc genBreak(c: var Con; n: PNode) =
-  inc c.interestingInstructions
   if n[0].kind == nkSym:
     for i in countdown(c.blocks.high, 0):
       if not c.blocks[i].isTryBlock and c.blocks[i].label == n[0].sym:
@@ -303,9 +519,9 @@ proc genTry(c: var Con; n: PNode) =
   for i in 1..<n.len:
     let it = n[i]
     if it.kind != nkFinally:
-      forkT:
+      forkT(it):
         c.gen(it.lastSon)
-        endings.add c.gotoI()
+        endings.add c.gotoI(it)
   for i in countdown(endings.high, 0):
     c.patch(endings[i])
 
@@ -313,12 +529,11 @@ proc genTry(c: var Con; n: PNode) =
   if fin.kind == nkFinally:
     c.gen(fin[0])
 
-template genNoReturn(c: var Con) =
+template genNoReturn(c: var Con; n: PNode) =
   # leave the graph
-  c.code.add Instr(kind: goto, dest: high(int) - c.code.len)
+  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 proc genRaise(c: var Con; n: PNode) =
-  inc c.interestingInstructions
   gen(c, n[0])
   if c.inTryStmt > 0:
     for i in countdown(c.blocks.high, 0):
@@ -327,14 +542,13 @@ proc genRaise(c: var Con; n: PNode) =
         return
     assert false #Unreachable
   else:
-    genNoReturn(c)
+    genNoReturn(c, n)
 
 proc genImplicitReturn(c: var Con) =
   if c.owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter} and resultPos < c.owner.ast.len:
     gen(c, c.owner.ast[resultPos])
 
 proc genReturn(c: var Con; n: PNode) =
-  inc c.interestingInstructions
   if n[0].kind != nkEmpty:
     gen(c, n[0])
   else:
@@ -343,6 +557,89 @@ proc genReturn(c: var Con; n: PNode) =
 
 const
   InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp}
+  PathKinds0 = {nkDotExpr, nkCheckedFieldExpr,
+                nkBracketExpr, nkDerefExpr, nkHiddenDeref,
+                nkAddr, nkHiddenAddr,
+                nkObjDownConv, nkObjUpConv}
+  PathKinds1 = {nkHiddenStdConv, nkHiddenSubConv}
+
+proc skipConvDfa*(n: PNode): PNode =
+  result = n
+  while true:
+    case result.kind
+    of nkObjDownConv, nkObjUpConv:
+      result = result[0]
+    of PathKinds1:
+      result = result[1]
+    else: break
+
+proc aliases*(obj, field: PNode): bool =
+  var n = field
+  var obj = obj
+  while true:
+    case obj.kind
+    of {nkObjDownConv, nkObjUpConv, nkAddr, nkHiddenAddr, nkDerefExpr, nkHiddenDeref}:
+      obj = obj[0]
+    of PathKinds1:
+      obj = obj[1]
+    else: break
+  while true:
+    if sameTrees(obj, n): return true
+    case n.kind
+    of PathKinds0:
+      n = n[0]
+    of PathKinds1:
+      n = n[1]
+    else: break
+
+type InstrTargetKind* = enum
+  None, Full, Partial
+
+proc instrTargets*(insloc, loc: PNode): InstrTargetKind =
+  if sameTrees(insloc, loc) or insloc.aliases(loc):
+    Full    # x -> x; x -> x.f
+  elif loc.aliases(insloc):
+    Partial # x.f -> x
+  else: None
+
+proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
+  var n = orig
+  while true:
+    case n.kind
+    of PathKinds0 - {nkBracketExpr, nkHiddenDeref, nkDerefExpr}:
+      n = n[0]
+    of PathKinds1:
+      n = n[1]
+    of nkBracketExpr:
+      # in a[i] the 'i' must be known
+      if n.len > 1 and n[1].kind in {nkCharLit..nkUInt64Lit}:
+        n = n[0]
+      else:
+        return false
+    of nkHiddenDeref, nkDerefExpr:
+      # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
+      # pointer indirection.
+      # bug #14159, we cannot reason about sinkParam[].location as it can
+      # still be shared for tyRef.
+      n = n[0]
+      return n.kind == nkSym and n.sym.owner == owner and
+         (n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
+    else: break
+  # XXX Allow closure deref operations here if we know
+  # the owner controlled the closure allocation?
+  result = n.kind == nkSym and n.sym.owner == owner and
+    {sfGlobal, sfThread, sfCursor} * n.sym.flags == {} and
+    (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar)
+  # Note: There is a different move analyzer possible that checks for
+  # consume(param.key); param.key = newValue  for all paths. Then code like
+  #
+  #   let splited = split(move self.root, x)
+  #   self.root = merge(splited.lower, splited.greater)
+  #
+  # could be written without the ``move self.root``. However, this would be
+  # wrong! Then the write barrier for the ``self.root`` assignment would
+  # free the old data and all is lost! Lesson: Don't be too smart, trust the
+  # lower level C++ optimizer to specialize this code.
 
 proc skipTrivials(c: var Con, n: PNode): PNode =
   result = n
@@ -360,20 +657,16 @@ proc skipTrivials(c: var Con, n: PNode): PNode =
 proc genUse(c: var Con; orig: PNode) =
   let n = c.skipTrivials(orig)
 
-  if n.kind == nkSym:
-    if n.sym.kind in InterestingSyms and n.sym == c.root:
-      c.code.add Instr(kind: use, n: orig)
-      inc c.interestingInstructions
-  else:
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    c.code.add Instr(n: orig, kind: use)
+  elif n.kind in nkCallKinds:
     gen(c, n)
 
 proc genDef(c: var Con; orig: PNode) =
   let n = c.skipTrivials(orig)
 
   if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    if n.sym == c.root:
-      c.code.add Instr(kind: def, n: orig)
-      inc c.interestingInstructions
+    c.code.add Instr(n: orig, kind: def)
 
 proc genCall(c: var Con; n: PNode) =
   gen(c, n[0])
@@ -381,17 +674,18 @@ proc genCall(c: var Con; n: PNode) =
   if t != nil: t = t.skipTypes(abstractInst)
   for i in 1..<n.len:
     gen(c, n[i])
-    if t != nil and i < t.len and isOutParam(t[i]):
-      # Pass by 'out' is a 'must def'. Good enough for a move optimizer.
-      genDef(c, n[i])
+    when false:
+      if t != nil and i < t.len and t[i].kind == tyOut:
+        # Pass by 'out' is a 'must def'. Good enough for a move optimizer.
+        genDef(c, n[i])
   # every call can potentially raise:
-  if false: # c.inTryStmt > 0 and canRaiseConservative(n[0]):
+  if c.inTryStmt > 0 and canRaiseConservative(n[0]):
     # we generate the instruction sequence:
     # fork lab1
     # goto exceptionHandler (except or finally)
     # lab1:
     # join F1
-    forkT:
+    forkT(n):
       for i in countdown(c.blocks.high, 0):
         if c.blocks[i].isTryBlock:
           genBreakOrRaiseAux(c, i, n)
@@ -429,18 +723,12 @@ proc gen(c: var Con; n: PNode) =
       else:
         genCall(c, n)
       if sfNoReturn in n[0].sym.flags:
-        genNoReturn(c)
+        genNoReturn(c, n)
     else:
       genCall(c, n)
   of nkCharLit..nkNilLit: discard
-  of nkAsgn, nkFastAsgn, nkSinkAsgn:
+  of nkAsgn, nkFastAsgn:
     gen(c, n[1])
-
-    if n[0].kind in PathKinds0:
-      let a = c.skipTrivials(n[0])
-      if a.kind in nkCallKinds:
-        gen(c, a)
-
     # watch out: 'obj[i].f2 = value' sets 'f2' but
     # "uses" 'i'. But we are only talking about builtin array indexing so
     # it doesn't matter and 'x = 34' is NOT a usage of 'x'.
@@ -470,32 +758,13 @@ proc gen(c: var Con; n: PNode) =
   of nkDefer: doAssert false, "dfa construction pass requires the elimination of 'defer'"
   else: discard
 
-when false:
-  proc optimizeJumps(c: var ControlFlowGraph) =
-    for i in 0..<c.len:
-      case c[i].kind
-      of goto, fork:
-        var pc = i + c[i].dest
-        if pc < c.len and c[pc].kind == goto:
-          while pc < c.len and c[pc].kind == goto:
-            let newPc = pc + c[pc].dest
-            if newPc > pc:
-              pc = newPc
-            else:
-              break
-          c[i].dest = pc - i
-      of loop, def, use: discard
-
-proc constructCfg*(s: PSym; body: PNode; root: PSym): ControlFlowGraph =
+proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
   ## constructs a control flow graph for ``body``.
-  var c = Con(code: @[], blocks: @[], owner: s, root: root)
+  var c = Con(code: @[], blocks: @[], owner: s)
   withBlock(s):
     gen(c, body)
-    if root.kind == skResult:
-      genImplicitReturn(c)
+    genImplicitReturn(c)
   when defined(gcArc) or defined(gcOrc):
     result = c.code # will move
   else:
     shallowCopy(result, c.code)
-  when false:
-    optimizeJumps result

compiler/docgen2.nim

@@ -11,7 +11,7 @@
 # semantic checking.
 
 import
-  options, ast, msgs, passes, docgen, lineinfos, pathutils, packages
+  options, ast, msgs, passes, docgen, lineinfos, pathutils
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -23,7 +23,7 @@ type
   PGen = ref TGen
 
 proc shouldProcess(g: PGen): bool =
-  (optWholeProject in g.doc.conf.globalOptions and g.doc.conf.belongsToProjectPackage(g.module)) or
+  (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or
       sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
 
 template closeImpl(body: untyped) {.dirty.} =
@@ -31,7 +31,6 @@ template closeImpl(body: untyped) {.dirty.} =
   let useWarning = sfMainModule notin g.module.flags
   let groupedToc = true
   if shouldProcess(g):
-    finishGenerateDoc(g.doc)
     body
     try:
       generateIndex(g.doc)
@@ -64,22 +63,18 @@ template myOpenImpl(ext: untyped) {.dirty.} =
   g.module = module
   g.config = graph.config
   var d = newDocumentor(AbsoluteFile toFullPath(graph.config, FileIndex module.position),
-      graph.cache, graph.config, ext, module, hasToc = true)
+      graph.cache, graph.config, ext, module)
+  d.hasToc = true
   g.doc = d
   result = g
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
   myOpenImpl(HtmlExt)
 
-proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
-  myOpenImpl(TexExt)
-
-proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc myOpenJson(graph: ModuleGraph; module: PSym): PPassContext =
   myOpenImpl(JsonExt)
 
 const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
-const docgen2TexPass* = makePass(open = myOpenTex, process = processNode,
-                                 close = close)
 const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
                                   close = closeJson)
 

compiler/enumtostr.nim

@@ -1,20 +1,16 @@
 
 import ast, idents, lineinfos, modulegraphs, magicsys
 
-when defined(nimPreviewSlimSystem):
-  import std/assertions
+proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
+  result = newSym(skProc, getIdent(g.cache, "$"), t.owner, info)
 
-
-proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
-  result = newSym(skProc, getIdent(g.cache, "$"), nextSymId idgen, t.owner, info)
-
-  let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info)
+  let dest = newSym(skParam, getIdent(g.cache, "e"), result, info)
   dest.typ = t
 
-  let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), result, info)
   res.typ = getSysType(g, info, tyString)
 
-  result.typ = newType(tyProc, nextTypeId idgen, t.owner)
+  result.typ = newType(tyProc, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
   result.typ.n.add newNodeI(nkEffectList, info)
@@ -30,7 +26,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGener
     assert(t.n[i].kind == nkSym)
     var field = t.n[i].sym
     let val = if field.ast == nil: field.name.s else: field.ast.strVal
-    caseStmt.add newTree(nkOfBranch, newIntTypeNode(field.position, t),
+    caseStmt.add newTree(nkOfBranch, newSymNode(field),
       newTree(nkStmtList, newTree(nkFastAsgn, newSymNode(res), newStrNode(val, info))))
     #newIntTypeNode(nkIntLit, field.position, t)
 
@@ -66,16 +62,16 @@ proc searchObjCase(t: PType; field: PSym): PNode =
     result = searchObjCase(t[0].skipTypes({tyAlias, tyGenericInst, tyRef, tyPtr}), field)
   doAssert result != nil
 
-proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
-  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), nextSymId idgen, t.owner, info)
+proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph): PSym =
+  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), t.owner, info)
 
-  let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info)
+  let dest = newSym(skParam, getIdent(g.cache, "e"), result, info)
   dest.typ = field.typ
 
-  let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), result, info)
   res.typ = getSysType(g, info, tyUInt8)
 
-  result.typ = newType(tyProc, nextTypeId idgen, t.owner)
+  result.typ = newType(tyProc, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
   result.typ.n.add newNodeI(nkEffectList, info)

compiler/errorhandling.nim

@@ -1,82 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2021 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module contains support code for new-styled error
-## handling via an `nkError` node kind.
-
-import ast, renderer, options, strutils, types
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
-type
-  ErrorKind* = enum ## expand as you need.
-    RawTypeMismatchError
-    ExpressionCannotBeCalled
-    CustomError
-    WrongNumberOfArguments
-    AmbiguousCall
-
-proc errorSubNode*(n: PNode): PNode =
-  case n.kind
-  of nkEmpty..nkNilLit:
-    result = nil
-  of nkError:
-    result = n
-  else:
-    result = nil
-    for i in 0..<n.len:
-      result = errorSubNode(n[i])
-      if result != nil: break
-
-proc newError*(wrongNode: PNode; k: ErrorKind; args: varargs[PNode]): PNode =
-  assert wrongNode.kind != nkError
-  let innerError = errorSubNode(wrongNode)
-  if innerError != nil:
-    return innerError
-  result = newNodeIT(nkError, wrongNode.info, newType(tyError, ItemId(module: -1, item: -1), nil))
-  result.add wrongNode
-  result.add newIntNode(nkIntLit, ord(k))
-  for a in args: result.add a
-
-proc newError*(wrongNode: PNode; msg: string): PNode =
-  assert wrongNode.kind != nkError
-  let innerError = errorSubNode(wrongNode)
-  if innerError != nil:
-    return innerError
-  result = newNodeIT(nkError, wrongNode.info, newType(tyError, ItemId(module: -1, item: -1), nil))
-  result.add wrongNode
-  result.add newIntNode(nkIntLit, ord(CustomError))
-  result.add newStrNode(msg, wrongNode.info)
-
-proc errorToString*(config: ConfigRef; n: PNode): string =
-  assert n.kind == nkError
-  assert n.len > 1
-  let wrongNode = n[0]
-  case ErrorKind(n[1].intVal)
-  of RawTypeMismatchError:
-    result = "type mismatch"
-  of ExpressionCannotBeCalled:
-    result = "expression '$1' cannot be called" % wrongNode[0].renderTree
-  of CustomError:
-    result = n[2].strVal
-  of WrongNumberOfArguments:
-    result = "wrong number of arguments"
-  of AmbiguousCall:
-    let a = n[2].sym
-    let b = n[3].sym
-    var args = "("
-    for i in 1..<wrongNode.len:
-      if i > 1: args.add(", ")
-      args.add(typeToString(wrongNode[i].typ))
-    args.add(")")
-    result = "ambiguous call; both $1 and $2 match for: $3" % [
-      getProcHeader(config, a),
-      getProcHeader(config, b),
-      args]

compiler/evalffi.nim

@@ -65,16 +65,16 @@ proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode =
     if (lib.isNil or lib.kind == libHeader) and not gExeHandle.isNil:
       libPathMsg = "current exe: " & getAppFilename() & " nor libc: " & libcDll
       # first try this exe itself:
-      theAddr = gExeHandle.symAddr(name.cstring)
+      theAddr = gExeHandle.symAddr(name)
       # then try libc:
       if theAddr.isNil:
         let dllhandle = getDll(conf, gDllCache, libcDll, sym.info)
-        theAddr = dllhandle.symAddr(name.cstring)
+        theAddr = dllhandle.symAddr(name)
     elif not lib.isNil:
       let dll = if lib.kind == libHeader: libcDll else: lib.path.strVal
       libPathMsg = dll
       let dllhandle = getDll(conf, gDllCache, dll, sym.info)
-      theAddr = dllhandle.symAddr(name.cstring)
+      theAddr = dllhandle.symAddr(name)
     if theAddr.isNil: globalError(conf, sym.info,
       "cannot import symbol: " & name & " from " & libPathMsg)
     result.intVal = cast[ByteAddress](theAddr)
@@ -92,7 +92,7 @@ proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type =
     else: result = nil
   of tyFloat, tyFloat64: result = addr libffi.type_double
   of tyFloat32: result = addr libffi.type_float
-  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCstring, tySequence, tyString, tyUntyped,
+  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyUntyped,
      tyTyped, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
     result = addr libffi.type_pointer
   of tyDistinct, tyAlias, tySink:
@@ -110,8 +110,8 @@ proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI
   else:
     globalError(conf, info, "cannot map calling convention to FFI")
 
-template rd(typ, p: untyped): untyped = (cast[ptr typ](p))[]
-template wr(typ, p, v: untyped): untyped = (cast[ptr typ](p))[] = v
+template rd(T, p: untyped): untyped = (cast[ptr T](p))[]
+template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v
 template `+!`(x, y: untyped): untyped =
   cast[pointer](cast[ByteAddress](x) + y)
 
@@ -177,8 +177,8 @@ const maxPackDepth = 20
 var packRecCheck = 0
 
 proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
-  template awr(typ, v: untyped): untyped =
-    wr(typ, res, v)
+  template awr(T, v: untyped): untyped =
+    wr(T, res, v)
 
   case typ.kind
   of tyBool: awr(bool, v.intVal != 0)
@@ -205,7 +205,7 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
   of tyFloat32: awr(float32, v.floatVal)
   of tyFloat64: awr(float64, v.floatVal)
 
-  of tyPointer, tyProc,  tyCstring, tyString:
+  of tyPointer, tyProc,  tyCString, tyString:
     if v.kind == nkNilLit:
       # nothing to do since the memory is 0 initialized anyway
       discard
@@ -386,7 +386,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
     result = unpackObject(conf, x, typ, n)
   of tyArray:
     result = unpackArray(conf, x, typ, n)
-  of tyCstring, tyString:
+  of tyCString, tyString:
     let p = rd(cstring, x)
     if p.isNil:
       setNil()
@@ -402,7 +402,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
 
 proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode =
   if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyLent, tyPointer,
-                                           tyProc, tyCstring, tyString,
+                                           tyProc, tyCString, tyString,
                                            tySequence}:
     result = newNodeIT(x.kind, x.info, destTyp)
     result.intVal = x.intVal

compiler/evaltempl.nim

@@ -22,11 +22,10 @@ type
     config: ConfigRef
     ic: IdentCache
     instID: int
-    idgen: IdGenerator
 
 proc copyNode(ctx: TemplCtx, a, b: PNode): PNode =
   result = copyNode(a)
-  if ctx.instLines: setInfoRecursive(result, b.info)
+  if ctx.instLines: result.info = b.info
 
 proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
   template handleParam(param) =
@@ -49,7 +48,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
         internalAssert c.config, sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
         if x == nil:
-          x = copySym(s, nextSymId(c.idgen))
+          x = copySym(s)
           # sem'check needs to set the owner properly later, see bug #9476
           x.owner = nil # c.genSymOwner
           #if x.kind == skParam and x.owner.kind == skModule:
@@ -83,14 +82,10 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
         not c.isDeclarative:
       c.isDeclarative = true
       isDeclarative = true
-    if (not c.isDeclarative) and templ.kind in nkCallKinds and isRunnableExamples(templ[0]):
-      # fixes bug #16993, bug #18054
-      discard
-    else:
-      var res = copyNode(c, templ, actual)
-      for i in 0..<templ.len:
-        evalTemplateAux(templ[i], actual, c, res)
-      result.add res
+    var res = copyNode(c, templ, actual)
+    for i in 0..<templ.len:
+      evalTemplateAux(templ[i], actual, c, res)
+    result.add res
     if isDeclarative: c.isDeclarative = false
 
 const
@@ -173,7 +168,6 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
 proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
                    conf: ConfigRef;
                    ic: IdentCache; instID: ref int;
-                   idgen: IdGenerator;
                    fromHlo=false): PNode =
   inc(conf.evalTemplateCounter)
   if conf.evalTemplateCounter > evalTemplateLimit:
@@ -189,9 +183,8 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   ctx.ic = ic
   initIdTable(ctx.mapping)
   ctx.instID = instID[]
-  ctx.idgen = idgen
 
-  let body = tmpl.ast[bodyPos]
+  let body = tmpl.getBody
   #echo "instantion of ", renderTree(body, {renderIds})
   if isAtom(body):
     result = newNodeI(nkPar, body.info)
@@ -204,7 +197,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
     result = copyNode(body)
     ctx.instLines = sfCallsite in tmpl.flags
     if ctx.instLines:
-      setInfoRecursive(result, n.info)
+      result.info = n.info
     for i in 0..<body.safeLen:
       evalTemplateAux(body[i], args, ctx, result)
   result.flags.incl nfFromTemplate

compiler/extccomp.nim

@@ -12,14 +12,9 @@
 # from a lineinfos file, to provide generalized procedures to compile
 # nim files.
 
-import ropes, platform, condsyms, options, msgs, lineinfos, pathutils, modulepaths
-
-import std/[os, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar, parseutils]
-
-import std / strutils except addf
-
-when defined(nimPreviewSlimSystem):
-  import std/syncio
+import
+  ropes, os, strutils, osproc, platform, condsyms, options, msgs,
+  lineinfos, std / sha1, streams, pathutils, sequtils, times, strtabs
 
 type
   TInfoCCProp* = enum         # properties of the C compiler:
@@ -88,10 +83,10 @@ compiler gcc:
     linkLibCmd: " -l$1",
     debug: "",
     pic: "-fPIC",
-    asmStmtFrmt: "__asm__($1);$n",
+    asmStmtFrmt: "asm($1);$n",
     structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name
     produceAsm: gnuAsmListing,
-    cppXsupport: "-std=gnu++17 -funsigned-char",
+    cppXsupport: "-std=gnu++14 -funsigned-char",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm,
             hasAttribute})
 
@@ -118,7 +113,7 @@ compiler nintendoSwitchGCC:
     asmStmtFrmt: "asm($1);$n",
     structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name
     produceAsm: gnuAsmListing,
-    cppXsupport: "-std=gnu++17 -funsigned-char",
+    cppXsupport: "-std=gnu++14 -funsigned-char",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm,
             hasAttribute})
 
@@ -129,8 +124,8 @@ compiler llvmGcc:
   result.name = "llvm_gcc"
   result.compilerExe = "llvm-gcc"
   result.cppCompiler = "llvm-g++"
-  when defined(macosx) or defined(openbsd):
-    # `llvm-ar` not available
+  when defined(macosx):
+    # OS X has no 'llvm-ar' tool:
     result.buildLib = "ar rcs $libfile $objfiles"
   else:
     result.buildLib = "llvm-ar rcs $libfile $objfiles"
@@ -155,7 +150,7 @@ compiler vcc:
     compileTmpl: "/c$vccplatform $options $include /nologo /Fo$objfile $file",
     buildGui: " /SUBSYSTEM:WINDOWS user32.lib ",
     buildDll: " /LD",
-    buildLib: "vccexe --command:lib$vccplatform /nologo /OUT:$libfile $objfiles",
+    buildLib: "lib /OUT:$libfile $objfiles",
     linkerExe: "cl",
     linkTmpl: "$builddll$vccplatform /Fe$exefile $objfiles $buildgui /nologo $options",
     includeCmd: " /I",
@@ -257,7 +252,7 @@ compiler envcc:
     buildGui: "",
     buildDll: " -shared ",
     buildLib: "", # XXX: not supported yet
-    linkerExe: "",
+    linkerExe: "cc",
     linkTmpl: "-o $exefile $buildgui $builddll $objfiles $options",
     includeCmd: " -I",
     linkDirCmd: "", # XXX: not supported yet
@@ -286,11 +281,6 @@ const
 
   hExt* = ".h"
 
-template writePrettyCmdsStderr(cmd) =
-  if cmd.len > 0:
-    flushDot(conf)
-    stderr.writeLine(cmd)
-
 proc nameToCC*(name: string): TSystemCC =
   ## Returns the kind of compiler referred to by `name`, or ccNone
   ## if the name doesn't refer to any known compiler.
@@ -372,7 +362,6 @@ proc initVars*(conf: ConfigRef) =
 
 proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile,
                         createSubDir: bool = true): AbsoluteFile =
-  ## Generate the absolute file path to the generated modules.
   result = completeGeneratedFilePath(conf, cfile, createSubDir)
 
 proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile =
@@ -383,7 +372,7 @@ proc addFileToCompile*(conf: ConfigRef; cf: Cfile) =
   conf.toCompile.add(cf)
 
 proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) =
-  let key = completeCfilePath(conf, mangleModuleName(conf, nimfile).AbsoluteFile).string
+  let key = completeCfilePath(conf, withPackageName(conf, nimfile)).string
   var value = conf.cfileSpecificOptions.getOrDefault(key)
   if strutils.find(value, option, 0) < 0:
     addOpt(value, option)
@@ -436,13 +425,7 @@ proc noAbsolutePaths(conf: ConfigRef): bool {.inline.} =
   # really: Cross compilation from Linux to Linux for example is entirely
   # reasonable.
   # `optGenMapping` is included here for niminst.
-  # We use absolute paths for vcc / cl, see issue #19883.
-  let options =
-    if conf.cCompiler == ccVcc:
-      {optGenMapping}
-    else:
-      {optGenScript, optGenMapping}
-  result = conf.globalOptions * options != {}
+  result = conf.globalOptions * {optGenScript, optGenMapping} != {}
 
 proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string =
   result = conf.compileOptions
@@ -495,10 +478,7 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} =
            (conf.target.hostOS == osWindows)
 
 proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool =
-  # List of possible file extensions taken from gcc
-  for ext in [".C", ".cc", ".cpp", ".CPP", ".c++", ".cp", ".cxx"]:
-    if cfile.string.endsWith(ext): return true
-  false
+  conf.backend == backendCpp and not cfile.string.endsWith(".c")
 
 proc envFlags(conf: ConfigRef): string =
   result = if conf.backend == backendCpp:
@@ -506,14 +486,14 @@ proc envFlags(conf: ConfigRef): string =
           else:
             getEnv("CFLAGS")
 
-proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; isCpp: bool): string =
+proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string =
   if compiler == ccEnv:
-    result = if isCpp:
+    result = if useCpp(conf, cfile):
               getEnv("CXX")
             else:
               getEnv("CC")
   else:
-    result = if isCpp:
+    result = if useCpp(conf, cfile):
               CC[compiler].cppCompiler
             else:
               CC[compiler].compilerExe
@@ -527,36 +507,47 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool =
     result = false # assume an old or crappy GCC
     var exe = getConfigVar(conf, conf.cCompiler, ".exe")
     if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
-    # NOTE: should we need the full version, use -dumpfullversion
-    let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except: ("", 1)
+    let (s, exitCode) = try: execCmdEx(exe & " --version") except: ("", 1)
     if exitCode == 0:
-      var major: int
-      discard parseInt(s, major)
-      result = major >= 5
+      var i = 0
+      var j = 0
+      # the version is the last part of the first line:
+      while i < s.len and s[i] != '\n':
+        if s[i] in {' ', '\t'}: j = i+1
+        inc i
+      if j > 0:
+        var major = 0
+        while j < s.len and s[j] in {'0'..'9'}:
+          major = major * 10 + (ord(s[j]) - ord('0'))
+          inc j
+        if i < s.len and s[j] == '.': inc j
+        while j < s.len and s[j] in {'0'..'9'}:
+          inc j
+        if j+1 < s.len and s[j] == '.' and s[j+1] in {'0'..'9'}:
+          # we found a third version number, chances are high
+          # we really parsed the version:
+          result = major >= 5
   else:
     result = conf.cCompiler == ccCLang
 
 proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
-           else: getCompilerExe(conf, compiler, optMixedMode in conf.globalOptions or conf.backend == backendCpp)
+           elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler
+           else: getCompilerExe(conf, compiler, AbsoluteFile"")
 
 proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
                          isMainFile = false; produceOutput = false): string =
-  let
-    c = conf.cCompiler
-    isCpp = useCpp(conf, cfile.cname)
+  let c = conf.cCompiler
   # We produce files like module.nim.cpp, so the absolute Nim filename is not
   # cfile.name but `cfile.cname.changeFileExt("")`:
   var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string)
-  if isCpp:
+  if useCpp(conf, cfile.cname):
     # needs to be prepended so that --passc:-std=c++17 can override default.
     # we could avoid allocation by making cFileSpecificOptions inplace
     options = CC[c].cppXsupport & ' ' & options
-    # If any C++ file was compiled, we need to use C++ driver for linking as well
-    incl conf.globalOptions, optMixedMode
 
   var exe = getConfigVar(conf, c, ".exe")
-  if exe.len == 0: exe = getCompilerExe(conf, c, isCpp)
+  if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname)
 
   if needsExeExt(conf): exe = addFileExt(exe, "exe")
   if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and
@@ -576,14 +567,14 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
 
     compilePattern = joinPath(conf.cCompilerPath, exe)
   else:
-    compilePattern = getCompilerExe(conf, c, isCpp)
+    compilePattern = getCompilerExe(conf, c, cfile.cname)
 
   includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)]))
 
-  let cf = if noAbsolutePaths(conf): AbsoluteFile extractFilename(cfile.cname.string)
+  var cf = if noAbsolutePaths(conf): AbsoluteFile extractFilename(cfile.cname.string)
            else: cfile.cname
 
-  let objfile =
+  var objfile =
     if cfile.obj.isEmpty:
       if CfileFlag.External notin cfile.flags or noAbsolutePaths(conf):
         toObjFile(conf, cf).string
@@ -618,7 +609,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
           "for the selected C compiler: " & CC[conf.cCompiler].name)
 
   result.add(' ')
-  strutils.addf(result, CC[c].compileTmpl, [
+  result.addf(CC[c].compileTmpl, [
     "dfile", dfile,
     "file", cfsh, "objfile", quoteShell(objfile),
     "options", options, "include", includeCmd,
@@ -638,8 +629,8 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
 proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
   if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good
 
-  let hashFile = toGeneratedFile(conf, conf.mangleModuleName(cfile.cname).AbsoluteFile, "sha1")
-  let currentHash = footprint(conf, cfile)
+  var hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1")
+  var currentHash = footprint(conf, cfile)
   var f: File
   if open(f, hashFile.string, fmRead):
     let oldHash = parseSecureHash(f.readLine())
@@ -653,10 +644,8 @@ proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
       close(f)
 
 proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) =
-  # we want to generate the hash file unconditionally
-  let extFileChanged = externalFileChanged(conf, c)
   if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and
-      not extFileChanged:
+      not externalFileChanged(conf, c):
     c.flags.incl CfileFlag.Cached
   else:
     # make sure Nim keeps recompiling the external file on reruns
@@ -671,13 +660,10 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) =
   addExternalFileToCompile(conf, c)
 
 proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
-                objfiles: string, isDllBuild: bool, removeStaticFile: bool): string =
+                objfiles: string, isDllBuild: bool): string =
   if optGenStaticLib in conf.globalOptions:
-    if removeStaticFile:
-      removeFile output # fixes: bug #16947
     result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(output),
-                                            "objfiles", objfiles,
-                                            "vccplatform", vccplatform(conf)]
+                                            "objfiles", objfiles]
   else:
     var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe")
     if linkerExe.len == 0: linkerExe = getLinkerExe(conf, conf.cCompiler)
@@ -710,7 +696,7 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
         "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles,
         "exefile", exefile, "nim", getPrefixDir(conf).string, "lib", conf.libpath.string])
     result.add ' '
-    strutils.addf(result, linkTmpl, ["builddll", builddll,
+    result.addf(linkTmpl, ["builddll", builddll,
         "mapfile", mapfile,
         "buildgui", buildgui, "options", linkOptions,
         "objfiles", objfiles, "exefile", exefile,
@@ -758,9 +744,8 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
   if optCDebug in conf.globalOptions and conf.cCompiler == ccVcc:
     result.add " /Zi /FS /Od"
 
-template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string,
-                    removeStaticFile = false): string =
-  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions, removeStaticFile)
+template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string): string =
+  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions)
 
 template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped) =
   try:
@@ -847,9 +832,9 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu
 proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string =
   if conf.hasHint(hintCC):
     if optListCmd in conf.globalOptions or conf.verbosity > 1:
-      result = MsgKindToStr[hintCC] % (demangleModuleName(path.splitFile.name) & ": " & compileCmd)
+      result = MsgKindToStr[hintCC] % (demanglePackageName(path.splitFile.name) & ": " & compileCmd)
     else:
-      result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name)
+      result = MsgKindToStr[hintCC] % demanglePackageName(path.splitFile.name)
 
 proc callCCompiler*(conf: ConfigRef) =
   var
@@ -859,10 +844,11 @@ proc callCCompiler*(conf: ConfigRef) =
     return # speed up that call if only compiling and no script shall be
            # generated
   #var c = cCompiler
-  var script: Rope = ""
+  var script: Rope = nil
   var cmds: TStringSeq
   var prettyCmds: TStringSeq
-  let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx])
+  let prettyCb = proc (idx: int) =
+    if prettyCmds[idx].len > 0: echo prettyCmds[idx]
 
   for idx, it in conf.toCompile:
     # call the C compiler for the .c file:
@@ -900,7 +886,7 @@ proc callCCompiler*(conf: ConfigRef) =
         let objFile = conf.getObjFilePath(x)
         let buildDll = idx != mainFileIdx
         let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll)
-        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll, removeStaticFile = true))
+        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll))
         # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache
         # for more info check the comment inside of getLinkCmd() where the /PDB:<filename> MSVC flag is used
         if isVSCompatible(conf):
@@ -913,8 +899,8 @@ proc callCCompiler*(conf: ConfigRef) =
       # only if not cached - copy the resulting main file from the nimcache folder to its originally intended destination
       if CfileFlag.Cached notin conf.toCompile[mainFileIdx].flags:
         let mainObjFile = getObjFilePath(conf, conf.toCompile[mainFileIdx])
-        let src = conf.hcrLinkTargetName(mainObjFile, true)
-        let dst = conf.prepareToWriteOutput
+        var src = conf.hcrLinkTargetName(mainObjFile, true)
+        var dst = conf.prepareToWriteOutput
         copyFileWithPermissions(src.string, dst.string)
     else:
       for x in conf.toCompile:
@@ -922,9 +908,8 @@ proc callCCompiler*(conf: ConfigRef) =
         objfiles.add(' ')
         objfiles.add(quoteShell(objFile))
       let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput
-                       else: AbsoluteFile(conf.outFile)
-
-      linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true)
+                       else: AbsoluteFile(conf.projectName)
+      linkCmd = getLinkCmd(conf, mainOutput, objfiles)
       extraCmds = getExtraCmds(conf, mainOutput)
       if optCompileOnly notin conf.globalOptions:
         const MaxCmdLen = when defined(windows): 8_000 else: 32_000
@@ -944,99 +929,191 @@ proc callCCompiler*(conf: ConfigRef) =
     script.add("\n")
     generateScript(conf, script)
 
+#from json import escapeJson
+import json, std / sha1
+
 template hashNimExe(): string = $secureHashFile(os.getAppFilename())
 
-proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile =
-  # `outFile` is better than `projectName`, as it allows having different json
-  # files for a given source file compiled with different options; it also
-  # works out of the box with `hashMainCompilationParams`.
-  result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json")
-
-const cacheVersion = "D20210525T193831" # update when `BuildCache` spec changes
-type BuildCache = object
-  cacheVersion: string
-  outputFile: string
-  compile: seq[(string, string)]
-  link: seq[string]
-  linkcmd: string
-  extraCmds: seq[string]
-  configFiles: seq[string] # the hash shouldn't be needed
-  stdinInput: bool
-  projectIsCmd: bool
-  cmdInput: string
-  currentDir: string
-  cmdline: string
-  depfiles: seq[(string, string)]
-  nimexe: string
-
 proc writeJsonBuildInstructions*(conf: ConfigRef) =
-  var linkFiles = collect(for it in conf.externalToLink:
-    var it = it
-    if conf.noAbsolutePaths: it = it.extractFilename
-    it.addFileExt(CC[conf.cCompiler].objExt))
-  for it in conf.toCompile: linkFiles.add it.obj.string
-  var bcache = BuildCache(
-    cacheVersion: cacheVersion,
-    outputFile: conf.absOutFile.string,
-    compile: collect(for i, it in conf.toCompile:
-      if CfileFlag.Cached notin it.flags: (it.cname.string, getCompileCFileCmd(conf, it))),
-    link: linkFiles,
-    linkcmd: getLinkCmd(conf, conf.absOutFile, linkFiles.quoteShellCommand),
-    extraCmds: getExtraCmds(conf, conf.absOutFile),
-    stdinInput: conf.projectIsStdin,
-    projectIsCmd: conf.projectIsCmd,
-    cmdInput: conf.cmdInput,
-    configFiles: conf.configFiles.mapIt(it.string),
-    currentDir: getCurrentDir())
-  if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"):
-    bcache.cmdline = conf.commandLine
-    bcache.depfiles = collect(for it in conf.m.fileInfos:
+  template lit(x: untyped) = f.write x
+  template str(x: untyped) =
+    when compiles(escapeJson(x, buf)):
+      buf.setLen 0
+      escapeJson(x, buf)
+      f.write buf
+    else:
+      f.write escapeJson(x)
+
+  proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) =
+    var comma = false
+    for i, it in clist:
+      if CfileFlag.Cached in it.flags: continue
+      let compileCmd = getCompileCFileCmd(conf, it)
+      if comma: lit ",\L" else: comma = true
+      lit "["
+      str it.cname.string
+      lit ", "
+      str compileCmd
+      lit "]"
+
+  proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList;
+                 llist: seq[string]) =
+    var pastStart = false
+    for it in llist:
+      let objfile = if noAbsolutePaths(conf): it.extractFilename
+                    else: it
+      let objstr = addFileExt(objfile, CC[conf.cCompiler].objExt)
+      objfiles.add(' ')
+      objfiles.add(objstr)
+      if pastStart: lit ",\L"
+      str objstr
+      pastStart = true
+
+    for it in clist:
+      let objstr = quoteShell(it.obj)
+      objfiles.add(' ')
+      objfiles.add(objstr)
+      if pastStart: lit ",\L"
+      str objstr
+      pastStart = true
+    lit "\L"
+
+  proc depfiles(conf: ConfigRef; f: File) =
+    var i = 0
+    for it in conf.m.fileInfos:
       let path = it.fullPath.string
       if isAbsolute(path): # TODO: else?
-        (path, $secureHashFile(path)))
-    bcache.nimexe = hashNimExe()
-  conf.jsonBuildFile = conf.jsonBuildInstructionsFile
-  conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty)
+        if i > 0: lit "],\L"
+        lit "["
+        str path
+        lit ", "
+        str $secureHashFile(path)
+        inc i
+    lit "]\L"
 
-proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool =
-  if not fileExists(jsonFile) or not fileExists(conf.absOutFile): return true
-  var bcache: BuildCache
-  try: bcache.fromJson(jsonFile.string.parseFile)
+
+  var buf = newStringOfCap(50)
+
+  let jsonFile = conf.getNimcacheDir / RelativeFile(conf.projectName & ".json")
+  conf.jsonBuildFile = jsonFile
+  let output = conf.absOutFile
+
+  var f: File
+  if open(f, jsonFile.string, fmWrite):
+    lit "{\L"
+    lit "\"outputFile\": "
+    str $output
+
+    lit ",\L\"compile\":[\L"
+    cfiles(conf, f, buf, conf.toCompile, false)
+    lit "],\L\"link\":[\L"
+    var objfiles = ""
+    # XXX add every file here that is to link
+    linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink)
+
+    lit "],\L\"linkcmd\": "
+    str getLinkCmd(conf, output, objfiles)
+
+    lit ",\L\"extraCmds\": "
+    lit $(%* getExtraCmds(conf, conf.absOutFile))
+
+    lit ",\L\"stdinInput\": "
+    lit $(%* conf.projectIsStdin)
+
+    if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"):
+      lit ",\L\"cmdline\": "
+      str conf.commandLine
+      lit ",\L\"depfiles\":[\L"
+      depfiles(conf, f)
+      lit "],\L\"nimexe\": \L"
+      str hashNimExe()
+      lit "\L"
+
+    lit "\L}\L"
+    close(f)
+
+proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile): bool =
+  let jsonFile = toGeneratedFile(conf, projectfile, "json")
+  if not fileExists(jsonFile): return true
+  if not fileExists(conf.absOutFile): return true
+  result = false
+  try:
+    let data = json.parseFile(jsonFile.string)
+    if not data.hasKey("depfiles") or not data.hasKey("cmdline"):
+      return true
+    let oldCmdLine = data["cmdline"].getStr
+    if conf.commandLine != oldCmdLine:
+      return true
+    if hashNimExe() != data["nimexe"].getStr:
+      return true
+    if not data.hasKey("stdinInput"): return true
+    let stdinInput = data["stdinInput"].getBool
+    if conf.projectIsStdin or stdinInput:
+      # could optimize by returning false if stdin input was the same,
+      # but I'm not sure how to get full stding input
+      return true
+
+    let depfilesPairs = data["depfiles"]
+    doAssert depfilesPairs.kind == JArray
+    for p in depfilesPairs:
+      doAssert p.kind == JArray
+      # >= 2 for forwards compatibility with potential later .json files:
+      doAssert p.len >= 2
+      let depFilename = p[0].getStr
+      let oldHashValue = p[1].getStr
+      let newHashValue = $secureHashFile(depFilename)
+      if oldHashValue != newHashValue:
+        return true
   except IOError, OSError, ValueError:
-    stderr.write "Warning: JSON processing failed for: $#\n" % jsonFile.string
-    return true
-  if bcache.currentDir != getCurrentDir() or # fixes bug #16271
-     bcache.configFiles != conf.configFiles.mapIt(it.string) or
-     bcache.cacheVersion != cacheVersion or bcache.outputFile != conf.absOutFile.string or
-     bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or
-     bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true
-  if bcache.stdinInput or conf.projectIsStdin: return true
-    # xxx optimize by returning false if stdin input was the same
-  for (file, hash) in bcache.depfiles:
-    if $secureHashFile(file) != hash: return true
+    echo "Warning: JSON processing failed: ", getCurrentExceptionMsg()
+    result = true
+
+proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) =
+  let jsonFile = toGeneratedFile(conf, projectfile, "json")
+  try:
+    let data = json.parseFile(jsonFile.string)
+
+    let output = data["outputFile"].getStr
+    createDir output.parentDir
+    let outputCurrent = $conf.absOutFile
+    if output != outputCurrent:
+      # previously, any specified output file would be silently ignored;
+      # simply copying won't work in some cases, for example with `extraCmds`,
+      # so we just make it an error, user should use same command for jsonscript
+      # as was used with --compileOnly.
+      globalError(conf, gCmdLineInfo, "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % [outputCurrent, output, jsonFile.string])
+
+    let toCompile = data["compile"]
+    doAssert toCompile.kind == JArray
+    var cmds: TStringSeq
+    var prettyCmds: TStringSeq
+    let prettyCb = proc (idx: int) =
+      if prettyCmds[idx].len > 0: echo prettyCmds[idx]
+
+    for c in toCompile:
+      doAssert c.kind == JArray
+      doAssert c.len >= 2
+
+      cmds.add(c[1].getStr)
+      prettyCmds.add displayProgressCC(conf, c[0].getStr, c[1].getStr)
+
+    execCmdsInParallel(conf, cmds, prettyCb)
+
+    let linkCmd = data["linkcmd"]
+    doAssert linkCmd.kind == JString
+    execLinkCmd(conf, linkCmd.getStr)
+    if data.hasKey("extraCmds"):
+      let extraCmds = data["extraCmds"]
+      doAssert extraCmds.kind == JArray
+      for cmd in extraCmds:
+        doAssert cmd.kind == JString, $cmd.kind
+        let cmd2 = cmd.getStr
+        execExternalProgram(conf, cmd2, hintExecuting)
 
-proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) =
-  var bcache: BuildCache
-  try: bcache.fromJson(jsonFile.string.parseFile)
   except:
     let e = getCurrentException()
-    conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" %
-      [e.msg, e.getStackTrace(), jsonFile.string]
-  let output = bcache.outputFile
-  createDir output.parentDir
-  let outputCurrent = $conf.absOutFile
-  if output != outputCurrent or bcache.cacheVersion != cacheVersion:
-    globalError(conf, gCmdLineInfo,
-      "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " %
-      [outputCurrent, output, jsonFile.string])
-  var cmds, prettyCmds: TStringSeq
-  let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx])
-  for (name, cmd) in bcache.compile:
-    cmds.add cmd
-    prettyCmds.add displayProgressCC(conf, name, cmd)
-  execCmdsInParallel(conf, cmds, prettyCb)
-  execLinkCmd(conf, bcache.linkcmd)
-  for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting)
+    quit "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() &
+         "error evaluating JSON file: " & jsonFile.string
 
 proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope =
   for it in list:

compiler/filter_tmpl.nim

@@ -22,7 +22,7 @@ type
     info: TLineInfo
     indent, emitPar: int
     x: string                # the current input line
-    outp: PLLStream          # the output will be parsed by parser
+    outp: PLLStream          # the output will be parsed by pnimsyn
     subsChar, nimDirective: char
     emit, conc, toStr: string
     curly, bracket, par: int

compiler/gorgeimpl.nim

@@ -12,9 +12,6 @@
 import msgs, std / sha1, os, osproc, streams, options,
   lineinfos, pathutils
 
-when defined(nimPreviewSlimSystem):
-  import std/syncio
-
 proc readOutput(p: Process): (string, int) =
   result[0] = ""
   var output = p.outputStream
@@ -49,11 +46,7 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
       if result[1] == 0:
         writeFile(filename, result[0])
     except IOError, OSError:
-      if not readSuccessful:
-        when defined(nimLegacyGorgeErrors):
-          result = ("", -1)
-        else:
-          result = ("Error running startProcess: " & getCurrentExceptionMsg(), -1)
+      if not readSuccessful: result = ("", -1)
   else:
     try:
       var p = startProcess(cmd, workingDir,
@@ -64,7 +57,4 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
       result = p.readOutput
       p.close()
     except IOError, OSError:
-      when defined(nimLegacyGorgeErrors):
-        result = ("", -1)
-      else:
-        result = ("Error running startProcess: " & getCurrentExceptionMsg(), -1)
+      result = ("", -1)

compiler/guards.nim

@@ -12,9 +12,6 @@
 import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents,
   saturate, modulegraphs, options, lineinfos, int128
 
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
 const
   someEq = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
     mEqStr, mEqSet, mEqCString}
@@ -68,12 +65,9 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
     case n.kind
     of nkDotExpr, nkCheckedFieldExpr, nkObjUpConv, nkObjDownConv:
       n = n[0]
-    of nkDerefExpr:
+    of nkDerefExpr, nkHiddenDeref:
       n = n[0]
       inc derefs
-    of nkHiddenDeref:
-      n = n[0]
-      if not isApprox: inc derefs
     of nkBracketExpr:
       if isConstExpr(n[1]) or isLet(n[1]) or isConstExpr(n[1].skipConv):
         n = n[0]
@@ -88,6 +82,26 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
 
 proc interestingCaseExpr*(m: PNode): bool = isLetLocation(m, true)
 
+type
+  Operators* = object
+    opNot*, opContains*, opLe*, opLt*, opAnd*, opOr*, opIsNil*, opEq*: PSym
+    opAdd*, opSub*, opMul*, opDiv*, opLen*: PSym
+
+proc initOperators*(g: ModuleGraph): Operators =
+  result.opLe = createMagic(g, "<=", mLeI)
+  result.opLt = createMagic(g, "<", mLtI)
+  result.opAnd = createMagic(g, "and", mAnd)
+  result.opOr = createMagic(g, "or", mOr)
+  result.opIsNil = createMagic(g, "isnil", mIsNil)
+  result.opEq = createMagic(g, "==", mEqI)
+  result.opAdd = createMagic(g, "+", mAddI)
+  result.opSub = createMagic(g, "-", mSubI)
+  result.opMul = createMagic(g, "*", mMulI)
+  result.opDiv = createMagic(g, "div", mDivI)
+  result.opLen = createMagic(g, "len", mLengthSeq)
+  result.opNot = createMagic(g, "not", mNot)
+  result.opContains = createMagic(g, "contains", mInSet)
+
 proc swapArgs(fact: PNode, newOp: PSym): PNode =
   result = newNodeI(nkCall, fact.info, 3)
   result[0] = newSymNode(newOp)
@@ -203,7 +217,7 @@ proc highBound*(conf: ConfigRef; x: PNode; o: Operators): PNode =
              nkIntLit.newIntNode(lastOrd(conf, typ))
            elif typ.kind == tySequence and x.kind == nkSym and
                x.sym.kind == skConst:
-             nkIntLit.newIntNode(x.sym.astdef.len-1)
+             nkIntLit.newIntNode(x.sym.ast.len-1)
            else:
              o.opAdd.buildCall(o.opLen.buildCall(x), minusOne())
   result.info = x.info
@@ -390,16 +404,16 @@ proc usefulFact(n: PNode; o: Operators): PNode =
 type
   TModel* = object
     s*: seq[PNode] # the "knowledge base"
-    g*: ModuleGraph
+    o*: Operators
     beSmart*: bool
 
 proc addFact*(m: var TModel, nn: PNode) =
-  let n = usefulFact(nn, m.g.operators)
+  let n = usefulFact(nn, m.o)
   if n != nil:
     if not m.beSmart:
       m.s.add n
     else:
-      let c = canon(n, m.g.operators)
+      let c = canon(n, m.o)
       if c.getMagic == mAnd:
         addFact(m, c[1])
         addFact(m, c[2])
@@ -407,7 +421,7 @@ proc addFact*(m: var TModel, nn: PNode) =
         m.s.add c
 
 proc addFactNeg*(m: var TModel, n: PNode) =
-  let n = n.neg(m.g.operators)
+  let n = n.neg(m.o)
   if n != nil: addFact(m, n)
 
 proc sameOpr(a, b: PSym): bool =
@@ -447,14 +461,8 @@ proc sameTree*(a, b: PNode): bool =
 proc hasSubTree(n, x: PNode): bool =
   if n.sameTree(x): result = true
   else:
-    case n.kind
-    of nkEmpty..nkNilLit:
-      result = n.sameTree(x)
-    of nkFormalParams:
-      discard
-    else:
-      for i in 0..<n.len:
-        if hasSubTree(n[i], x): return true
+    for i in 0..n.safeLen-1:
+      if hasSubTree(n[i], x): return true
 
 proc invalidateFacts*(s: var seq[PNode], n: PNode) =
   # We are able to guard local vars (as opposed to 'let' variables)!
@@ -732,7 +740,7 @@ proc doesImply*(facts: TModel, prop: PNode): TImplication =
       if result != impUnknown: return
 
 proc impliesNotNil*(m: TModel, arg: PNode): TImplication =
-  result = doesImply(m, m.g.operators.opIsNil.buildCall(arg).neg(m.g.operators))
+  result = doesImply(m, m.o.opIsNil.buildCall(arg).neg(m.o))
 
 proc simpleSlice*(a, b: PNode): BiggestInt =
   # returns 'c' if a..b matches (i+c)..(i+c), -1 otherwise. (i)..(i) is matched
@@ -786,7 +794,12 @@ macro `=~`(x: PNode, pat: untyped): bool =
 
   var conds = newTree(nnkBracket)
   m(x, pat, conds)
-  result = nestList(ident"and", conds)
+  when compiles(nestList(ident"and", conds)):
+    result = nestList(ident"and", conds)
+  #elif declared(macros.toNimIdent):
+  #  result = nestList(toNimIdent"and", conds)
+  else:
+    result = nestList(!"and", conds)
 
 proc isMinusOne(n: PNode): bool =
   n.kind in {nkCharLit..nkUInt64Lit} and n.intVal == -1
@@ -820,7 +833,7 @@ proc ple(m: TModel; a, b: PNode): TImplication =
   if b.getMagic in someAdd:
     if zero() <=? b[2] and a <=? b[1]: return impYes
     # x <= y-c  if x+c <= y
-    if b[2] <=? zero() and (canon(m.g.operators.opSub.buildCall(a, b[2]), m.g.operators) <=? b[1]):
+    if b[2] <=? zero() and (canon(m.o.opSub.buildCall(a, b[2]), m.o) <=? b[1]):
       return impYes
 
   #   x+c <= y  if c <= 0 and x <= y
@@ -834,20 +847,20 @@ proc ple(m: TModel; a, b: PNode): TImplication =
   if a.getMagic in someMul and a[2].isValue and a[1].getMagic in someDiv and
       a[1][2].isValue:
     # simplify   (x div 4) * 2 <= y   to  x div (c div d)  <= y
-    if ple(m, buildCall(m.g.operators.opDiv, a[1][1], `|div|`(a[1][2], a[2])), b) == impYes:
+    if ple(m, buildCall(m.o.opDiv, a[1][1], `|div|`(a[1][2], a[2])), b) == impYes:
       return impYes
 
   # x*3 + x == x*4. It follows that:
   # x*3 + y <= x*4  if  y <= x  and 3 <= 4
   if a =~ x*dc + y and b =~ x2*ec:
     if sameTree(x, x2):
-      let ec1 = m.g.operators.opAdd.buildCall(ec, minusOne())
+      let ec1 = m.o.opAdd.buildCall(ec, minusOne())
       if x >=? 1 and ec >=? 1 and dc >=? 1 and dc <=? ec1 and y <=? x:
         return impYes
   elif a =~ x*dc and b =~ x2*ec + y:
     #echo "BUG cam ehrer e ", a, " <=? ", b
     if sameTree(x, x2):
-      let ec1 = m.g.operators.opAdd.buildCall(ec, minusOne())
+      let ec1 = m.o.opAdd.buildCall(ec, minusOne())
       if x >=? 1 and ec >=? 1 and dc >=? 1 and dc <=? ec1 and y <=? zero():
         return impYes
 
@@ -950,12 +963,12 @@ proc pleViaModel(model: TModel; aa, bb: PNode): TImplication =
   var b = bb
   if replacements.len > 0:
     m.s = @[]
-    m.g = model.g
+    m.o = model.o
     # make the other facts consistent:
     for fact in model.s:
       if fact != nil and fact.getMagic notin someEq:
         # XXX 'canon' should not be necessary here, but it is
-        m.s.add applyReplacements(fact, replacements).canon(m.g.operators)
+        m.s.add applyReplacements(fact, replacements).canon(m.o)
     a = applyReplacements(aa, replacements)
     b = applyReplacements(bb, replacements)
   else:
@@ -964,25 +977,24 @@ proc pleViaModel(model: TModel; aa, bb: PNode): TImplication =
   result = pleViaModelRec(m, a, b)
 
 proc proveLe*(m: TModel; a, b: PNode): TImplication =
-  let x = canon(m.g.operators.opLe.buildCall(a, b), m.g.operators)
+  let x = canon(m.o.opLe.buildCall(a, b), m.o)
   #echo "ROOT ", renderTree(x[1]), " <=? ", renderTree(x[2])
   result = ple(m, x[1], x[2])
   if result == impUnknown:
     # try an alternative:  a <= b  iff  not (b < a)  iff  not (b+1 <= a):
-    let y = canon(m.g.operators.opLe.buildCall(m.g.operators.opAdd.buildCall(b, one()), a), m.g.operators)
+    let y = canon(m.o.opLe.buildCall(m.o.opAdd.buildCall(b, one()), a), m.o)
     result = ~ple(m, y[1], y[2])
 
 proc addFactLe*(m: var TModel; a, b: PNode) =
-  m.s.add canon(m.g.operators.opLe.buildCall(a, b), m.g.operators)
+  m.s.add canon(m.o.opLe.buildCall(a, b), m.o)
 
 proc addFactLt*(m: var TModel; a, b: PNode) =
-  let bb = m.g.operators.opAdd.buildCall(b, minusOne())
+  let bb = m.o.opAdd.buildCall(b, minusOne())
   addFactLe(m, a, bb)
 
 proc settype(n: PNode): PType =
-  result = newType(tySet, ItemId(module: -1, item: -1), n.typ.owner)
-  var idgen: IdGenerator
-  addSonSkipIntLit(result, n.typ, idgen)
+  result = newType(tySet, n.typ.owner)
+  addSonSkipIntLit(result, n.typ)
 
 proc buildOf(it, loc: PNode; o: Operators): PNode =
   var s = newNodeI(nkCurly, it.info, it.len-1)
@@ -1008,14 +1020,14 @@ proc buildElse(n: PNode; o: Operators): PNode =
 
 proc addDiscriminantFact*(m: var TModel, n: PNode) =
   var fact = newNodeI(nkCall, n.info, 3)
-  fact[0] = newSymNode(m.g.operators.opEq)
+  fact[0] = newSymNode(m.o.opEq)
   fact[1] = n[0]
   fact[2] = n[1]
   m.s.add fact
 
 proc addAsgnFact*(m: var TModel, key, value: PNode) =
   var fact = newNodeI(nkCall, key.info, 3)
-  fact[0] = newSymNode(m.g.operators.opEq)
+  fact[0] = newSymNode(m.o.opEq)
   fact[1] = key
   fact[2] = value
   m.s.add fact
@@ -1031,7 +1043,7 @@ proc sameSubexprs*(m: TModel; a, b: PNode): bool =
   # However, nil checking requires exactly the same mechanism! But for now
   # we simply use sameTree and live with the unsoundness of the analysis.
   var check = newNodeI(nkCall, a.info, 3)
-  check[0] = newSymNode(m.g.operators.opEq)
+  check[0] = newSymNode(m.o.opEq)
   check[1] = a
   check[2] = b
   result = m.doesImply(check) == impYes
@@ -1039,9 +1051,9 @@ proc sameSubexprs*(m: TModel; a, b: PNode): bool =
 proc addCaseBranchFacts*(m: var TModel, n: PNode, i: int) =
   let branch = n[i]
   if branch.kind == nkOfBranch:
-    m.s.add buildOf(branch, n[0], m.g.operators)
+    m.s.add buildOf(branch, n[0], m.o)
   else:
-    m.s.add n.buildElse(m.g.operators).neg(m.g.operators)
+    m.s.add n.buildElse(m.o).neg(m.o)
 
 proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
   if check[1].kind == nkCurly:
@@ -1057,12 +1069,8 @@ proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
     assert check.getMagic == mNot
     result = buildProperFieldCheck(access, check[1], o).neg(o)
 
-proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef; produceError: bool) =
+proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef) =
   for i in 1..<n.len:
-    let check = buildProperFieldCheck(n[0], n[i], m.g.operators)
+    let check = buildProperFieldCheck(n[0], n[i], m.o)
     if check != nil and m.doesImply(check) != impYes:
-      if produceError:
-        localError(conf, n.info, "field access outside of valid case branch: " & renderTree(n[0]))
-      else:
-        message(conf, n.info, warnProveField, renderTree(n[0]))
-      break
+      message(conf, n.info, warnProveField, renderTree(n[0])); break

compiler/hlo.nim

@@ -8,10 +8,6 @@
 #
 
 # This include implements the high level optimization pass.
-# included from sem.nim
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
 
 proc hlo(c: PContext, n: PNode): PNode
 
@@ -71,7 +67,7 @@ proc hlo(c: PContext, n: PNode): PNode =
     # already processed (special cases in semstmts.nim)
     result = n
   else:
-    if n.kind in {nkFastAsgn, nkAsgn, nkSinkAsgn, nkIdentDefs, nkVarTuple} and
+    if n.kind in {nkFastAsgn, nkAsgn, nkIdentDefs, nkVarTuple} and
         n[0].kind == nkSym and
         {sfGlobal, sfPure} * n[0].sym.flags == {sfGlobal, sfPure}:
       # do not optimize 'var g {.global} = re(...)' again!
@@ -90,9 +86,9 @@ proc hlo(c: PContext, n: PNode): PNode =
       else:
         result = fitNode(c, n.typ, result, n.info)
       # optimization has been applied so check again:
-      result = commonOptimizations(c.graph, c.idgen, c.module, result)
+      result = commonOptimizations(c.graph, c.module, result)
       result = hlo(c, result)
-      result = commonOptimizations(c.graph, c.idgen, c.module, result)
+      result = commonOptimizations(c.graph, c.module, result)
 
 proc hloBody(c: PContext, n: PNode): PNode =
   # fast exit:

compiler/ic/bitabs.nim

@@ -1,171 +0,0 @@
-## A BiTable is a table that can be seen as an optimized pair
-## of `(Table[LitId, Val], Table[Val, LitId])`.
-
-import hashes, rodfiles
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
-type
-  LitId* = distinct uint32
-
-  BiTable*[T] = object
-    vals: seq[T] # indexed by LitId
-    keys: seq[LitId]  # indexed by hash(val)
-
-proc nextTry(h, maxHash: Hash): Hash {.inline.} =
-  result = (h + 1) and maxHash
-
-template maxHash(t): untyped = high(t.keys)
-template isFilled(x: LitId): bool = x.uint32 > 0'u32
-
-proc `$`*(x: LitId): string {.borrow.}
-proc `<`*(x, y: LitId): bool {.borrow.}
-proc `<=`*(x, y: LitId): bool {.borrow.}
-proc `==`*(x, y: LitId): bool {.borrow.}
-proc hash*(x: LitId): Hash {.borrow.}
-
-
-proc len*[T](t: BiTable[T]): int = t.vals.len
-
-proc mustRehash(length, counter: int): bool {.inline.} =
-  assert(length > counter)
-  result = (length * 2 < counter * 3) or (length - counter < 4)
-
-const
-  idStart = 256 ##
-  ## Ids do not start with 0 but with this value. The IR needs it.
-  ## TODO: explain why
-
-template idToIdx(x: LitId): int = x.int - idStart
-
-proc hasLitId*[T](t: BiTable[T]; x: LitId): bool =
-  let idx = idToIdx(x)
-  result = idx >= 0 and idx < t.vals.len
-
-proc enlarge[T](t: var BiTable[T]) =
-  var n: seq[LitId]
-  newSeq(n, len(t.keys) * 2)
-  swap(t.keys, n)
-  for i in 0..high(n):
-    let eh = n[i]
-    if isFilled(eh):
-      var j = hash(t.vals[idToIdx eh]) and maxHash(t)
-      while isFilled(t.keys[j]):
-        j = nextTry(j, maxHash(t))
-      t.keys[j] = move n[i]
-
-proc getKeyId*[T](t: BiTable[T]; v: T): LitId =
-  let origH = hash(v)
-  var h = origH and maxHash(t)
-  if t.keys.len != 0:
-    while true:
-      let litId = t.keys[h]
-      if not isFilled(litId): break
-      if t.vals[idToIdx t.keys[h]] == v: return litId
-      h = nextTry(h, maxHash(t))
-  return LitId(0)
-
-proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId =
-  let origH = hash(v)
-  var h = origH and maxHash(t)
-  if t.keys.len != 0:
-    while true:
-      let litId = t.keys[h]
-      if not isFilled(litId): break
-      if t.vals[idToIdx t.keys[h]] == v: return litId
-      h = nextTry(h, maxHash(t))
-    # not found, we need to insert it:
-    if mustRehash(t.keys.len, t.vals.len):
-      enlarge(t)
-      # recompute where to insert:
-      h = origH and maxHash(t)
-      while true:
-        let litId = t.keys[h]
-        if not isFilled(litId): break
-        h = nextTry(h, maxHash(t))
-  else:
-    setLen(t.keys, 16)
-    h = origH and maxHash(t)
-
-  result = LitId(t.vals.len + idStart)
-  t.keys[h] = result
-  t.vals.add v
-
-
-proc `[]`*[T](t: var BiTable[T]; litId: LitId): var T {.inline.} =
-  let idx = idToIdx litId
-  assert idx < t.vals.len
-  result = t.vals[idx]
-
-proc `[]`*[T](t: BiTable[T]; litId: LitId): lent T {.inline.} =
-  let idx = idToIdx litId
-  assert idx < t.vals.len
-  result = t.vals[idx]
-
-proc hash*[T](t: BiTable[T]): Hash =
-  ## as the keys are hashes of the values, we simply use them instead
-  var h: Hash = 0
-  for i, n in pairs t.keys:
-    h = h !& hash((i, n))
-  result = !$h
-
-proc store*[T](f: var RodFile; t: BiTable[T]) =
-  storeSeq(f, t.vals)
-  storeSeq(f, t.keys)
-
-proc load*[T](f: var RodFile; t: var BiTable[T]) =
-  loadSeq(f, t.vals)
-  loadSeq(f, t.keys)
-
-when isMainModule:
-
-  var t: BiTable[string]
-
-  echo getOrIncl(t, "hello")
-
-  echo getOrIncl(t, "hello")
-  echo getOrIncl(t, "hello3")
-  echo getOrIncl(t, "hello4")
-  echo getOrIncl(t, "helloasfasdfdsa")
-  echo getOrIncl(t, "hello")
-  echo getKeyId(t, "hello")
-  echo getKeyId(t, "none")
-
-  for i in 0 ..< 100_000:
-    discard t.getOrIncl($i & "___" & $i)
-
-  for i in 0 ..< 100_000:
-    assert t.getOrIncl($i & "___" & $i).idToIdx == i + 4
-  echo "begin"
-  echo t.vals.len
-
-  echo t.vals[0]
-  echo t.vals[1004]
-
-  echo "middle"
-
-  var tf: BiTable[float]
-
-  discard tf.getOrIncl(0.4)
-  discard tf.getOrIncl(16.4)
-  discard tf.getOrIncl(32.4)
-  echo getKeyId(tf, 32.4)
-
-  var f2 = open("testblah.bin", fmWrite)
-  echo store(f2, tf)
-  f2.close
-
-  var f1 = open("testblah.bin", fmRead)
-
-  var t2: BiTable[float]
-
-  echo f1.load(t2)
-  echo t2.vals.len
-
-  echo getKeyId(t2, 32.4)
-
-  echo "end"
-
-
-  f1.close

compiler/ic/cbackend.nim

@@ -1,178 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2021 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## New entry point into our C/C++ code generator. Ideally
-## somebody would rewrite the old backend (which is 8000 lines of crufty Nim code)
-## to work on packed trees directly and produce the C code as an AST which can
-## then be rendered to text in a very simple manner. Unfortunately nobody wrote
-## this code. So instead we wrap the existing cgen.nim and its friends so that
-## we call directly into the existing code generation logic but avoiding the
-## naive, outdated `passes` design. Thus you will see some
-## `useAliveDataFromDce in flags` checks in the old code -- the old code is
-## also doing cross-module dependency tracking and DCE that we don't need
-## anymore. DCE is now done as prepass over the entire packed module graph.
-
-import std/packedsets, algorithm, tables
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
-import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen,
-  pathutils, extccomp, msgs, modulepaths]
-
-import packed_ast, ic, dce, rodfiles
-
-proc unpackTree(g: ModuleGraph; thisModule: int;
-                tree: PackedTree; n: NodePos): PNode =
-  var decoder = initPackedDecoder(g.config, g.cache)
-  result = loadNodes(decoder, g.packed, thisModule, tree, n)
-
-proc setupBackendModule(g: ModuleGraph; m: var LoadedModule) =
-  if g.backend == nil:
-    g.backend = cgendata.newModuleList(g)
-  assert g.backend != nil
-  var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config)
-  bmod.idgen = idgenFromLoadedModule(m)
-
-proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) =
-  var bmod = BModuleList(g.backend).modules[m.module.position]
-  assert bmod != nil
-  bmod.flags.incl useAliveDataFromDce
-  bmod.alive = move alive[m.module.position]
-
-  for p in allNodes(m.fromDisk.topLevel):
-    let n = unpackTree(g, m.module.position, m.fromDisk.topLevel, p)
-    cgen.genTopLevelStmt(bmod, n)
-
-  finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info))
-  m.fromDisk.backendFlags = cgen.whichInitProcs(bmod)
-
-proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) =
-  for x in mitems(m.fromDisk.emittedTypeInfo):
-    #echo "found type ", x, " for file ", int(origin)
-    g.emittedTypeInfo[x] = origin
-
-proc addFileToLink(config: ConfigRef; m: PSym) =
-  let filename = AbsoluteFile toFullPath(config, m.position.FileIndex)
-  let ext =
-      if config.backend == backendCpp: ".nim.cpp"
-      elif config.backend == backendObjc: ".nim.m"
-      else: ".nim.c"
-  let cfile = changeFileExt(completeCfilePath(config,
-                            mangleModuleName(config, filename).AbsoluteFile), ext)
-  let objFile = completeCfilePath(config, toObjFile(config, cfile))
-  if fileExists(objFile):
-    var cf = Cfile(nimname: m.name.s, cname: cfile,
-                   obj: objFile,
-                   flags: {CfileFlag.Cached})
-    addFileToCompile(config, cf)
-
-when defined(debugDce):
-  import os, std/packedsets
-
-proc storeAliveSymsImpl(asymFile: AbsoluteFile; s: seq[int32]) =
-  var f = rodfiles.create(asymFile.string)
-  f.storeHeader()
-  f.storeSection aliveSymsSection
-  f.storeSeq(s)
-  close f
-
-template prepare {.dirty.} =
-  let asymFile = toRodFile(config, AbsoluteFile toFullPath(config, position.FileIndex), ".alivesyms")
-  var s = newSeqOfCap[int32](alive[position].len)
-  for a in items(alive[position]): s.add int32(a)
-  sort(s)
-
-proc storeAliveSyms(config: ConfigRef; position: int; alive: AliveSyms) =
-  prepare()
-  storeAliveSymsImpl(asymFile, s)
-
-proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool =
-  prepare()
-  var f2 = rodfiles.open(asymFile.string)
-  f2.loadHeader()
-  f2.loadSection aliveSymsSection
-  var oldData: seq[int32]
-  f2.loadSeq(oldData)
-  f2.close
-  if f2.err == ok and oldData == s:
-    result = false
-  else:
-    when defined(debugDce):
-      let oldAsSet = toPackedSet[int32](oldData)
-      let newAsSet = toPackedSet[int32](s)
-      echo "set of live symbols changed ", asymFile.changeFileExt("rod"), " ", position, " ", f2.err
-      echo "in old but not in new ", oldAsSet.difference(newAsSet), " number of entries in old ", oldAsSet.len
-      echo "in new but not in old ", newAsSet.difference(oldAsSet), " number of entries in new ", newAsSet.len
-      #if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0:
-      #  echo "command failed"
-    result = true
-    storeAliveSymsImpl(asymFile, s)
-
-proc genPackedModule(g: ModuleGraph, i: int; alive: var AliveSyms) =
-  # case statement here to enforce exhaustive checks.
-  case g.packed[i].status
-  of undefined:
-    discard "nothing to do"
-  of loading, stored:
-    assert false
-  of storing, outdated:
-    storeAliveSyms(g.config, g.packed[i].module.position, alive)
-    generateCodeForModule(g, g.packed[i], alive)
-    closeRodFile(g, g.packed[i].module)
-  of loaded:
-    if g.packed[i].loadedButAliveSetChanged:
-      generateCodeForModule(g, g.packed[i], alive)
-    else:
-      addFileToLink(g.config, g.packed[i].module)
-      replayTypeInfo(g, g.packed[i], FileIndex(i))
-
-      if g.backend == nil:
-        g.backend = cgendata.newModuleList(g)
-      registerInitProcs(BModuleList(g.backend), g.packed[i].module, g.packed[i].fromDisk.backendFlags)
-
-proc generateCode*(g: ModuleGraph) =
-  ## The single entry point, generate C(++) code for the entire
-  ## Nim program aka `ModuleGraph`.
-  resetForBackend(g)
-  var alive = computeAliveSyms(g.packed, g.config)
-
-  when false:
-    for i in 0..high(g.packed):
-      echo i, " is of status ", g.packed[i].status, " ", toFullPath(g.config, FileIndex(i))
-
-  # First pass: Setup all the backend modules for all the modules that have
-  # changed:
-  for i in 0..high(g.packed):
-    # case statement here to enforce exhaustive checks.
-    case g.packed[i].status
-    of undefined:
-      discard "nothing to do"
-    of loading, stored:
-      assert false
-    of storing, outdated:
-      setupBackendModule(g, g.packed[i])
-    of loaded:
-      # Even though this module didn't change, DCE might trigger a change.
-      # Consider this case: Module A uses symbol S from B and B does not use
-      # S itself. A is then edited not to use S either. Thus we have to
-      # recompile B in order to remove S from the final result.
-      if aliveSymsChanged(g.config, g.packed[i].module.position, alive):
-        g.packed[i].loadedButAliveSetChanged = true
-        setupBackendModule(g, g.packed[i])
-
-  # Second pass: Code generation.
-  let mainModuleIdx = g.config.projectMainIdx2.int
-  # We need to generate the main module last, because only then
-  # all init procs have been registered:
-  for i in 0..high(g.packed):
-    if i != mainModuleIdx:
-      genPackedModule(g, i, alive)
-  if mainModuleIdx >= 0:
-    genPackedModule(g, mainModuleIdx, alive)

compiler/ic/dce.nim

@@ -1,165 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2021 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Dead code elimination (=DCE) for IC.
-
-import std/[intsets, tables]
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
-import ".." / [ast, options, lineinfos, types]
-
-import packed_ast, ic, bitabs
-
-type
-  AliveSyms* = seq[IntSet]
-  AliveContext* = object ## Purpose is to fill the 'alive' field.
-    stack: seq[(int, TOptions, NodePos)] ## A stack for marking symbols as alive.
-    decoder: PackedDecoder ## We need a PackedDecoder for module ID address translations.
-    thisModule: int  ## The module we're currently analysing for DCE.
-    alive: AliveSyms ## The final result of our computation.
-    options: TOptions
-    compilerProcs: Table[string, (int, int32)]
-
-proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): bool =
-  ## "Exported to C" procs are special (these are marked with '.exportc') because these
-  ## must not be optimized away!
-  let symPtr = unsafeAddr g[c.thisModule].fromDisk.syms[symId]
-  let flags = symPtr.flags
-  # due to a bug/limitation in the lambda lifting, unused inner procs
-  # are not transformed correctly; issue (#411). However, the whole purpose here
-  # is to eliminate unused procs. So there is no special logic required for this case.
-  if sfCompileTime notin flags:
-    if ({sfExportc, sfCompilerProc} * flags != {}) or
-        (symPtr.kind == skMethod):
-      result = true
-      # XXX: This used to be a condition to:
-      #  (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or
-    if sfCompilerProc in flags:
-      c.compilerProcs[g[c.thisModule].fromDisk.strings[symPtr.name]] = (c.thisModule, symId)
-
-template isNotGeneric(n: NodePos): bool = ithSon(tree, n, genericParamsPos).kind == nkEmpty
-
-proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: int32) =
-  ## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will
-  ## be analysed.
-  if not c.alive[module].containsOrIncl(item):
-    var body = g[module].fromDisk.syms[item].ast
-    if body != emptyNodeId:
-      let opt = g[module].fromDisk.syms[item].options
-      if g[module].fromDisk.syms[item].kind in routineKinds:
-        body = NodeId ithSon(g[module].fromDisk.bodies, NodePos body, bodyPos)
-      c.stack.add((module, opt, NodePos(body)))
-
-    when false:
-      let nid = g[module].fromDisk.syms[item].name
-      if nid != LitId(0):
-        let name = g[module].fromDisk.strings[nid]
-        if name in ["nimFrame", "callDepthLimitReached"]:
-          echo "I was called! ", name, " body exists: ", body != emptyNodeId, " ", module, " ", item
-
-proc requestCompilerProc(c: var AliveContext; g: PackedModuleGraph; name: string) =
-  let (module, item) = c.compilerProcs[name]
-  followLater(c, g, module, item)
-
-proc loadTypeKind(t: PackedItemId; c: AliveContext; g: PackedModuleGraph; toSkip: set[TTypeKind]): TTypeKind =
-  template kind(t: ItemId): TTypeKind = g[t.module].fromDisk.types[t.item].kind
-
-  var t2 = translateId(t, g, c.thisModule, c.decoder.config)
-  result = t2.kind
-  while result in toSkip:
-    t2 = translateId(g[t2.module].fromDisk.types[t2.item].types[^1], g, t2.module, c.decoder.config)
-    result = t2.kind
-
-proc rangeCheckAnalysis(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) =
-  ## Replicates the logic of `ccgexprs.genRangeChck`.
-  ## XXX Refactor so that the duplicated logic is avoided. However, for now it's not clear
-  ## the approach has enough merit.
-  var dest = loadTypeKind(n.typ, c, g, abstractVar)
-  if optRangeCheck notin c.options or dest in {tyUInt..tyUInt64}:
-    discard "no need to generate a check because it was disabled"
-  else:
-    let n0t = loadTypeKind(n.firstSon.typ, c, g, {})
-    if n0t in {tyUInt, tyUInt64}:
-      c.requestCompilerProc(g, "raiseRangeErrorNoArgs")
-    else:
-      let raiser =
-        case loadTypeKind(n.typ, c, g, abstractVarRange)
-        of tyUInt..tyUInt64, tyChar: "raiseRangeErrorU"
-        of tyFloat..tyFloat128: "raiseRangeErrorF"
-        else: "raiseRangeErrorI"
-      c.requestCompilerProc(g, raiser)
-
-proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) =
-  ## Marks the symbols we encounter when we traverse the AST at `tree[n]` as alive, unless
-  ## it is purely in a declarative context (type section etc.).
-  case n.kind
-  of nkNone..pred(nkSym), succ(nkSym)..nkNilLit:
-    discard "ignore non-sym atoms"
-  of nkSym:
-    # This symbol is alive and everything its body references.
-    followLater(c, g, c.thisModule, n.operand)
-  of nkModuleRef:
-    let (n1, n2) = sons2(tree, n)
-    assert n1.kind == nkInt32Lit
-    assert n2.kind == nkInt32Lit
-    let m = n1.litId
-    let item = n2.operand
-    let otherModule = toFileIndexCached(c.decoder, g, c.thisModule, m).int
-    followLater(c, g, otherModule, item)
-  of nkMacroDef, nkTemplateDef, nkTypeSection, nkTypeOfExpr,
-     nkCommentStmt, nkIncludeStmt,
-     nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
-     nkFromStmt, nkStaticStmt:
-    discard
-  of nkVarSection, nkLetSection, nkConstSection:
-    # XXX ignore the defining local variable name?
-    for son in sonsReadonly(tree, n):
-      aliveCode(c, g, tree, son)
-  of nkChckRangeF, nkChckRange64, nkChckRange:
-    rangeCheckAnalysis(c, g, tree, n)
-  of nkProcDef, nkConverterDef, nkMethodDef, nkFuncDef, nkIteratorDef:
-    if n.firstSon.kind == nkSym and isNotGeneric(n):
-      let item = n.firstSon.operand
-      if isExportedToC(c, g, item):
-        # This symbol is alive and everything its body references.
-        followLater(c, g, c.thisModule, item)
-  else:
-    for son in sonsReadonly(tree, n):
-      aliveCode(c, g, tree, son)
-
-proc followNow(c: var AliveContext; g: PackedModuleGraph) =
-  ## Mark all entries in the stack. Marking can add more entries
-  ## to the stack but eventually we have looked at every alive symbol.
-  while c.stack.len > 0:
-    let (modId, opt, ast) = c.stack.pop()
-    c.thisModule = modId
-    c.options = opt
-    aliveCode(c, g, g[modId].fromDisk.bodies, ast)
-
-proc computeAliveSyms*(g: PackedModuleGraph; conf: ConfigRef): AliveSyms =
-  ## Entry point for our DCE algorithm.
-  var c = AliveContext(stack: @[], decoder: PackedDecoder(config: conf),
-                       thisModule: -1, alive: newSeq[IntSet](g.len),
-                       options: conf.options)
-  for i in countdown(high(g), 0):
-    if g[i].status != undefined:
-      c.thisModule = i
-      for p in allNodes(g[i].fromDisk.topLevel):
-        aliveCode(c, g, g[i].fromDisk.topLevel, p)
-
-  followNow(c, g)
-  result = move(c.alive)
-
-proc isAlive*(a: AliveSyms; module: int, item: int32): bool =
-  ## Backends use this to query if a symbol is `alive` which means
-  ## we need to produce (C/C++/etc) code for it.
-  result = a[module].contains(item)
-

compiler/ic/design.rst

@@ -1,56 +0,0 @@
-====================================
-  Incremental Recompilations
-====================================
-
-We split the Nim compiler into a frontend and a backend.
-The frontend produces a set of `.rod` files. Every `.nim` module
-produces its own `.rod` file.
-
-- The IR must be a faithful representation of the AST in memory.
-- The backend can do its own caching but doesn't have to. In the
-  current implementation the backend also caches its results.
-
-Advantage of the "set of files" vs the previous global database:
-- By construction, we either read from the `.rod` file or from the
-  `.nim` file, there can be no inconsistency. There can also be no
-  partial updates.
-- No dependency to external packages (SQLite). SQLite simply is too
-  slow and the old way of serialization was too slow too. We use a
-  format designed for Nim and expect to base further tools on this
-  file format.
-
-References to external modules must be (moduleId, symId) pairs.
-The symbol IDs are module specific. This way no global ID increment
-mechanism needs to be implemented that we could get wrong. ModuleIds
-are rod-file specific too.
-
-
-
-Global state
-------------
-
-There is no global state.
-
-Rod File Format
----------------
-
-It's a simple binary file format. `rodfiles.nim` contains some details.
-
-
-Backend
--------
-
-Nim programmers have to come to enjoy whole-program dead code elimination,
-by default. Since this is a "whole program" optimization, it does break
-modularity. However, thanks to the packed AST representation we can perform
-this global analysis without having to unpack anything. This is basically
-a mark&sweep GC algorithm:
-
-- Start with the top level statements. Every symbol that is referenced
-  from a top level statement is not "dead" and needs to be compiled by
-  the backend.
-- Every symbol referenced from a referenced symbol also has to be
-  compiled.
-
-Caching logic: Only if the set of alive symbols is different from the
-last run, the module has to be regenerated.

compiler/ic/integrity.nim

@@ -1,153 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2021 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Integrity checking for a set of .rod files.
-## The set must cover a complete Nim project.
-
-import sets
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
-import ".." / [ast, modulegraphs]
-import packed_ast, bitabs, ic
-
-type
-  CheckedContext = object
-    g: ModuleGraph
-    thisModule: int32
-    checkedSyms: HashSet[ItemId]
-    checkedTypes: HashSet[ItemId]
-
-proc checkType(c: var CheckedContext; typeId: PackedItemId)
-proc checkForeignSym(c: var CheckedContext; symId: PackedItemId)
-proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos)
-
-proc checkTypeObj(c: var CheckedContext; typ: PackedType) =
-  for child in typ.types:
-    checkType(c, child)
-  if typ.n != emptyNodeId:
-    checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos typ.n)
-  if typ.sym != nilItemId:
-    checkForeignSym(c, typ.sym)
-  if typ.owner != nilItemId:
-    checkForeignSym(c, typ.owner)
-  checkType(c, typ.typeInst)
-
-proc checkType(c: var CheckedContext; typeId: PackedItemId) =
-  if typeId == nilItemId: return
-  let itemId = translateId(typeId, c.g.packed, c.thisModule, c.g.config)
-  if not c.checkedTypes.containsOrIncl(itemId):
-    let oldThisModule = c.thisModule
-    c.thisModule = itemId.module
-    checkTypeObj c, c.g.packed[itemId.module].fromDisk.types[itemId.item]
-    c.thisModule = oldThisModule
-
-proc checkSym(c: var CheckedContext; s: PackedSym) =
-  if s.name != LitId(0):
-    assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId s.name
-  checkType c, s.typ
-  if s.ast != emptyNodeId:
-    checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos s.ast)
-  if s.owner != nilItemId:
-    checkForeignSym(c, s.owner)
-
-proc checkLocalSym(c: var CheckedContext; item: int32) =
-  let itemId = ItemId(module: c.thisModule, item: item)
-  if not c.checkedSyms.containsOrIncl(itemId):
-    checkSym c, c.g.packed[c.thisModule].fromDisk.syms[item]
-
-proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) =
-  let itemId = translateId(symId, c.g.packed, c.thisModule, c.g.config)
-  if not c.checkedSyms.containsOrIncl(itemId):
-    let oldThisModule = c.thisModule
-    c.thisModule = itemId.module
-    checkSym c, c.g.packed[itemId.module].fromDisk.syms[itemId.item]
-    c.thisModule = oldThisModule
-
-proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) =
-  if tree[n.int].typeId != nilItemId:
-    checkType(c, tree[n.int].typeId)
-  case n.kind
-  of nkEmpty, nkNilLit, nkType, nkNilRodNode:
-    discard
-  of nkIdent:
-    assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId
-  of nkSym:
-    checkLocalSym(c, tree.nodes[n.int].operand)
-  of directIntLit:
-    discard
-  of externIntLit, nkFloatLit..nkFloat128Lit:
-    assert c.g.packed[c.thisModule].fromDisk.numbers.hasLitId n.litId
-  of nkStrLit..nkTripleStrLit:
-    assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId
-  of nkModuleRef:
-    let (n1, n2) = sons2(tree, n)
-    assert n1.kind == nkInt32Lit
-    assert n2.kind == nkInt32Lit
-    checkForeignSym(c, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand))
-  else:
-    for n0 in sonsReadonly(tree, n):
-      checkNode(c, tree, n0)
-
-proc checkTree(c: var CheckedContext; t: PackedTree) =
-  for p in allNodes(t): checkNode(c, t, p)
-
-proc checkLocalSymIds(c: var CheckedContext; m: PackedModule; symIds: seq[int32]) =
-  for symId in symIds:
-    assert symId >= 0 and symId < m.syms.len, $symId & " " & $m.syms.len
-
-proc checkModule(c: var CheckedContext; m: PackedModule) =
-  # We check that:
-  # - Every symbol references existing types and symbols.
-  # - Every tree node references existing types and symbols.
-  for i in 0..high(m.syms):
-    checkLocalSym c, int32(i)
-
-  checkTree c, m.toReplay
-  checkTree c, m.topLevel
-
-  for e in m.exports:
-    assert e[1] >= 0 and e[1] < m.syms.len
-    assert e[0] == m.syms[e[1]].name
-
-  for e in m.compilerProcs:
-    assert e[1] >= 0 and e[1] < m.syms.len
-    assert e[0] == m.syms[e[1]].name
-
-  checkLocalSymIds c, m, m.converters
-  checkLocalSymIds c, m, m.methods
-  checkLocalSymIds c, m, m.trmacros
-  checkLocalSymIds c, m, m.pureEnums
-  #[
-    To do: Check all these fields:
-
-    reexports*: seq[(LitId, PackedItemId)]
-    macroUsages*: seq[(PackedItemId, PackedLineInfo)]
-
-    typeInstCache*: seq[(PackedItemId, PackedItemId)]
-    procInstCache*: seq[PackedInstantiation]
-    attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)]
-    methodsPerType*: seq[(PackedItemId, int, PackedItemId)]
-    enumToStringProcs*: seq[(PackedItemId, PackedItemId)]
-  ]#
-
-proc checkIntegrity*(g: ModuleGraph) =
-  var c = CheckedContext(g: g)
-  for i in 0..high(g.packed):
-    # case statement here to enforce exhaustive checks.
-    case g.packed[i].status
-    of undefined:
-      discard "nothing to do"
-    of loading:
-      assert false, "cannot check integrity: Module still loading"
-    of stored, storing, outdated, loaded:
-      c.thisModule = int32 i
-      checkModule(c, g.packed[i].fromDisk)
-

compiler/ic/navigator.nim

@@ -1,170 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2021 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Supports the "nim check --ic:on --defusages:FILE,LINE,COL"
-## IDE-like features. It uses the set of .rod files to accomplish
-## its task. The set must cover a complete Nim project.
-
-import sets
-
-from os import nil
-from std/private/miscdollars import toLocation
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
-import ".." / [ast, modulegraphs, msgs, options]
-import packed_ast, bitabs, ic
-
-type
-  NavContext = object
-    g: ModuleGraph
-    thisModule: int32
-    trackPos: PackedLineInfo
-    alreadyEmitted: HashSet[string]
-    outputSep: char # for easier testing, use short filenames and spaces instead of tabs.
-
-proc isTracked(current, trackPos: PackedLineInfo, tokenLen: int): bool =
-  if current.file == trackPos.file and current.line == trackPos.line:
-    let col = trackPos.col
-    if col >= current.col and col < current.col+tokenLen:
-      return true
-
-proc searchLocalSym(c: var NavContext; s: PackedSym; info: PackedLineInfo): bool =
-  result = s.name != LitId(0) and
-    isTracked(info, c.trackPos, c.g.packed[c.thisModule].fromDisk.strings[s.name].len)
-
-proc searchForeignSym(c: var NavContext; s: ItemId; info: PackedLineInfo): bool =
-  let name = c.g.packed[s.module].fromDisk.syms[s.item].name
-  result = name != LitId(0) and
-    isTracked(info, c.trackPos, c.g.packed[s.module].fromDisk.strings[name].len)
-
-const
-  EmptyItemId = ItemId(module: -1'i32, item: -1'i32)
-
-proc search(c: var NavContext; tree: PackedTree): ItemId =
-  # We use the linear representation here directly:
-  for i in 0..high(tree.nodes):
-    case tree.nodes[i].kind
-    of nkSym:
-      let item = tree.nodes[i].operand
-      if searchLocalSym(c, c.g.packed[c.thisModule].fromDisk.syms[item], tree.nodes[i].info):
-        return ItemId(module: c.thisModule, item: item)
-    of nkModuleRef:
-      if tree.nodes[i].info.line == c.trackPos.line and tree.nodes[i].info.file == c.trackPos.file:
-        let (n1, n2) = sons2(tree, NodePos i)
-        assert n1.kind == nkInt32Lit
-        assert n2.kind == nkInt32Lit
-        let pId = PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)
-        let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config)
-        if searchForeignSym(c, itemId, tree.nodes[i].info):
-          return itemId
-    else: discard
-  return EmptyItemId
-
-proc isDecl(tree: PackedTree; n: NodePos): bool =
-  # XXX This is not correct yet.
-  const declarativeNodes = procDefs + {nkMacroDef, nkTemplateDef,
-    nkLetSection, nkVarSection, nkUsingStmt, nkConstSection, nkTypeSection,
-    nkIdentDefs, nkEnumTy, nkVarTuple}
-  result = n.int >= 0 and tree[n.int].kind in declarativeNodes
-
-proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) =
-  var m = ""
-  var file = c.g.packed[c.thisModule].fromDisk.strings[info.file]
-  if c.outputSep == ' ':
-    file = os.extractFilename file
-  toLocation(m, file, info.line.int, info.col.int + ColOffset)
-  if not c.alreadyEmitted.containsOrIncl(m):
-    msgWriteln c.g.config, (if isDecl: "def" else: "usage") & c.outputSep & m
-
-proc list(c: var NavContext; tree: PackedTree; sym: ItemId) =
-  for i in 0..high(tree.nodes):
-    case tree.nodes[i].kind
-    of nkSym:
-      let item = tree.nodes[i].operand
-      if sym.item == item and sym.module == c.thisModule:
-        usage(c, tree.nodes[i].info, isDecl(tree, parent(NodePos i)))
-    of nkModuleRef:
-      let (n1, n2) = sons2(tree, NodePos i)
-      assert n1.kind == nkInt32Lit
-      assert n2.kind == nkInt32Lit
-      let pId = PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)
-      let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config)
-      if itemId.item == sym.item and sym.module == itemId.module:
-        usage(c, tree.nodes[i].info, isDecl(tree, parent(NodePos i)))
-    else: discard
-
-proc searchForIncludeFile(g: ModuleGraph; fullPath: string): int =
-  for i in 0..high(g.packed):
-    for k in 1..high(g.packed[i].fromDisk.includes):
-      # we start from 1 because the first "include" file is
-      # the module's filename.
-      if os.cmpPaths(g.packed[i].fromDisk.strings[g.packed[i].fromDisk.includes[k][0]], fullPath) == 0:
-        return i
-  return -1
-
-proc nav(g: ModuleGraph) =
-  # translate the track position to a packed position:
-  let unpacked = g.config.m.trackPos
-  var mid = unpacked.fileIndex.int
-
-  let fullPath = toFullPath(g.config, unpacked.fileIndex)
-
-  if g.packed[mid].status == undefined:
-    # check if 'mid' is an include file of some other module:
-    mid = searchForIncludeFile(g, fullPath)
-
-  if mid < 0:
-    localError(g.config, unpacked, "unknown file name: " & fullPath)
-    return
-
-  let fileId = g.packed[mid].fromDisk.strings.getKeyId(fullPath)
-
-  if fileId == LitId(0):
-    internalError(g.config, unpacked, "cannot find a valid file ID")
-    return
-
-  var c = NavContext(
-    g: g,
-    thisModule: int32 mid,
-    trackPos: PackedLineInfo(line: unpacked.line, col: unpacked.col, file: fileId),
-    outputSep: if isDefined(g.config, "nimIcNavigatorTests"): ' ' else: '\t'
-  )
-  var symId = search(c, g.packed[mid].fromDisk.topLevel)
-  if symId == EmptyItemId:
-    symId = search(c, g.packed[mid].fromDisk.bodies)
-
-  if symId == EmptyItemId:
-    localError(g.config, unpacked, "no symbol at this position")
-    return
-
-  for i in 0..high(g.packed):
-    # case statement here to enforce exhaustive checks.
-    case g.packed[i].status
-    of undefined:
-      discard "nothing to do"
-    of loading:
-      assert false, "cannot check integrity: Module still loading"
-    of stored, storing, outdated, loaded:
-      c.thisModule = int32 i
-      list(c, g.packed[i].fromDisk.topLevel, symId)
-      list(c, g.packed[i].fromDisk.bodies, symId)
-
-proc navDefinition*(g: ModuleGraph) = nav(g)
-proc navUsages*(g: ModuleGraph) = nav(g)
-proc navDefusages*(g: ModuleGraph) = nav(g)
-
-proc writeRodFiles*(g: ModuleGraph) =
-  for i in 0..high(g.packed):
-    case g.packed[i].status
-    of undefined, loading, stored, loaded:
-      discard "nothing to do"
-    of storing, outdated:
-      closeRodFile(g, g.packed[i].module)

compiler/ic/packed_ast.nim

@@ -1,419 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2020 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Packed AST representation, mostly based on a seq of nodes.
-## For IC support. Far future: Rewrite the compiler passes to
-## use this representation directly in all the transformations,
-## it is superior.
-
-import hashes, tables, strtabs
-import bitabs
-import ".." / [ast, options]
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
-type
-  SymId* = distinct int32
-  ModuleId* = distinct int32
-  NodePos* = distinct int
-
-  NodeId* = distinct int32
-
-  PackedItemId* = object
-    module*: LitId       # 0 if it's this module
-    item*: int32         # same as the in-memory representation
-
-const
-  nilItemId* = PackedItemId(module: LitId(0), item: -1.int32)
-
-const
-  emptyNodeId* = NodeId(-1)
-
-type
-  PackedLineInfo* = object
-    line*: uint16
-    col*: int16
-    file*: LitId
-
-  PackedLib* = object
-    kind*: TLibKind
-    generated*: bool
-    isOverriden*: bool
-    name*: LitId
-    path*: NodeId
-
-  PackedSym* = object
-    kind*: TSymKind
-    name*: LitId
-    typ*: PackedItemId
-    flags*: TSymFlags
-    magic*: TMagic
-    info*: PackedLineInfo
-    ast*: NodeId
-    owner*: PackedItemId
-    guard*: PackedItemId
-    bitsize*: int
-    alignment*: int # for alignment
-    options*: TOptions
-    position*: int
-    offset*: int
-    externalName*: LitId # instead of TLoc
-    locFlags*: TLocFlags
-    annex*: PackedLib
-    when hasFFI:
-      cname*: LitId
-    constraint*: NodeId
-
-  PackedType* = object
-    kind*: TTypeKind
-    callConv*: TCallingConvention
-    #nodekind*: TNodeKind
-    flags*: TTypeFlags
-    types*: seq[PackedItemId]
-    n*: NodeId
-    #nodeflags*: TNodeFlags
-    sym*: PackedItemId
-    owner*: PackedItemId
-    size*: BiggestInt
-    align*: int16
-    paddingAtEnd*: int16
-    # not serialized: loc*: TLoc because it is backend-specific
-    typeInst*: PackedItemId
-    nonUniqueId*: int32
-
-  PackedNode* = object     # 28 bytes
-    kind*: TNodeKind
-    flags*: TNodeFlags
-    operand*: int32  # for kind in {nkSym, nkSymDef}: SymId
-                     # for kind in {nkStrLit, nkIdent, nkNumberLit}: LitId
-                     # for kind in nkInt32Lit: direct value
-                     # for non-atom kinds: the number of nodes (for easy skipping)
-    typeId*: PackedItemId
-    info*: PackedLineInfo
-
-  PackedTree* = object ## usually represents a full Nim module
-    nodes*: seq[PackedNode]
-
-  PackedInstantiation* = object
-    key*, sym*: PackedItemId
-    concreteTypes*: seq[PackedItemId]
-
-proc `==`*(a, b: SymId): bool {.borrow.}
-proc hash*(a: SymId): Hash {.borrow.}
-
-proc `==`*(a, b: NodePos): bool {.borrow.}
-#proc `==`*(a, b: PackedItemId): bool {.borrow.}
-proc `==`*(a, b: NodeId): bool {.borrow.}
-
-proc newTreeFrom*(old: PackedTree): PackedTree =
-  result.nodes = @[]
-  when false: result.sh = old.sh
-
-when false:
-  proc declareSym*(tree: var PackedTree; kind: TSymKind;
-                  name: LitId; info: PackedLineInfo): SymId =
-    result = SymId(tree.sh.syms.len)
-    tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info)
-
-  proc litIdFromName*(tree: PackedTree; name: string): LitId =
-    result = tree.sh.strings.getOrIncl(name)
-
-  proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) =
-    tree.nodes.add PackedNode(kind: kind, info: info,
-                              operand: int32 getOrIncl(tree.sh.strings, token))
-
-  proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) =
-    tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
-
-proc throwAwayLastNode*(tree: var PackedTree) =
-  tree.nodes.setLen(tree.nodes.len-1)
-
-proc addIdent*(tree: var PackedTree; s: LitId; info: PackedLineInfo) =
-  tree.nodes.add PackedNode(kind: nkIdent, operand: int32(s), info: info)
-
-proc addSym*(tree: var PackedTree; s: int32; info: PackedLineInfo) =
-  tree.nodes.add PackedNode(kind: nkSym, operand: s, info: info)
-
-proc addModuleId*(tree: var PackedTree; s: ModuleId; info: PackedLineInfo) =
-  tree.nodes.add PackedNode(kind: nkInt32Lit, operand: int32(s), info: info)
-
-proc addSymDef*(tree: var PackedTree; s: SymId; info: PackedLineInfo) =
-  tree.nodes.add PackedNode(kind: nkSym, operand: int32(s), info: info)
-
-proc isAtom*(tree: PackedTree; pos: int): bool {.inline.} = tree.nodes[pos].kind <= nkNilLit
-
-proc copyTree*(dest: var PackedTree; tree: PackedTree; n: NodePos) =
-  # and this is why the IR is superior. We can copy subtrees
-  # via a linear scan.
-  let pos = n.int
-  let L = if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
-  let d = dest.nodes.len
-  dest.nodes.setLen(d + L)
-  for i in 0..<L:
-    dest.nodes[d+i] = tree.nodes[pos+i]
-
-when false:
-  proc copySym*(dest: var PackedTree; tree: PackedTree; s: SymId): SymId =
-    result = SymId(dest.sh.syms.len)
-    assert int(s) < tree.sh.syms.len
-    let oldSym = tree.sh.syms[s.int]
-    dest.sh.syms.add oldSym
-
-type
-  PatchPos = distinct int
-
-when false:
-  proc prepare*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo): PatchPos =
-    result = PatchPos tree.nodes.len
-    tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
-
-proc prepare*(tree: var PackedTree; kind: TNodeKind; flags: TNodeFlags; typeId: PackedItemId; info: PackedLineInfo): PatchPos =
-  result = PatchPos tree.nodes.len
-  tree.nodes.add PackedNode(kind: kind, flags: flags, operand: 0, info: info,
-                            typeId: typeId)
-
-proc prepare*(dest: var PackedTree; source: PackedTree; sourcePos: NodePos): PatchPos =
-  result = PatchPos dest.nodes.len
-  dest.nodes.add source.nodes[sourcePos.int]
-
-proc patch*(tree: var PackedTree; pos: PatchPos) =
-  let pos = pos.int
-  assert tree.nodes[pos].kind > nkNilLit
-  let distance = int32(tree.nodes.len - pos)
-  tree.nodes[pos].operand = distance
-
-proc len*(tree: PackedTree): int {.inline.} = tree.nodes.len
-
-proc `[]`*(tree: PackedTree; i: int): lent PackedNode {.inline.} =
-  tree.nodes[i]
-
-proc nextChild(tree: PackedTree; pos: var int) {.inline.} =
-  if tree.nodes[pos].kind > nkNilLit:
-    assert tree.nodes[pos].operand > 0
-    inc pos, tree.nodes[pos].operand
-  else:
-    inc pos
-
-iterator sonsReadonly*(tree: PackedTree; n: NodePos): NodePos =
-  var pos = n.int
-  assert tree.nodes[pos].kind > nkNilLit
-  let last = pos + tree.nodes[pos].operand
-  inc pos
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-iterator sons*(dest: var PackedTree; tree: PackedTree; n: NodePos): NodePos =
-  let patchPos = prepare(dest, tree, n)
-  for x in sonsReadonly(tree, n): yield x
-  patch dest, patchPos
-
-iterator isons*(dest: var PackedTree; tree: PackedTree;
-                n: NodePos): (int, NodePos) =
-  var i = 0
-  for ch0 in sons(dest, tree, n):
-    yield (i, ch0)
-    inc i
-
-iterator sonsFrom1*(tree: PackedTree; n: NodePos): NodePos =
-  var pos = n.int
-  assert tree.nodes[pos].kind > nkNilLit
-  let last = pos + tree.nodes[pos].operand
-  inc pos
-  if pos < last:
-    nextChild tree, pos
-  while pos < last:
-    yield NodePos pos
-    nextChild tree, pos
-
-iterator sonsWithoutLast2*(tree: PackedTree; n: NodePos): NodePos =
-  var count = 0
-  for child in sonsReadonly(tree, n):
-    inc count
-  var pos = n.int
-  assert tree.nodes[pos].kind > nkNilLit
-  let last = pos + tree.nodes[pos].operand
-  inc pos
-  while pos < last and count > 2:
-    yield NodePos pos
-    dec count
-    nextChild tree, pos
-
-proc parentImpl(tree: PackedTree; n: NodePos): NodePos =
-  # finding the parent of a node is rather easy:
-  var pos = n.int - 1
-  while pos >= 0 and (isAtom(tree, pos) or (pos + tree.nodes[pos].operand - 1 < n.int)):
-    dec pos
-  #assert pos >= 0, "node has no parent"
-  result = NodePos(pos)
-
-template parent*(n: NodePos): NodePos = parentImpl(tree, n)
-
-proc hasXsons*(tree: PackedTree; n: NodePos; x: int): bool =
-  var count = 0
-  if tree.nodes[n.int].kind > nkNilLit:
-    for child in sonsReadonly(tree, n): inc count
-  result = count == x
-
-proc hasAtLeastXsons*(tree: PackedTree; n: NodePos; x: int): bool =
-  if tree.nodes[n.int].kind > nkNilLit:
-    var count = 0
-    for child in sonsReadonly(tree, n):
-      inc count
-      if count >= x: return true
-  return false
-
-proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} =
-  NodePos(n.int+1)
-proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} =
-  tree.nodes[n.int].kind
-proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} =
-  LitId tree.nodes[n.int].operand
-proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} =
-  tree.nodes[n.int].info
-
-template typ*(n: NodePos): PackedItemId =
-  tree.nodes[n.int].typeId
-template flags*(n: NodePos): TNodeFlags =
-  tree.nodes[n.int].flags
-
-template operand*(n: NodePos): int32 =
-  tree.nodes[n.int].operand
-
-proc span*(tree: PackedTree; pos: int): int {.inline.} =
-  if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
-
-proc sons2*(tree: PackedTree; n: NodePos): (NodePos, NodePos) =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  result = (NodePos a, NodePos b)
-
-proc sons3*(tree: PackedTree; n: NodePos): (NodePos, NodePos, NodePos) =
-  assert(not isAtom(tree, n.int))
-  let a = n.int+1
-  let b = a + span(tree, a)
-  let c = b + span(tree, b)
-  result = (NodePos a, NodePos b, NodePos c)
-
-proc ithSon*(tree: PackedTree; n: NodePos; i: int): NodePos =
-  if tree.nodes[n.int].kind > nkNilLit:
-    var count = 0
-    for child in sonsReadonly(tree, n):
-      if count == i: return child
-      inc count
-  assert false, "node has no i-th child"
-
-when false:
-  proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} =
-    tree.sh.strings[lit]
-
-template kind*(n: NodePos): TNodeKind = tree.nodes[n.int].kind
-template info*(n: NodePos): PackedLineInfo = tree.nodes[n.int].info
-template litId*(n: NodePos): LitId = LitId tree.nodes[n.int].operand
-
-template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].operand
-
-proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
-
-when false:
-  # xxx `nkStrLit` or `nkStrLit..nkTripleStrLit:` below?
-  proc strLit*(tree: PackedTree; n: NodePos): lent string =
-    assert n.kind == nkStrLit
-    result = tree.sh.strings[LitId tree.nodes[n.int].operand]
-
-  proc strVal*(tree: PackedTree; n: NodePos): string =
-    assert n.kind == nkStrLit
-    result = tree.sh.strings[LitId tree.nodes[n.int].operand]
-    #result = cookedStrLit(raw)
-
-  proc filenameVal*(tree: PackedTree; n: NodePos): string =
-    case n.kind
-    of nkStrLit:
-      result = strVal(tree, n)
-    of nkIdent:
-      result = tree.sh.strings[n.litId]
-    of nkSym:
-      result = tree.sh.strings[tree.sh.syms[int n.symId].name]
-    else:
-      result = ""
-
-  proc identAsStr*(tree: PackedTree; n: NodePos): lent string =
-    assert n.kind == nkIdent
-    result = tree.sh.strings[LitId tree.nodes[n.int].operand]
-
-const
-  externIntLit* = {nkCharLit,
-    nkIntLit,
-    nkInt8Lit,
-    nkInt16Lit,
-    nkInt64Lit,
-    nkUIntLit,
-    nkUInt8Lit,
-    nkUInt16Lit,
-    nkUInt32Lit,
-    nkUInt64Lit} # nkInt32Lit is missing by design!
-
-  externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt64Lit}
-  externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit}
-  directIntLit* = nkInt32Lit
-
-when false:
-  proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
-    if n.kind == nkIdent:
-      result = n.litId
-    elif n.kind == nkSym:
-      result = tree.sh.syms[int n.symId].name
-    else:
-      result = LitId(0)
-
-  template identId*(n: NodePos): LitId = identIdImpl(tree, n)
-
-template copyInto*(dest, n, body) =
-  let patchPos = prepare(dest, tree, n)
-  body
-  patch dest, patchPos
-
-template copyIntoKind*(dest, kind, info, body) =
-  let patchPos = prepare(dest, kind, info)
-  body
-  patch dest, patchPos
-
-when false:
-  proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool =
-    let litId = tree.sh.strings.getKeyId(pragma)
-    if litId == LitId(0):
-      return false
-    assert n.kind == nkPragma
-    for ch0 in sonsReadonly(tree, n):
-      if ch0.kind == nkExprColonExpr:
-        if ch0.firstSon.identId == litId:
-          return true
-      elif ch0.identId == litId:
-        return true
-
-proc getNodeId*(tree: PackedTree): NodeId {.inline.} = NodeId tree.nodes.len
-
-when false:
-  proc produceError*(dest: var PackedTree; tree: PackedTree; n: NodePos; msg: string) =
-    let patchPos = prepare(dest, nkError, n.info)
-    dest.add nkStrLit, msg, n.info
-    copyTree(dest, tree, n)
-    patch dest, patchPos
-
-iterator allNodes*(tree: PackedTree): NodePos =
-  var p = 0
-  while p < tree.len:
-    yield NodePos(p)
-    let s = span(tree, p)
-    inc p, s
-
-proc toPackedItemId*(item: int32): PackedItemId {.inline.} =
-  PackedItemId(module: LitId(0), item: item)

compiler/ic/replayer.nim

@@ -1,160 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2020 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Module that contains code to replay global VM state changes and pragma
-## state like ``{.compile: "foo.c".}``. For IC (= Incremental compilation)
-## support.
-
-import ".." / [ast, modulegraphs, trees, extccomp, btrees,
-  msgs, lineinfos, pathutils, options, cgmeth]
-
-import tables
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
-import packed_ast, ic, bitabs
-
-proc replayStateChanges*(module: PSym; g: ModuleGraph) =
-  let list = module.ast
-  assert list != nil
-  assert list.kind == nkStmtList
-  for n in list:
-    assert n.kind == nkReplayAction
-    # Fortunately only a tiny subset of the available pragmas need to
-    # be replayed here. This is always a subset of ``pragmas.stmtPragmas``.
-    if n.len >= 2:
-      internalAssert g.config, n[0].kind == nkStrLit and n[1].kind == nkStrLit
-      case n[0].strVal
-      of "hint": message(g.config, n.info, hintUser, n[1].strVal)
-      of "warning": message(g.config, n.info, warnUser, n[1].strVal)
-      of "error": localError(g.config, n.info, errUser, n[1].strVal)
-      of "compile":
-        internalAssert g.config, n.len == 4 and n[2].kind == nkStrLit
-        let cname = AbsoluteFile n[1].strVal
-        var cf = Cfile(nimname: splitFile(cname).name, cname: cname,
-                       obj: AbsoluteFile n[2].strVal,
-                       flags: {CfileFlag.External},
-                       customArgs: n[3].strVal)
-        extccomp.addExternalFileToCompile(g.config, cf)
-      of "link":
-        extccomp.addExternalFileToLink(g.config, AbsoluteFile n[1].strVal)
-      of "passl":
-        extccomp.addLinkOption(g.config, n[1].strVal)
-      of "passc":
-        extccomp.addCompileOption(g.config, n[1].strVal)
-      of "localpassc":
-        extccomp.addLocalCompileOption(g.config, n[1].strVal, toFullPathConsiderDirty(g.config, module.info.fileIndex))
-      of "cppdefine":
-        options.cppDefine(g.config, n[1].strVal)
-      of "inc":
-        let destKey = n[1].strVal
-        let by = n[2].intVal
-        let v = getOrDefault(g.cacheCounters, destKey)
-        g.cacheCounters[destKey] = v+by
-      of "put":
-        let destKey = n[1].strVal
-        let key = n[2].strVal
-        let val = n[3]
-        if not contains(g.cacheTables, destKey):
-          g.cacheTables[destKey] = initBTree[string, PNode]()
-        if not contains(g.cacheTables[destKey], key):
-          g.cacheTables[destKey].add(key, val)
-        else:
-          internalError(g.config, n.info, "key already exists: " & key)
-      of "incl":
-        let destKey = n[1].strVal
-        let val = n[2]
-        if not contains(g.cacheSeqs, destKey):
-          g.cacheSeqs[destKey] = newTree(nkStmtList, val)
-        else:
-          block search:
-            for existing in g.cacheSeqs[destKey]:
-              if exprStructuralEquivalent(existing, val, strictSymEquality=true):
-                break search
-            g.cacheSeqs[destKey].add val
-      of "add":
-        let destKey = n[1].strVal
-        let val = n[2]
-        if not contains(g.cacheSeqs, destKey):
-          g.cacheSeqs[destKey] = newTree(nkStmtList, val)
-        else:
-          g.cacheSeqs[destKey].add val
-      else:
-        internalAssert g.config, false
-
-proc replayBackendProcs*(g: ModuleGraph; module: int) =
-  for it in mitems(g.packed[module].fromDisk.attachedOps):
-    let key = translateId(it[0], g.packed, module, g.config)
-    let op = it[1]
-    let tmp = translateId(it[2], g.packed, module, g.config)
-    let symId = FullId(module: tmp.module, packed: it[2])
-    g.attachedOps[op][key] = LazySym(id: symId, sym: nil)
-
-  for it in mitems(g.packed[module].fromDisk.enumToStringProcs):
-    let key = translateId(it[0], g.packed, module, g.config)
-    let tmp = translateId(it[1], g.packed, module, g.config)
-    let symId = FullId(module: tmp.module, packed: it[1])
-    g.enumToStringProcs[key] = LazySym(id: symId, sym: nil)
-
-proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
-  ## We remember the generic instantiations a module performed
-  ## in order to to avoid the code bloat that generic code tends
-  ## to imply. This is cheaper than deduplication of identical
-  ## generic instantiations. However, deduplication is more
-  ## powerful and general and I hope to implement it soon too
-  ## (famous last words).
-  assert g.packed[module].status == loaded
-  for it in g.packed[module].fromDisk.typeInstCache:
-    let key = translateId(it[0], g.packed, module, g.config)
-    g.typeInstCache.mgetOrPut(key, @[]).add LazyType(id: FullId(module: module, packed: it[1]), typ: nil)
-
-  for it in mitems(g.packed[module].fromDisk.procInstCache):
-    let key = translateId(it.key, g.packed, module, g.config)
-    let sym = translateId(it.sym, g.packed, module, g.config)
-    var concreteTypes = newSeq[FullId](it.concreteTypes.len)
-    for i in 0..high(it.concreteTypes):
-      let tmp = translateId(it.concreteTypes[i], g.packed, module, g.config)
-      concreteTypes[i] = FullId(module: tmp.module, packed: it.concreteTypes[i])
-
-    g.procInstCache.mgetOrPut(key, @[]).add LazyInstantiation(
-      module: module, sym: FullId(module: sym.module, packed: it.sym),
-      concreteTypes: concreteTypes, inst: nil)
-
-  for it in mitems(g.packed[module].fromDisk.methodsPerType):
-    let key = translateId(it[0], g.packed, module, g.config)
-    let col = it[1]
-    let tmp = translateId(it[2], g.packed, module, g.config)
-    let symId = FullId(module: tmp.module, packed: it[2])
-    g.methodsPerType.mgetOrPut(key, @[]).add (col, LazySym(id: symId, sym: nil))
-
-  replayBackendProcs(g, module)
-
-  for it in mitems(g.packed[module].fromDisk.methods):
-    let sym = loadSymFromId(g.config, g.cache, g.packed, module,
-                            PackedItemId(module: LitId(0), item: it))
-    methodDef(g, g.idgen, sym)
-
-  when false:
-    # not used anymore:
-    for it in mitems(g.packed[module].fromDisk.compilerProcs):
-      let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it[1]))
-      g.lazyCompilerprocs[g.packed[module].fromDisk.sh.strings[it[0]]] = symId
-
-  for it in mitems(g.packed[module].fromDisk.converters):
-    let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it))
-    g.ifaces[module].converters.add LazySym(id: symId, sym: nil)
-
-  for it in mitems(g.packed[module].fromDisk.trmacros):
-    let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it))
-    g.ifaces[module].patterns.add LazySym(id: symId, sym: nil)
-
-  for it in mitems(g.packed[module].fromDisk.pureEnums):
-    let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it))
-    g.ifaces[module].pureEnums.add LazySym(id: symId, sym: nil)

compiler/ic/rodfiles.nim

@@ -1,249 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2020 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Low level binary format used by the compiler to store and load various AST
-## and related data.
-##
-## NB: this is incredibly low level and if you're interested in how the
-##     compiler works and less a storage format, you're probably looking for
-##     the `ic` or `packed_ast` modules to understand the logical format.
-
-from typetraits import supportsCopyMem
-
-when defined(nimPreviewSlimSystem):
-  import std/[syncio, assertions]
-
-## Overview
-## ========
-## `RodFile` represents a Rod File (versioned binary format), and the
-## associated data for common interactions such as IO and error tracking
-## (`RodFileError`). The file format broken up into sections (`RodSection`)
-## and preceeded by a header (see: `cookie`). The precise layout, section
-## ordering and data following the section are determined by the user. See
-## `ic.loadRodFile`.
-##
-## A basic but "wrong" example of the lifecycle:
-## ---------------------------------------------
-## 1. `create` or `open`        - create a new one or open an existing
-## 2. `storeHeader`             - header info
-## 3. `storePrim` or `storeSeq` - save your stuff
-## 4. `close`                   - and we're done
-##
-## Now read the bits below to understand what's missing.
-##
-## ### Issues with the Example
-## Missing Sections:
-## This is a low level API, so headers and sections need to be stored and
-## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` &
-## `loadSection`, respectively.
-##
-## No Error Handling:
-## The API is centered around IO and prone to error, each operation checks or
-## sets the `RodFile.err` field. A user of this API needs to handle these
-## appropriately.
-##
-## API Notes
-## =========
-##
-## Valid inputs for Rod files
-## --------------------------
-## ASTs, hopes, dreams, and anything as long as it and any children it may have
-## support `copyMem`. This means anything that is not a pointer and that does not contain a pointer. At a glance these are:
-## * string
-## * objects & tuples (fields are recursed)
-## * sequences AKA `seq[T]`
-##
-## Note on error handling style
-## ----------------------------
-## A flag based approach is used where operations no-op in case of a
-## preexisting error and set the flag if they encounter one.
-##
-## Misc
-## ----
-## * 'Prim' is short for 'primitive', as in a non-sequence type
-
-type
-  RodSection* = enum
-    versionSection
-    configSection
-    stringsSection
-    checkSumsSection
-    depsSection
-    numbersSection
-    exportsSection
-    hiddenSection
-    reexportsSection
-    compilerProcsSection
-    trmacrosSection
-    convertersSection
-    methodsSection
-    pureEnumsSection
-    toReplaySection
-    topLevelSection
-    bodiesSection
-    symsSection
-    typesSection
-    typeInstCacheSection
-    procInstCacheSection
-    attachedOpsSection
-    methodsPerTypeSection
-    enumToStringProcsSection
-    typeInfoSection  # required by the backend
-    backendFlagsSection
-    aliveSymsSection # beware, this is stored in a `.alivesyms` file.
-
-  RodFileError* = enum
-    ok, tooBig, cannotOpen, ioFailure, wrongHeader, wrongSection, configMismatch,
-    includeFileChanged
-
-  RodFile* = object
-    f*: File
-    currentSection*: RodSection # for error checking
-    err*: RodFileError # little experiment to see if this works
-                       # better than exceptions.
-
-const
-  RodVersion = 1
-  cookie = [byte(0), byte('R'), byte('O'), byte('D'),
-            byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)]
-
-proc setError(f: var RodFile; err: RodFileError) {.inline.} =
-  f.err = err
-  #raise newException(IOError, "IO error")
-
-proc storePrim*(f: var RodFile; s: string) =
-  ## Stores a string.
-  ## The len is prefixed to allow for later retreival.
-  if f.err != ok: return
-  if s.len >= high(int32):
-    setError f, tooBig
-    return
-  var lenPrefix = int32(s.len)
-  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    setError f, ioFailure
-  else:
-    if s.len != 0:
-      if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
-        setError f, ioFailure
-
-proc storePrim*[T](f: var RodFile; x: T) =
-  ## Stores a non-sequence/string `T`.
-  ## If `T` doesn't support `copyMem` and is an object or tuple then the fields
-  ## are written -- the user from context will need to know which `T` to load.
-  if f.err != ok: return
-  when supportsCopyMem(T):
-    if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
-      setError f, ioFailure
-  elif T is tuple:
-    for y in fields(x):
-      storePrim(f, y)
-  elif T is object:
-    for y in fields(x):
-      when y is seq:
-        storeSeq(f, y)
-      else:
-        storePrim(f, y)
-  else:
-    {.error: "unsupported type for 'storePrim'".}
-
-proc storeSeq*[T](f: var RodFile; s: seq[T]) =
-  ## Stores a sequence of `T`s, with the len as a prefix for later retrieval.
-  if f.err != ok: return
-  if s.len >= high(int32):
-    setError f, tooBig
-    return
-  var lenPrefix = int32(s.len)
-  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    setError f, ioFailure
-  else:
-    for i in 0..<s.len:
-      storePrim(f, s[i])
-
-proc loadPrim*(f: var RodFile; s: var string) =
-  ## Read a string, the length was stored as a prefix
-  if f.err != ok: return
-  var lenPrefix = int32(0)
-  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    setError f, ioFailure
-  else:
-    s = newString(lenPrefix)
-    if lenPrefix > 0:
-      if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
-        setError f, ioFailure
-
-proc loadPrim*[T](f: var RodFile; x: var T) =
-  ## Load a non-sequence/string `T`.
-  if f.err != ok: return
-  when supportsCopyMem(T):
-    if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
-      setError f, ioFailure
-  elif T is tuple:
-    for y in fields(x):
-      loadPrim(f, y)
-  elif T is object:
-    for y in fields(x):
-      when y is seq:
-        loadSeq(f, y)
-      else:
-        loadPrim(f, y)
-  else:
-    {.error: "unsupported type for 'loadPrim'".}
-
-proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
-  ## `T` must be compatible with `copyMem`, see `loadPrim`
-  if f.err != ok: return
-  var lenPrefix = int32(0)
-  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    setError f, ioFailure
-  else:
-    s = newSeq[T](lenPrefix)
-    for i in 0..<lenPrefix:
-      loadPrim(f, s[i])
-
-proc storeHeader*(f: var RodFile) =
-  ## stores the header which is described by `cookie`.
-  if f.err != ok: return
-  if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len:
-    setError f, ioFailure
-
-proc loadHeader*(f: var RodFile) =
-  ## Loads the header which is described by `cookie`.
-  if f.err != ok: return
-  var thisCookie: array[cookie.len, byte]
-  if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len:
-    setError f, ioFailure
-  elif thisCookie != cookie:
-    setError f, wrongHeader
-
-proc storeSection*(f: var RodFile; s: RodSection) =
-  ## update `currentSection` and writes the bytes value of s.
-  if f.err != ok: return
-  assert f.currentSection < s
-  f.currentSection = s
-  storePrim(f, s)
-
-proc loadSection*(f: var RodFile; expected: RodSection) =
-  ## read the bytes value of s, sets and error if the section is incorrect.
-  if f.err != ok: return
-  var s: RodSection
-  loadPrim(f, s)
-  if expected != s and f.err == ok:
-    setError f, wrongSection
-
-proc create*(filename: string): RodFile =
-  ## create the file and open it for writing
-  if not open(result.f, filename, fmWrite):
-    setError result, cannotOpen
-
-proc close*(f: var RodFile) = close(f.f)
-
-proc open*(filename: string): RodFile =
-  ## open the file for reading
-  if not open(result.f, filename, fmRead):
-    setError result, cannotOpen

compiler/idents.nim

@@ -14,13 +14,13 @@
 import
   hashes, wordrecg
 
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
 type
-  PIdent* = ref TIdent
-  TIdent*{.acyclic.} = object
+  TIdObj* = object of RootObj
     id*: int # unique id; use this for comparisons and not the pointers
+
+  PIdObj* = ref TIdObj
+  PIdent* = ref TIdent
+  TIdent*{.acyclic.} = object of TIdObj
     s*: string
     next*: PIdent             # for hash-table chaining
     h*: Hash                 # hash value of s
@@ -116,8 +116,3 @@ proc newIdentCache*(): IdentCache =
 proc whichKeyword*(id: PIdent): TSpecialWord =
   if id.id < 0: result = wInvalid
   else: result = TSpecialWord(id.id)
-
-proc hash*(x: PIdent): Hash {.inline.} = x.h
-proc `==`*(a, b: PIdent): bool {.inline.} =
-  if a.isNil or b.isNil: result = system.`==`(a, b)
-  else: result = a.id == b.id

compiler/idgen.nim

@@ -0,0 +1,59 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2012 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module contains a simple persistent id generator.
+
+import idents, strutils, options, pathutils
+
+var gFrontEndId*: int
+
+const
+  debugIds* = false
+
+when debugIds:
+  import intsets
+
+  var usedIds = initIntSet()
+
+proc registerID*(id: PIdObj) =
+  when debugIds:
+    if id.id == -1 or containsOrIncl(usedIds, id.id):
+      internalError("ID already used: " & $id.id)
+
+proc getID*(): int {.inline.} =
+  result = gFrontEndId
+  inc(gFrontEndId)
+
+proc setId*(id: int) {.inline.} =
+  gFrontEndId = max(gFrontEndId, id + 1)
+
+proc idSynchronizationPoint*(idRange: int) =
+  gFrontEndId = (gFrontEndId div idRange + 1) * idRange + 1
+
+proc toGid(conf: ConfigRef; f: AbsoluteFile): string =
+  # we used to use ``f.addFileExt("gid")`` (aka ``$project.gid``), but this
+  # will cause strange bugs if multiple projects are in the same folder, so
+  # we simply use a project independent name:
+  result = options.completeGeneratedFilePath(conf, AbsoluteFile"nim.gid").string
+
+proc saveMaxIds*(conf: ConfigRef; project: AbsoluteFile) =
+  var f = open(toGid(conf, project), fmWrite)
+  f.writeLine($gFrontEndId)
+  f.close()
+
+proc loadMaxIds*(conf: ConfigRef; project: AbsoluteFile) =
+  var f: File
+  if open(f, toGid(conf, project), fmRead):
+    var line = newStringOfCap(20)
+    if f.readLine(line):
+      var frontEndId = parseInt(line)
+      if f.readLine(line):
+        # var backEndId = parseInt(line)
+        gFrontEndId = max(gFrontEndId, frontEndId)
+    f.close()

compiler/importer.nim

@@ -11,12 +11,7 @@
 
 import
   intsets, ast, astalgo, msgs, options, idents, lookups,
-  semdata, modulepaths, sigmatch, lineinfos, sets,
-  modulegraphs, wordrecg, tables
-from strutils import `%`
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
+  semdata, modulepaths, sigmatch, lineinfos, sets
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
@@ -25,67 +20,30 @@ proc readExceptSet*(c: PContext, n: PNode): IntSet =
     let ident = lookups.considerQuotedIdent(c, n[i])
     result.incl(ident.id)
 
-proc declarePureEnumField*(c: PContext; s: PSym) =
-  # XXX Remove the outer 'if' statement and see what breaks.
-  var amb = false
-  if someSymFromImportTable(c, s.name, amb) == nil:
-    strTableAdd(c.pureEnumFields, s)
-    when false:
-      let checkB = strTableGet(c.pureEnumFields, s.name)
-      if checkB == nil:
-        strTableAdd(c.pureEnumFields, s)
-    when false:
+proc importPureEnumField*(c: PContext; s: PSym) =
+  let check = strTableGet(c.importTable.symbols, s.name)
+  if check == nil:
+    let checkB = strTableGet(c.pureEnumFields, s.name)
+    if checkB == nil:
+      strTableAdd(c.pureEnumFields, s)
+    else:
       # mark as ambiguous:
       incl(c.ambiguousSymbols, checkB.id)
       incl(c.ambiguousSymbols, s.id)
 
-proc importPureEnumField(c: PContext; s: PSym) =
-  var amb = false
-  if someSymFromImportTable(c, s.name, amb) == nil:
-    strTableAdd(c.pureEnumFields, s)
-    when false:
-      let checkB = strTableGet(c.pureEnumFields, s.name)
-      if checkB == nil:
-        strTableAdd(c.pureEnumFields, s)
-    when false:
-      # mark as ambiguous:
-      incl(c.ambiguousSymbols, checkB.id)
-      incl(c.ambiguousSymbols, s.id)
-
-proc importPureEnumFields(c: PContext; s: PSym; etyp: PType) =
-  assert sfPure in s.flags
-  for j in 0..<etyp.n.len:
-    var e = etyp.n[j].sym
-    if e.kind != skEnumField:
-      internalError(c.config, s.info, "rawImportSymbol")
-      # BUGFIX: because of aliases for enums the symbol may already
-      # have been put into the symbol table
-      # BUGFIX: but only iff they are the same symbols!
-    for check in importedItems(c, e.name):
-      if check.id == e.id:
-        e = nil
-        break
-    if e != nil:
-      importPureEnumField(c, e)
-
-proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) =
+proc rawImportSymbol(c: PContext, s, origin: PSym) =
   # This does not handle stubs, because otherwise loading on demand would be
   # pointless in practice. So importing stubs is fine here!
   # check if we have already a symbol of the same name:
-  when false:
-    var check = someSymFromImportTable(c, s.name)
-    if check != nil and check.id != s.id:
-      if s.kind notin OverloadableSyms or check.kind notin OverloadableSyms:
-        # s and check need to be qualified:
-        incl(c.ambiguousSymbols, s.id)
-        incl(c.ambiguousSymbols, check.id)
+  var check = strTableGet(c.importTable.symbols, s.name)
+  if check != nil and check.id != s.id:
+    if s.kind notin OverloadableSyms or check.kind notin OverloadableSyms:
+      # s and check need to be qualified:
+      incl(c.ambiguousSymbols, s.id)
+      incl(c.ambiguousSymbols, check.id)
   # thanks to 'export' feature, it could be we import the same symbol from
-  # multiple sources, so we need to call 'strTableAdd' here:
-  when false:
-    # now lazy. Speeds up the compiler and is a prerequisite for IC.
-    strTableAdd(c.importTable.symbols, s)
-  else:
-    importSet.incl s.id
+  # multiple sources, so we need to call 'StrTableAdd' here:
+  strTableAdd(c.importTable.symbols, s)
   if s.kind == skType:
     var etyp = s.typ
     if etyp.kind in {tyBool, tyEnum}:
@@ -96,42 +54,27 @@ proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) =
           # BUGFIX: because of aliases for enums the symbol may already
           # have been put into the symbol table
           # BUGFIX: but only iff they are the same symbols!
-        for check in importedItems(c, e.name):
+        var it: TIdentIter
+        check = initIdentIter(it, c.importTable.symbols, e.name)
+        while check != nil:
           if check.id == e.id:
             e = nil
             break
+          check = nextIdentIter(it, c.importTable.symbols)
         if e != nil:
           if sfPure notin s.flags:
-            rawImportSymbol(c, e, origin, importSet)
+            rawImportSymbol(c, e, origin)
           else:
             importPureEnumField(c, e)
   else:
-    if s.kind == skConverter: addConverter(c, LazySym(sym: s))
-    if hasPattern(s): addPattern(c, LazySym(sym: s))
+    if s.kind == skConverter: addConverter(c, s)
+    if hasPattern(s): addPattern(c, s)
   if s.owner != origin:
     c.exportIndirections.incl((origin.id, s.id))
 
-proc splitPragmas(c: PContext, n: PNode): (PNode, seq[TSpecialWord]) =
-  template bail = globalError(c.config, n.info, "invalid pragma")
-  if n.kind == nkPragmaExpr:
-    if n.len == 2 and n[1].kind == nkPragma:
-      result[0] = n[0]
-      for ni in n[1]:
-        if ni.kind == nkIdent: result[1].add whichKeyword(ni.ident)
-        else: bail()
-    else: bail()
-  else:
-    result[0] = n
-    if result[0].safeLen > 0:
-      (result[0][^1], result[1]) = splitPragmas(c, result[0][^1])
-
-proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
-  let (n, kws) = splitPragmas(c, n)
-  if kws.len > 0:
-    globalError(c.config, n.info, "unexpected pragma")
-
+proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
   let ident = lookups.considerQuotedIdent(c, n)
-  let s = someSym(c.graph, fromMod, ident)
+  let s = strTableGet(fromMod.tab, ident)
   if s == nil:
     errorUndeclaredIdentifier(c, n.info, ident.s)
   else:
@@ -141,77 +84,34 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
     # for an enumeration we have to add all identifiers
     if multiImport:
       # for a overloadable syms add all overloaded routines
-      var it: ModuleIter
-      var e = initModuleIter(it, c.graph, fromMod, s.name)
+      var it: TIdentIter
+      var e = initIdentIter(it, fromMod.tab, s.name)
       while e != nil:
         if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3")
         if s.kind in ExportableSymKinds:
-          rawImportSymbol(c, e, fromMod, importSet)
-        e = nextModuleIter(it, c.graph)
+          rawImportSymbol(c, e, fromMod)
+        e = nextIdentIter(it, fromMod.tab)
     else:
-      rawImportSymbol(c, s, fromMod, importSet)
-    suggestSym(c.graph, n.info, s, c.graph.usageSym, false)
-
-proc addImport(c: PContext; im: sink ImportedModule) =
-  for i in 0..high(c.imports):
-    if c.imports[i].m == im.m:
-      # we have already imported the module: Check which import
-      # is more "powerful":
-      case c.imports[i].mode
-      of importAll: discard "already imported all symbols"
-      of importSet:
-        case im.mode
-        of importAll, importExcept:
-          # XXX: slightly wrong semantics for 'importExcept'...
-          # But we should probably change the spec and disallow this case.
-          c.imports[i] = im
-        of importSet:
-          # merge the import sets:
-          c.imports[i].imported.incl im.imported
-      of importExcept:
-        case im.mode
-        of importAll:
-          c.imports[i] = im
-        of importSet:
-          discard
-        of importExcept:
-          var cut = initIntSet()
-          # only exclude what is consistent between the two sets:
-          for j in im.exceptSet:
-            if j in c.imports[i].exceptSet:
-              cut.incl j
-          c.imports[i].exceptSet = cut
-      return
-  c.imports.add im
-
-template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} =
-  for it in mitems c.graph.ifaces[fromMod.position].converters:
-    if filter:
-      loadPackedSym(c.graph, it)
-      if sfExported in it.sym.flags:
-        addConverter(c, it)
-  for it in mitems c.graph.ifaces[fromMod.position].patterns:
-    if filter:
-      loadPackedSym(c.graph, it)
-      if sfExported in it.sym.flags:
-        addPattern(c, it)
-  for it in mitems c.graph.ifaces[fromMod.position].pureEnums:
-    if filter:
-      loadPackedSym(c.graph, it)
-      importPureEnumFields(c, it.sym, it.sym.typ)
+      rawImportSymbol(c, s, fromMod)
+    suggestSym(c.config, n.info, s, c.graph.usageSym, false)
 
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
-  c.addImport ImportedModule(m: fromMod, mode: importExcept, exceptSet: exceptSet)
-  addUnnamedIt(c, fromMod, it.sym.name.id notin exceptSet)
+  var i: TTabIter
+  var s = initTabIter(i, fromMod.tab)
+  while s != nil:
+    if s.kind != skModule:
+      if s.kind != skEnumField:
+        if s.kind notin ExportableSymKinds:
+          internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s)
+        if exceptSet.isNil or s.name.id notin exceptSet:
+          rawImportSymbol(c, s, fromMod)
+    s = nextIter(i, fromMod.tab)
 
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
-  c.addImport ImportedModule(m: fromMod, mode: importAll)
-  addUnnamedIt(c, fromMod, true)
-  when false:
-    var exceptSet: IntSet
-    importAllSymbolsExcept(c, fromMod, exceptSet)
+  var exceptSet: IntSet
+  importAllSymbolsExcept(c, fromMod, exceptSet)
 
-proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; importSet: var IntSet) =
+proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym) =
   if n.isNil: return
   case n.kind
   of nkExportStmt:
@@ -221,55 +121,27 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; im
       if s.kind == skModule:
         importAllSymbolsExcept(c, s, exceptSet)
       elif exceptSet.isNil or s.name.id notin exceptSet:
-        rawImportSymbol(c, s, fromMod, importSet)
+        rawImportSymbol(c, s, fromMod)
   of nkExportExceptStmt:
     localError(c.config, n.info, "'export except' not implemented")
   else:
     for i in 0..n.safeLen-1:
-      importForwarded(c, n[i], exceptSet, fromMod, importSet)
+      importForwarded(c, n[i], exceptSet, fromMod)
 
-proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool): PSym =
+proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
   result = realModule
-  template createModuleAliasImpl(ident): untyped =
-    createModuleAlias(realModule, nextSymId c.idgen, ident, n.info, c.config.options)
+  c.unusedImports.add((realModule, n.info))
   if n.kind != nkImportAs: discard
   elif n.len != 2 or n[1].kind != nkIdent:
     localError(c.config, n.info, "module alias must be an identifier")
   elif n[1].ident.id != realModule.name.id:
     # some misguided guy will write 'import abc.foo as foo' ...
-    result = createModuleAliasImpl(n[1].ident)
-  if result == realModule:
-    # avoids modifying `realModule`, see D20201209T194412 for `import {.all.}`
-    result = createModuleAliasImpl(realModule.name)
-  if importHidden:
-    result.options.incl optImportHidden
-  c.unusedImports.add((result, n.info))
-  c.importModuleMap[result.id] = realModule.id
+    result = createModuleAlias(realModule, n[1].ident, realModule.info,
+                               c.config.options)
 
-proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] =
-  var ret: typeof(result)
-  proc processPragma(n2: PNode): PNode =
-    let (result2, kws) = splitPragmas(c, n2)
-    result = result2
-    for ai in kws:
-      case ai
-      of wImportHidden: ret.importHidden = true
-      else: globalError(c.config, n.info, "invalid pragma, expected: " & ${wImportHidden})
-
-  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
-    ret.node = newNodeI(nkImportAs, n.info)
-    ret.node.add n[1].processPragma
-    ret.node.add n[2]
-  else:
-    ret.node = n.processPragma
-  return ret
-
-proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym =
-  let transf = transformImportAs(c, n)
-  n = transf.node
+proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
   let f = checkModuleName(c.config, n)
   if f != InvalidFileIdx:
-    addImportFileDep(c, f)
     let L = c.graph.importStack.len
     let recursion = c.graph.importStack.find(f)
     c.graph.importStack.add f
@@ -282,42 +154,43 @@ proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym =
                 toFullPath(c.config, c.graph.importStack[i+1])
       c.recursiveDep = err
 
-    var realModule: PSym
     discard pushOptionEntry(c)
-    realModule = c.graph.importModuleCallback(c.graph, c.module, f)
-    result = importModuleAs(c, n, realModule, transf.importHidden)
+    result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f))
     popOptionEntry(c)
 
     #echo "set back to ", L
     c.graph.importStack.setLen(L)
     # we cannot perform this check reliably because of
-    # test: modules/import_in_config) # xxx is that still true?
-    if realModule == c.module:
-      localError(c.config, n.info, "module '$1' cannot import itself" % realModule.name.s)
-    if sfDeprecated in realModule.flags:
-      var prefix = ""
-      if realModule.constraint != nil: prefix = realModule.constraint.strVal & "; "
-      message(c.config, n.info, warnDeprecated, prefix & realModule.name.s & " is deprecated")
-    suggestSym(c.graph, n.info, result, c.graph.usageSym, false)
+    # test: modules/import_in_config)
+    when true:
+      if result.info.fileIndex == c.module.info.fileIndex and
+          result.info.fileIndex == n.info.fileIndex:
+        localError(c.config, n.info, "A module cannot import itself")
+    if sfDeprecated in result.flags:
+      if result.constraint != nil:
+        message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated")
+      else:
+        message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated")
+    suggestSym(c.config, n.info, result, c.graph.usageSym, false)
     importStmtResult.add newSymNode(result, n.info)
     #newStrNode(toFullPath(c.config, f), n.info)
 
-proc afterImport(c: PContext, m: PSym) =
-  # fixes bug #17510, for re-exported symbols
-  let realModuleId = c.importModuleMap[m.id]
-  for s in allSyms(c.graph, m):
-    if s.owner.id != realModuleId:
-      c.exportIndirections.incl((m.id, s.id))
+proc transformImportAs(c: PContext; n: PNode): PNode =
+  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
+    result = newNodeI(nkImportAs, n.info)
+    result.add n[1]
+    result.add n[2]
+  else:
+    result = n
 
 proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
-  var it = it
+  let it = transformImportAs(c, it)
   let m = myImportModule(c, it, importStmtResult)
   if m != nil:
     # ``addDecl`` needs to be done before ``importAllSymbols``!
     addDecl(c, m, it.info) # add symbol to symbol table of module
     importAllSymbols(c, m)
     #importForwarded(c, m.ast, emptySet, m)
-    afterImport(c, m)
 
 proc evalImport*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
@@ -346,25 +219,22 @@ proc evalImport*(c: PContext, n: PNode): PNode =
 proc evalFrom*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
+  n[0] = transformImportAs(c, n[0])
   var m = myImportModule(c, n[0], result)
   if m != nil:
     n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
-
-    var im = ImportedModule(m: m, mode: importSet, imported: initIntSet())
     for i in 1..<n.len:
       if n[i].kind != nkNilLit:
-        importSymbol(c, n[i], m, im.imported)
-    c.addImport im
-    afterImport(c, m)
+        importSymbol(c, n[i], m)
 
 proc evalImportExcept*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
+  n[0] = transformImportAs(c, n[0])
   var m = myImportModule(c, n[0], result)
   if m != nil:
     n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
     importAllSymbolsExcept(c, m, readExceptSet(c, n))
     #importForwarded(c, m.ast, exceptSet, m)
-    afterImport(c, m)

compiler/incremental.nim

@@ -0,0 +1,198 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2018 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Basic type definitions the module graph needs in order to support
+## incremental compilations.
+
+const nimIncremental* = defined(nimIncremental)
+
+import options, lineinfos
+
+when nimIncremental:
+  import ast, msgs, intsets, btrees, db_sqlite, std / sha1, pathutils
+  from strutils import parseInt
+  from os import isAbsolute
+
+  type
+    Writer* = object
+      sstack*: seq[PSym]          # a stack of symbols to process
+      tstack*: seq[PType]         # a stack of types to process
+      tmarks*, smarks*: IntSet
+      forwardedSyms*: seq[PSym]
+
+    Reader* = object
+      syms*: BTree[int, PSym]
+      types*: BTree[int, PType]
+
+    IncrementalCtx* = object
+      db*: DbConn
+      w*: Writer
+      r*: Reader
+      configChanged*: bool
+
+  proc init*(incr: var IncrementalCtx) =
+    incr.w.sstack = @[]
+    incr.w.tstack = @[]
+    incr.w.tmarks = initIntSet()
+    incr.w.smarks = initIntSet()
+    incr.w.forwardedSyms = @[]
+    incr.r.syms = initBTree[int, PSym]()
+    incr.r.types = initBTree[int, PType]()
+
+
+  proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: AbsoluteFile): string =
+    result = msgs.getHash(conf, fileIdx)
+    if result.len == 0 and isAbsolute(string fullpath):
+      result = $secureHashFile(string fullpath)
+      msgs.setHash(conf, fileIdx, result)
+
+  proc toDbFileId*(incr: var IncrementalCtx; conf: ConfigRef; fileIdx: FileIndex): int =
+    if fileIdx == FileIndex(-1): return -1
+    let fullpath = toFullPath(conf, fileIdx)
+    let row = incr.db.getRow(sql"select id, fullhash from filenames where fullpath = ?",
+      fullpath)
+    let id = row[0]
+    let fullhash = hashFileCached(conf, fileIdx, AbsoluteFile fullpath)
+    if id.len == 0:
+      result = int incr.db.insertID(sql"insert into filenames(nimid, fullpath, fullhash) values (?, ?, ?)",
+        int(fileIdx), fullpath, fullhash)
+    else:
+      if row[1] != fullhash:
+        incr.db.exec(sql"update filenames set fullhash = ? where fullpath = ?", fullhash, fullpath)
+      result = parseInt(id)
+
+  proc fromDbFileId*(incr: var IncrementalCtx; conf: ConfigRef; dbId: int): FileIndex =
+    if dbId == -1: return FileIndex(-1)
+    let fullpath = incr.db.getValue(sql"select fullpath from filenames where id = ?", dbId)
+    doAssert fullpath.len > 0, "cannot find file name for DB ID " & $dbId
+    result = fileInfoIdx(conf, AbsoluteFile fullpath)
+
+
+  proc addModuleDep*(incr: var IncrementalCtx; conf: ConfigRef;
+                     module, fileIdx: FileIndex;
+                     isIncludeFile: bool) =
+    if conf.symbolFiles != v2Sf: return
+
+    let a = toDbFileId(incr, conf, module)
+    let b = toDbFileId(incr, conf, fileIdx)
+
+    incr.db.exec(sql"insert into deps(module, dependency, isIncludeFile) values (?, ?, ?)",
+      a, b, ord(isIncludeFile))
+
+  # --------------- Database model ---------------------------------------------
+
+  proc createDb*(db: DbConn) =
+    db.exec(sql"""
+      create table if not exists controlblock(
+        idgen integer not null
+      );
+    """)
+
+    db.exec(sql"""
+      create table if not exists config(
+        config varchar(8000) not null
+      );
+    """)
+
+    db.exec(sql"""
+      create table if not exists filenames(
+        id integer primary key,
+        nimid integer not null,
+        fullpath varchar(8000) not null,
+        fullHash varchar(256) not null
+      );
+    """)
+    db.exec sql"create index if not exists FilenameIx on filenames(fullpath);"
+
+    db.exec(sql"""
+      create table if not exists modules(
+        id integer primary key,
+        nimid integer not null,
+        fullpath varchar(8000) not null,
+        interfHash varchar(256) not null,
+        fullHash varchar(256) not null,
+
+        created timestamp not null default (DATETIME('now'))
+      );""")
+    db.exec(sql"""create unique index if not exists SymNameIx on modules(fullpath);""")
+
+    db.exec(sql"""
+      create table if not exists deps(
+        id integer primary key,
+        module integer not null,
+        dependency integer not null,
+        isIncludeFile integer not null,
+        foreign key (module) references filenames(id),
+        foreign key (dependency) references filenames(id)
+      );""")
+    db.exec(sql"""create index if not exists DepsIx on deps(module);""")
+
+    db.exec(sql"""
+      create table if not exists types(
+        id integer primary key,
+        nimid integer not null,
+        module integer not null,
+        data blob not null,
+        foreign key (module) references module(id)
+      );
+    """)
+    db.exec sql"create index TypeByModuleIdx on types(module);"
+    db.exec sql"create index TypeByNimIdIdx on types(nimid);"
+
+    db.exec(sql"""
+      create table if not exists syms(
+        id integer primary key,
+        nimid integer not null,
+        module integer not null,
+        name varchar(256) not null,
+        data blob not null,
+        exported int not null,
+        foreign key (module) references module(id)
+      );
+    """)
+    db.exec sql"create index if not exists SymNameIx on syms(name);"
+    db.exec sql"create index SymByNameAndModuleIdx on syms(name, module);"
+    db.exec sql"create index SymByModuleIdx on syms(module);"
+    db.exec sql"create index SymByNimIdIdx on syms(nimid);"
+
+
+    db.exec(sql"""
+      create table if not exists toplevelstmts(
+        id integer primary key,
+        position integer not null,
+        module integer not null,
+        data blob not null,
+        foreign key (module) references module(id)
+      );
+    """)
+    db.exec sql"create index TopLevelStmtByModuleIdx on toplevelstmts(module);"
+    db.exec sql"create index TopLevelStmtByPositionIdx on toplevelstmts(position);"
+
+    db.exec(sql"""
+      create table if not exists statics(
+        id integer primary key,
+        module integer not null,
+        data blob not null,
+        foreign key (module) references module(id)
+      );
+    """)
+    db.exec sql"create index StaticsByModuleIdx on toplevelstmts(module);"
+    db.exec sql"insert into controlblock(idgen) values (0)"
+
+
+else:
+  type
+    IncrementalCtx* = object
+
+  template init*(incr: IncrementalCtx) = discard
+
+  template addModuleDep*(incr: var IncrementalCtx; conf: ConfigRef;
+                     module, fileIdx: FileIndex;
+                     isIncludeFile: bool) =
+    discard

compiler/installer.ini

@@ -13,7 +13,6 @@ Platforms: """
   netbsd: i386;amd64
   openbsd: i386;amd64;arm;arm64
   dragonfly: i386;amd64
-  crossos: amd64
   haiku: i386;amd64
   android: i386;arm;arm64
   nintendoswitch: arm64
@@ -77,6 +76,7 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/fusion"
 
 Files: "tests"
 
@@ -120,6 +120,7 @@ Files: "bin/nim"
 InstallScript: "yes"
 UninstallScript: "yes"
 Files: "bin/nim-gdb"
+Files: "bin/nim-gdb.bash"
 
 
 [InnoSetup]
@@ -141,5 +142,5 @@ shortDesc: "The Nim Compiler"
 licenses: "bin/nim,MIT;lib/*,MIT;"
 
 [nimble]
-pkgName: "nim"
+pkgName: "compiler"
 pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt;doc/nimdoc.css"

compiler/int128.nim

@@ -1,16 +1,13 @@
 ## This module is for compiler internal use only. For reliable error
 ## messages and range checks, the compiler needs a data type that can
-## hold all from `low(BiggestInt)` to `high(BiggestUInt)`, This
+## hold all from ``low(BiggestInt)`` to ``high(BiggestUInt)``, This
 ## type is for that purpose.
 
 from math import trunc
 
-when defined(nimPreviewSlimSystem):
-  import std/assertions
-
 type
   Int128* = object
-    udata: array[4, uint32]
+    udata: array[4,uint32]
 
 template sdata(arg: Int128, idx: int): int32 =
   # udata and sdata was supposed to be in a union, but unions are
@@ -20,12 +17,12 @@ template sdata(arg: Int128, idx: int): int32 =
 # encoding least significant int first (like LittleEndian)
 
 const
-  Zero* = Int128(udata: [0'u32, 0, 0, 0])
-  One* = Int128(udata: [1'u32, 0, 0, 0])
-  Ten* = Int128(udata: [10'u32, 0, 0, 0])
-  Min = Int128(udata: [0'u32, 0, 0, 0x80000000'u32])
-  Max = Int128(udata: [high(uint32), high(uint32), high(uint32), uint32(high(int32))])
-  NegOne* = Int128(udata: [0xffffffff'u32, 0xffffffff'u32, 0xffffffff'u32, 0xffffffff'u32])
+  Zero* = Int128(udata: [0'u32,0,0,0])
+  One* = Int128(udata: [1'u32,0,0,0])
+  Ten* = Int128(udata: [10'u32,0,0,0])
+  Min = Int128(udata: [0'u32,0,0,0x80000000'u32])
+  Max = Int128(udata: [high(uint32),high(uint32),high(uint32),uint32(high(int32))])
+  NegOne* = Int128(udata: [0xffffffff'u32,0xffffffff'u32,0xffffffff'u32,0xffffffff'u32])
 
 template low*(t: typedesc[Int128]): Int128 = Min
 template high*(t: typedesc[Int128]): Int128 = Max
@@ -57,9 +54,15 @@ proc toInt128*[T: SomeInteger | bool](arg: T): Int128 =
 template isNegative(arg: Int128): bool =
   arg.sdata(3) < 0
 
-proc bitconcat(a, b: uint32): uint64 =
+template isNegative(arg: int32): bool =
+  arg < 0
+
+proc bitconcat(a,b: uint32): uint64 =
   (uint64(a) shl 32) or uint64(b)
 
+proc bitsplit(a: uint64): (uint32,uint32) =
+  (cast[uint32](a shr 32), cast[uint32](a))
+
 proc toInt64*(arg: Int128): int64 =
   if isNegative(arg):
     assert(arg.sdata(3) == -1, "out of range")
@@ -173,6 +176,7 @@ proc toHex*(arg: Int128): string =
   result.addToHex(arg)
 
 proc inc*(a: var Int128, y: uint32 = 1) =
+  let input = a
   a.udata[0] += y
   if unlikely(a.udata[0] < y):
     a.udata[1].inc
@@ -182,7 +186,7 @@ proc inc*(a: var Int128, y: uint32 = 1) =
         a.udata[3].inc
         doAssert(a.sdata(3) != low(int32), "overflow")
 
-proc cmp*(a, b: Int128): int =
+proc cmp*(a,b: Int128): int =
   let tmp1 = cmp(a.sdata(3), b.sdata(3))
   if tmp1 != 0: return tmp1
   let tmp2 = cmp(a.udata[2], b.udata[2])
@@ -192,38 +196,44 @@ proc cmp*(a, b: Int128): int =
   let tmp4 = cmp(a.udata[0], b.udata[0])
   return tmp4
 
-proc `<`*(a, b: Int128): bool =
-  cmp(a, b) < 0
+proc `<`*(a,b: Int128): bool =
+  cmp(a,b) < 0
 
-proc `<=`*(a, b: Int128): bool =
-  cmp(a, b) <= 0
+proc `<=`*(a,b: Int128): bool =
+  cmp(a,b) <= 0
 
-proc `==`*(a, b: Int128): bool =
+proc `==`*(a,b: Int128): bool =
   if a.udata[0] != b.udata[0]: return false
   if a.udata[1] != b.udata[1]: return false
   if a.udata[2] != b.udata[2]: return false
   if a.udata[3] != b.udata[3]: return false
   return true
 
+proc inplaceBitnot(a: var Int128) =
+  a.udata[0] = not a.udata[0]
+  a.udata[1] = not a.udata[1]
+  a.udata[2] = not a.udata[2]
+  a.udata[3] = not a.udata[3]
+
 proc bitnot*(a: Int128): Int128 =
   result.udata[0] = not a.udata[0]
   result.udata[1] = not a.udata[1]
   result.udata[2] = not a.udata[2]
   result.udata[3] = not a.udata[3]
 
-proc bitand*(a, b: Int128): Int128 =
+proc bitand*(a,b: Int128): Int128 =
   result.udata[0] = a.udata[0] and b.udata[0]
   result.udata[1] = a.udata[1] and b.udata[1]
   result.udata[2] = a.udata[2] and b.udata[2]
   result.udata[3] = a.udata[3] and b.udata[3]
 
-proc bitor*(a, b: Int128): Int128 =
+proc bitor*(a,b: Int128): Int128 =
   result.udata[0] = a.udata[0] or b.udata[0]
   result.udata[1] = a.udata[1] or b.udata[1]
   result.udata[2] = a.udata[2] or b.udata[2]
   result.udata[3] = a.udata[3] or b.udata[3]
 
-proc bitxor*(a, b: Int128): Int128 =
+proc bitxor*(a,b: Int128): Int128 =
   result.udata[0] = a.udata[0] xor b.udata[0]
   result.udata[1] = a.udata[1] xor b.udata[1]
   result.udata[2] = a.udata[2] xor b.udata[2]
@@ -278,7 +288,7 @@ proc `shl`*(a: Int128, b: int): Int128 =
     result.udata[2] = 0
     result.udata[3] = a.udata[0] shl (b and 31)
 
-proc `+`*(a, b: Int128): Int128 =
+proc `+`*(a,b: Int128): Int128 =
   let tmp0 = uint64(a.udata[0]) + uint64(b.udata[0])
   result.udata[0] = cast[uint32](tmp0)
   let tmp1 = uint64(a.udata[1]) + uint64(b.udata[1]) + (tmp0 shr 32)
@@ -295,7 +305,7 @@ proc `-`*(a: Int128): Int128 =
   result = bitnot(a)
   result.inc
 
-proc `-`*(a, b: Int128): Int128 =
+proc `-`*(a,b: Int128): Int128 =
   a + (-b)
 
 proc `-=`*(a: var Int128, b: Int128) =
@@ -332,7 +342,7 @@ proc `*`*(a: Int128, b: int32): Int128 =
 proc `*=`*(a: var Int128, b: int32): Int128 =
   result = result * b
 
-proc makeInt128(high, low: uint64): Int128 =
+proc makeInt128(high,low: uint64): Int128 =
   result.udata[0] = cast[uint32](low)
   result.udata[1] = cast[uint32](low shr 32)
   result.udata[2] = cast[uint32](high)
@@ -344,11 +354,24 @@ proc high64(a: Int128): uint64 =
 proc low64(a: Int128): uint64 =
   bitconcat(a.udata[1], a.udata[0])
 
-proc `*`*(lhs, rhs: Int128): Int128 =
-  let a32 = uint64(lhs.udata[1])
-  let a00 = uint64(lhs.udata[0])
-  let b32 = uint64(rhs.udata[1])
-  let b00 = uint64(rhs.udata[0])
+proc `*`*(lhs,rhs: Int128): Int128 =
+  let
+    a = cast[uint64](lhs.udata[0])
+    b = cast[uint64](lhs.udata[1])
+    c = cast[uint64](lhs.udata[2])
+    d = cast[uint64](lhs.udata[3])
+
+    e = cast[uint64](rhs.udata[0])
+    f = cast[uint64](rhs.udata[1])
+    g = cast[uint64](rhs.udata[2])
+    h = cast[uint64](rhs.udata[3])
+
+
+  let a32 = cast[uint64](lhs.udata[1])
+  let a00 = cast[uint64](lhs.udata[0])
+  let b32 = cast[uint64](rhs.udata[1])
+  let b00 = cast[uint64](rhs.udata[0])
+
   result = makeInt128(high64(lhs) * low64(rhs) + low64(lhs) * high64(rhs) + a32 * b32, a00 * b00)
   result += toInt128(a32 * b00) shl 32
   result += toInt128(a00 * b32) shl 32
@@ -366,7 +389,7 @@ proc fastLog2*(a: Int128): int =
   if a.udata[1] != 0:
     return 32 + fastLog2(a.udata[1])
   if a.udata[0] != 0:
-    return fastLog2(a.udata[0])
+    return      fastLog2(a.udata[0])
 
 proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
   assert(divisor != Zero)
@@ -418,18 +441,18 @@ proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
   else:
     result.remainder = dividend
 
-proc `div`*(a, b: Int128): Int128 =
-  let (a, _) = divMod(a, b)
+proc `div`*(a,b: Int128): Int128 =
+  let (a,b) = divMod(a,b)
   return a
 
-proc `mod`*(a, b: Int128): Int128 =
-  let (_, b) = divMod(a, b)
+proc `mod`*(a,b: Int128): Int128 =
+  let (a,b) = divMod(a,b)
   return b
 
 proc addInt128*(result: var string; value: Int128) =
   let initialSize = result.len
   if value == Zero:
-    result.add '0'
+    result.add "0"
   elif value == low(Int128):
     result.add "-170141183460469231731687303715884105728"
   else:
@@ -450,13 +473,11 @@ proc addInt128*(result: var string; value: Int128) =
       j -= 1
 
 proc `$`*(a: Int128): string =
-  # "-170141183460469231731687303715884105728".len == 41
-  result = newStringOfCap(41)
   result.addInt128(a)
 
 proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =
   assert(pos < arg.len)
-  assert(arg[pos] in {'-', '0'..'9'})
+  assert(arg[pos] in {'-','0'..'9'})
 
   var isNegative = false
   var pos = pos
@@ -476,13 +497,13 @@ proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =
 # fluff
 
 proc `<`*(a: Int128, b: BiggestInt): bool =
-  cmp(a, toInt128(b)) < 0
+  cmp(a,toInt128(b)) < 0
 
 proc `<`*(a: BiggestInt, b: Int128): bool =
   cmp(toInt128(a), b) < 0
 
 proc `<=`*(a: Int128, b: BiggestInt): bool =
-  cmp(a, toInt128(b)) <= 0
+  cmp(a,toInt128(b)) <= 0
 
 proc `<=`*(a: BiggestInt, b: Int128): bool =
   cmp(toInt128(a), b) <= 0
@@ -518,7 +539,7 @@ proc toFloat64*(arg: Int128): float64 =
 
 proc ldexp(x: float64, exp: cint): float64 {.importc: "ldexp", header: "<math.h>".}
 
-template bitor(a, b, c: Int128): Int128 = bitor(bitor(a, b), c)
+template bitor(a,b,c: Int128): Int128 = bitor(bitor(a,b), c)
 
 proc toInt128*(arg: float64): Int128 =
   let isNegative = arg < 0

compiler/isolation_check.nim

@@ -11,10 +11,7 @@
 ## https://github.com/nim-lang/RFCs/issues/244 for more details.
 
 import
-  ast, types, renderer, intsets
-
-when defined(nimPreviewSlimSystem):
-  import std/assertions
+  ast, types, renderer, idents, intsets
 
 proc canAlias(arg, ret: PType; marker: var IntSet): bool
 
@@ -67,30 +64,19 @@ proc canAlias(arg, ret: PType; marker: var IntSet): bool =
   else:
     result = false
 
-proc isValueOnlyType(t: PType): bool =
+proc isValueOnlyType(t: PType): bool = 
   # t doesn't contain pointers and references
   proc wrap(t: PType): bool {.nimcall.} = t.kind in {tyRef, tyPtr, tyVar, tyLent}
   result = not types.searchTypeFor(t, wrap)
 
 proc canAlias*(arg, ret: PType): bool =
   if isValueOnlyType(arg):
-    # can alias only with addr(arg.x) and we don't care if it is not safe
+    # can alias only with unsafeAddr(arg.x) and we don't care if it is not safe
     result = false
   else:
     var marker = initIntSet()
     result = canAlias(arg, ret, marker)
 
-proc containsVariable(n: PNode): bool =
-  case n.kind
-  of nodesToIgnoreSet:
-    result = false
-  of nkSym:
-    result = n.sym.kind in {skForVar, skParam, skVar, skLet, skConst, skResult, skTemp}
-  else:
-    for ch in n:
-      if containsVariable(ch): return true
-    result = false
-
 proc checkIsolate*(n: PNode): bool =
   if types.containsTyRef(n.typ):
     # XXX Maybe require that 'n.typ' is acyclic. This is not much
@@ -99,10 +85,7 @@ proc checkIsolate*(n: PNode): bool =
     of nkCharLit..nkNilLit:
       result = true
     of nkCallKinds:
-      # XXX: as long as we don't update the analysis while examining arguments
-      #      we can do an early check of the return type, otherwise this is a
-      #      bug and needs to be moved below
-      if tfNoSideEffect notin n[0].typ.flags:
+      if n[0].typ.flags * {tfGcSafe, tfNoSideEffect} == {}:
         return false
       for i in 1..<n.len:
         if checkIsolate(n[i]):
@@ -110,23 +93,14 @@ proc checkIsolate*(n: PNode): bool =
         else:
           let argType = n[i].typ
           if argType != nil and not isCompileTimeOnly(argType) and containsTyRef(argType):
-            if argType.canAlias(n.typ) or containsVariable(n[i]):
-              # bug #19013: Alias information is not enough, we need to check for potential
-              # "overlaps". I claim the problem can only happen by reading again from a location
-              # that materialized which is only possible if a variable that contains a `ref`
-              # is involved.
+            if argType.canAlias(n.typ):
               return false
       result = true
     of nkIfStmt, nkIfExpr:
       for it in n:
         result = checkIsolate(it.lastSon)
         if not result: break
-    of nkCaseStmt:
-      for i in 1..<n.len:
-        result = checkIsolate(n[i].lastSon)
-        if not result: break
-    of nkObjConstr:
-      result = true
+    of nkCaseStmt, nkObjConstr:
       for i in 1..<n.len:
         result = checkIsolate(n[i].lastSon)
         if not result: break
@@ -149,3 +123,4 @@ proc checkIsolate*(n: PNode): bool =
   else:
     # no ref, no cry:
     result = true
+

compiler/jstypes.nim

@@ -19,13 +19,13 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
     s, u: Rope
     field: PSym
     b: PNode
-  result = ""
+  result = nil
   case n.kind
   of nkRecList:
     if n.len == 1:
       result = genObjectFields(p, typ, n[0])
     else:
-      s = ""
+      s = nil
       for i in 0..<n.len:
         if i > 0: s.add(", \L")
         s.add(genObjectFields(p, typ, n[i]))
@@ -44,13 +44,13 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
     s = genTypeInfo(p, field.typ)
     for i in 1..<n.len:
       b = n[i]           # branch
-      u = ""
+      u = nil
       case b.kind
       of nkOfBranch:
         if b.len < 2:
           internalError(p.config, b.info, "genObjectFields; nkOfBranch broken")
         for j in 0..<b.len - 1:
-          if u != "": u.add(", ")
+          if u != nil: u.add(", ")
           if b[j].kind == nkRange:
             u.addf("[$1, $2]", [rope(getOrdValue(b[j][0])),
                                  rope(getOrdValue(b[j][1]))])
@@ -59,7 +59,7 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
       of nkElse:
         u = rope(lengthOrd(p.config, field.typ))
       else: internalError(p.config, n.info, "genObjectFields(nkRecCase)")
-      if result != "": result.add(", \L")
+      if result != nil: result.add(", \L")
       result.addf("[setConstr($1), $2]",
            [u, genObjectFields(p, typ, lastSon(b))])
     result = ("{kind: 3, offset: \"$1\", len: $3, " &
@@ -84,7 +84,7 @@ proc genObjectInfo(p: PProc, typ: PType, name: Rope) =
          [name, genTypeInfo(p, typ[0].skipTypes(skipPtrs))])
 
 proc genTupleFields(p: PProc, typ: PType): Rope =
-  var s: Rope = ""
+  var s: Rope = nil
   for i in 0..<typ.len:
     if i > 0: s.add(", \L")
     s.addf("{kind: 1, offset: \"Field$1\", len: 0, " &
@@ -102,7 +102,7 @@ proc genTupleInfo(p: PProc, typ: PType, name: Rope) =
   p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
 
 proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
-  var s: Rope = ""
+  var s: Rope = nil
   for i in 0..<typ.n.len:
     if (typ.n[i].kind != nkSym): internalError(p.config, typ.n.info, "genEnumInfo")
     let field = typ.n[i].sym
@@ -128,21 +128,21 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
   case t.kind
   of tyDistinct:
     result = genTypeInfo(p, t[0])
-  of tyPointer, tyProc, tyBool, tyChar, tyCstring, tyString, tyInt..tyUInt64:
+  of tyPointer, tyProc, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
       [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
   of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet:
     var s =
-      "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" %
+      "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
     p.g.typeInfo.addf("$1.base = $2;$n",
          [result, genTypeInfo(p, t.lastSon)])
   of tyArray:
     var s =
-      "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" %
+      "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
     p.g.typeInfo.addf("$1.base = $2;$n",