bump version to 0.19.6

(cherry picked from commit e54546bcff)

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: 'nim' # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: 'https://nim-lang.org/donate.html' # Replace with a single custom sponsorship URL

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,32 +1,37 @@
 ---
 name: Bug report
-about: Have you found an unexpected behavior? Use this template.
+about: You found an unexpected behaviour? Use this template.
+
 ---
 
 <!-- Think about the title, twice. -->
 
-<!-- Summarize the problem here, keep it short and simple. -->
-Function `echo` outputs the wrong string.
+<!-- Summarize the Problem here, keep it simple. -->
+<!-- e.g. `echo` outputs the wrong string. -->
 
 
 ### Example
-<!-- Paste your example in the code-block below. -->
+<!-- This should be a source code block. 
 ```nim
 echo "Hello World!"
 ```
+-->
 
 
 ### Current Output
+<!--
 ```
 Hola mundo!
 ```
+-->
 
 
 ### Expected Output
-<!-- What should be the correct output? -->
+<!--
 ```
 Hello World!
 ```
+-->
 
 
 ### Possible Solution
@@ -35,12 +40,6 @@ Hello World!
 
 ### Additional Information
 <!--- For Example:
-* Your Nim version (output of `nim -v`).
-* Was it working in the previous Nim releases?
-* A link to a related issue or discussion.
-* A project reference where (and how) the issue causes problems.
+   A link to a project where the issue is relevant.
+   A link to a related issue or discussion.
 -->
-```
-$ nim -v
-Nim Compiler Version 0.1.2
-```

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,18 +1,9 @@
 ---
 name: Feature request
-about: Do you want to suggest a new feature? Use this template.
+about: You want to suggest a new feature? Use this template.
 
 ---
 
-<!---
-Notice there is now a separate repository for the detailed RFCs and proposals:
-https://github.com/nim-lang/RFCs
-
-If you have a simple feature request, you can post it here using this template,
-but bear in mind that adding new features to the language is currently a low priority.
--->
-
-
 ### Summary
 <!--- Short summary of your proposed feature -->
 
@@ -27,6 +18,6 @@ but bear in mind that adding new features to the language is currently a low pri
 
 ### Additional Information
 <!--- For Example:
-* A link to a project where the issue is relevant.
-* A link to a related issue or discussion.
--->
+   A link to a project where the issue is relevant.
+   A link to a related issue or discussion.
+   -->

.gitignore

@@ -11,9 +11,7 @@ dnimcache/
 !/icons/*.o
 *.obj
 *.ilk
-*.exp
 *.pdb
-*.lib
 *.dll
 *.exe
 *.so
@@ -21,7 +19,6 @@ dnimcache/
 *.zip
 *.iss
 *.log
-*.pdb
 
 mapping.txt
 tags
@@ -33,7 +30,7 @@ doc/*.html
 doc/*.pdf
 doc/*.idx
 /web/upload
-/build/*
+build/*
 bin/*
 
 # iOS specific wildcards.
@@ -43,7 +40,6 @@ bin/*
 *.perspectivev3
 *.swp
 .DS_Store
-.tags
 project.xcworkspace/
 xcuserdata/
 
@@ -52,18 +48,12 @@ xcuserdata/
 /compiler/nim.dot
 /reject.json
 /run.json
-/tools/dochack/dochack.js
-*.json
-/pkgstemp/**/*
 # for `nim doc foo.nim`
 /*.html
-lib/**/*.html
 #/testresults.html #covered by /*.html
 
 /testresults.json
 testament.db
-/tests/**/*.json
-/tests/**/*.js
 /csources
 dist/
 
@@ -76,11 +66,3 @@ testresults/
 test.txt
 /test.ini
 
-tweeter.db
-tweeter_test.db
-megatest.nim
-
-/outputExpected.txt
-/outputGotten.txt
-
-/lib/pure/*.js

.gitlab-ci.yml

@@ -53,8 +53,9 @@ test-windows:
   <<: *win_set_path_def
   script:
     - call ci\deps.bat
-    - nim c testament\tester
+    - nim c --taintMode:on testament\tester
     - testament\tester.exe --pedantic all
   tags:
     - windows
     - fast
+

.travis.yml

@@ -1,79 +1,56 @@
 sudo: false
 language: c
+os:
+  - linux
+  - osx
+dist: trusty
 
-dist: xenial
-
-matrix:
-  fast_finish: true
-
-  include:
-    - os: linux
-      env:
-        - NIM_COMPILE_TO_CPP=false
-        - CPU=amd64
-
-    - os: linux
-      env:
-        - NIM_COMPILE_TO_CPP=false
-        - CPU=i386
-      addons:
-        apt:
-          packages:
-            - gcc-multilib
-            - g++-multilib
-            - libcurl4-openssl-dev:i386
-            - libgc-dev:i386
-            - libglib2.0-dev:i386
-            - libpulse-dev:i386
-            - libsdl1.2-dev:i386
-            - libsfml-dev:i386
-
-    - os: osx
-      env:
-        - NIM_COMPILE_TO_CPP=false
-        - CPU=amd64
-
-    - os: osx
-      env:
-        - NIM_COMPILE_TO_CPP=true
-        - CPU=amd64
-
-# To allow failures for a failing configuration, use something like:
+#matrix:
 #  allow_failures:
-#    - env: NIM_COMPILE_TO_CPP=true
 #    - os: osx
 
 addons:
   apt:
-    # update the list above if more deps are introduced
     packages:
     - libcurl4-openssl-dev
     - libsdl1.2-dev
     - libgc-dev
     - libsfml-dev
-  homebrew:
-    packages:
-    - boehmgc
-    - make
-    - sfml
-    update: true
 
-install:
-  - if [[ $CPU = i386 ]]; then echo -e "#!/bin/bash\n$(which gcc) -m32 \"\$@\"" > bin/gcc; fi
-  - if [[ $CPU = i386 ]]; then chmod 755 bin/gcc; fi
-  - if [[ $CPU = i386 ]]; then echo -e "#!/bin/bash\n$(which g++) -m32 \"\$@\"" > bin/g++; fi
-  - if [[ $CPU = i386 ]]; then chmod 755 bin/g++; fi
+before_install:
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install boehmgc; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install sfml; fi
 
 before_script:
+  - set -e
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then unset -f cd; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then shell_session_update() { :; }; fi
   - git clone --depth 1 https://github.com/nim-lang/csources.git
-  - export PATH="$PWD/bin${PATH:+:$PATH}"
-  - make -C csources -j 2 LD=$CC ucpu=$CPU
-
+  - cd csources
+  - sh build.sh
+  - cd ..
+  - export PATH=$(pwd)/bin${PATH:+:$PATH}
 script:
-  - echo "travis_fold:start:nim_c_koch"
   - nim c koch
-  - echo "travis_fold:end:nim_c_koch"
-  - ./koch runCI
+  - ./koch boot
+  - ./koch boot -d:release
+  - ./koch nimble
+  - nim e tests/test_nimscript.nims
+  #- nimble install zip -y
+  #- nimble install opengl
+  #- nimble install sdl1
+  #- nimble install jester@#head -y
+  #- nimble install niminst
+  - nim c --taintMode:on -d:nimCoroutines testament/tester
+  - testament/tester --pedantic all -d:nimCoroutines
+  - nim c -o:bin/nimpretty nimpretty/nimpretty.nim
+  - nim c -r nimpretty/tester.nim
+  - ./koch docs --git.commit:devel
+  - ./koch csource
+  - ./koch nimsuggest
+  - nim c -r nimsuggest/tester
+  - nim c -r nimdoc/tester
 
 before_deploy:
   # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html
@@ -87,6 +64,3 @@ deploy:                         # https://nim-lang.github.io/Nim
   keep-history: false
   on:
     branch: devel
-
-# Extract failed tests
-after_failure: nim c -r tools/ci_testresults.nim

appveyor.yml

@@ -1,39 +1,65 @@
 version: '{build}'
 
-environment:
-  DLLS_URL: https://nim-lang.org/download/dlls.zip
-  DLLS_ARCHIVE: dlls.zip
-  MINGW_DIR: mingw64
-  MINGW_URL: https://nim-lang.org/download/mingw64-6.3.0.7z
-  MINGW_ARCHIVE: mingw64-6.3.0.7z
-
-  matrix:
-    - NIM_TEST_PACKAGES: false
-    - NIM_TEST_PACKAGES: true
-
 cache:
-    - '%MINGW_ARCHIVE%'
-    - '%DLLS_ARCHIVE%'
+- x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z
+- sqlite-dll-win64-x64-3160200.zip
+# - i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
 
 matrix:
-  #allow_failures:
-  #  - NIM_TEST_PACKAGES: true
   fast_finish: true
 
+environment:
+  matrix:
+    - MINGW_DIR: mingw64
+      MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Personal%20Builds/mingw-builds/4.9.2/threads-win32/seh/x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z/download
+      MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z
+      SQLITE_URL: http://www.sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
+      SQLITE_ARCHIVE: sqlite-dll-win64-x64-3160200.zip
+      platform: x64
+    # - MINGW_DIR: mingw32
+    #   MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/4.9.2/threads-win32/dwarf/i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z/download
+    #   MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
+    #   SQLITE_URL: http://www.sqlite.org/2017/sqlite-dll-win32-x86-3160200.zip
+    #   SQLITE_ARCHIVE: sqlite-dll-win32-x86-3160200.zip
+    #   platform: x86
+
 install:
   - ps: Install-Product node 8 # node 8 or later is required to test js async stuff
-  - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%"
-  - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\BIN"> nul
+  - MKDIR %CD%\DIST
+  - MKDIR %CD%\DIST\PCRE
+  - nuget install pcre -Verbosity quiet -Version 8.33.0.1 -OutputDirectory %CD%\DIST\PCRE
+  - IF not exist "%SQLITE_ARCHIVE%" appveyor DownloadFile "%SQLITE_URL%" -FileName "%SQLITE_ARCHIVE%"
+  - 7z x -y "%SQLITE_ARCHIVE%" -o"%CD%\DIST"> nul
   - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%"
   - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul
   - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH%
+  - IF "%PLATFORM%" == "x64" ( copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay64.dll & copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl64.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl32.dll )
+    ELSE ( copy C:\OpenSSL-Win32\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win32\libssl32.dll %CD%\BIN\libssl32.dll )
+  - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\sqlite3.dll %CD%\BIN\sqlite3_64.dll ) ELSE ( copy %CD%\DIST\sqlite3.dll %CD%\BIN\sqlite3_32.dll )
+  - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\x64\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre64.dll ) ELSE ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\Win32\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre32.dll )
   - git clone --depth 1 https://github.com/nim-lang/csources
   - cd csources
-  - build64.bat
+  - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat )
   - cd ..
 
 build_script:
   - bin\nim c koch
-  - koch runCI
+  - koch boot -d:release
+  - koch nimble
+  - nim e tests/test_nimscript.nims
+  - nim c -o:bin/nimpretty.exe nimpretty/nimpretty.nim
+  - nim c -r nimpretty/tester.nim
+#  - nimble install zip -y
+#  - nimble install opengl -y
+#  - nimble install sdl1 -y
+#  - nimble install jester@#head -y
+  - nim c --taintMode:on -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester
+  - nim c --taintMode:on -d:nimCoroutines testament/tester
+
+test_script:
+  - testament\tester --pedantic all -d:nimCoroutines
+  - nim c -r nimdoc\tester
+#  - koch csource
+#  - koch zip
 
 deploy: off

bin/nim-gdb

@@ -3,9 +3,6 @@
 # Exit if anything fails
 set -e
 
-which nim > /dev/null || (echo "nim not in PATH"; exit 1)
-which gdb > /dev/null || (echo "gdb not in PATH"; exit 1)
-
 # Find out where the pretty printer Python module is
 NIM_SYSROOT=$(dirname $(dirname $(readlink -e $(which nim))))
 GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"

build_all.sh

@@ -17,17 +17,15 @@ build_nim_csources(){
   ## avoid changing dir in case of failure
   (
     echo_run cd csources
-    echo_run sh build.sh $@
+    echo_run sh build.sh
   )
   # keep $nim_csources in case needed to investigate bootstrap issues
   # without having to rebuild from csources
   echo_run cp bin/nim $nim_csources
 }
 
-[ -f $nim_csources ] || echo_run build_nim_csources $@
-
-# Note: if fails, may need to `cd csources && git pull`
-echo_run bin/nim c --skipUserCfg --skipParentCfg koch
+[ -f $nim_csources ] || echo_run build_nim_csources
 
+echo_run bin/nim c koch
 echo_run ./koch boot -d:release
 echo_run ./koch tools # Compile Nimble and other tools.

changelog.md

@@ -1,37 +1,12 @@
-# v1.1 - XXXX-XX-XX
+## v0.19.2 - 2018-11-04
+
+This is a bugfix release of version 0.19.0. No new features or breaking changes.
+The most important bugfixes are:
+
+- ``spawn`` can handle the empty seqs/strings that are internally
+  represented as ``nil``.
+- The most pressing bugs of the documentation generator have been fixed.
+- async streaming of the httpclient has been fixed.
 
 
-## Changes affecting backwards compatibility
-
-
-### Breaking changes in the standard library
-
-
-### Breaking changes in the compiler
-
-
-## Library additions
-
-
-## Library changes
-
-- Added `os.delEnv` and `nimscript.delEnv`. (#11466)
-
-- Enable Oid usage in hashtables. (#11472)
-
-#
-# Language additions
-
-
-## Language changes
-
-
-### Tool changes
-
-
-### Compiler changes
-
-- VM can now cast integer type arbitrarily. (#11459)
-
-
-## Bugfixes
+### Bugfixes

changelog_v020.md

@@ -1,31 +0,0 @@
-# v0.20.4 - xxxx-xx-xx
-
-
-## Changes affecting backwards compatibility
-
-
-### Breaking changes in the standard library
-
-
-### Breaking changes in the compiler
-
-
-## Library additions
-
-
-## Library changes
-
-
-## Language additions
-
-
-## Language changes
-
-
-### Tool changes
-
-
-### Compiler changes
-
-
-## Bugfixes

changelogs/changelog_0_19_0.md

@@ -223,7 +223,7 @@
   into the namespace "Nim" in order to avoid naming conflicts with existing
   C++ code. This is done for all Nim code - internal and exported.
 
-- Added ``macros.getProjectPath`` and ``os.putEnv`` procs to Nim's virtual
+- Added ``macros.getProjectPath`` and ``ospaths.putEnv`` procs to Nim's virtual
   machine.
 
 - The ``deadCodeElim`` option is now always turned on and the switch has no

changelogs/changelog_0_20_0.md

@@ -1,294 +0,0 @@
-## v0.20.0 - 2019-06-06
-
-
-### Changes affecting backwards compatibility
-
-- `shr` is now sign preserving. Use `-d:nimOldShiftRight` to enable
-  the old behavior globally.
-
-- The ``isLower``, ``isUpper`` family of procs in strutils/unicode
-  operating on **strings** have been
-  deprecated since it was unclear what these do. Note that the much more
-  useful procs that operate on ``char`` or ``Rune`` are not affected.
-
-- `strutils.editDistance` has been deprecated,
-  use `editdistance.editDistance` or `editdistance.editDistanceAscii`
-  instead.
-
-- The OpenMP parallel iterator \``||`\` now supports any `#pragma omp directive`
-  and not just `#pragma omp parallel for`. See
-  [OpenMP documentation](https://www.openmp.org/wp-content/uploads/OpenMP-4.5-1115-CPP-web.pdf).
-
-  The default annotation is `parallel for`, if you used OpenMP without annotation
-  the change is transparent, if you used annotations you will have to prefix
-  your previous annotations with `parallel for`.
-
-  Furthermore, an overload with positive stepping is available.
-
-- The `unchecked` pragma was removed, instead use `system.UncheckedArray`.
-
-- The undocumented ``#? strongSpaces`` parsing mode has been removed.
-
-- The `not` operator is now always a unary operator, this means that code like
-  ``assert not isFalse(3)`` compiles.
-
-- `getImpl` on a `var` or `let` symbol will now return the full `IdentDefs`
-  tree from the symbol declaration instead of just the initializer portion.
-
-- Methods are now ordinary "single" methods, only the first parameter is
-  used to select the variant at runtime. For backwards compatibility
-  use the new `--multimethods:on` switch.
-
-- Generic methods are now deprecated; they never worked well.
-
-- Compile time checks for integer and float conversions are now stricter.
-  For example, `const x = uint32(-1)` now gives a compile time error instead
-  of being equivalent to `const x = 0xFFFFFFFF'u32`.
-
-- Using `typed` as the result type in templates/macros now means
-  "expression with a type". The old meaning of `typed` is preserved
-  as `void` or no result type at all.
-
-- A bug allowed `macro foo(): int = 123` to compile even though a
-  macro has to return a `NimNode`. This has been fixed.
-
-- With the exception of `uint` and `uint64`, conversion to unsigned types
-  are now range checked during runtime.
-
-- Macro arguments of type `typedesc` are now passed to the macro as
-  `NimNode` like every other type except `static`. Use `typed` for a
-  behavior that is identical in new and old
-  Nim. See the RFC [Pass typedesc as NimNode to macros](https://github.com/nim-lang/RFCs/issues/148)
-  for more details.
-
-
-#### Breaking changes in the standard library
-
-- `osproc.execProcess` now also takes a `workingDir` parameter.
-
-- `std/sha1.secureHash` now accepts `openArray[char]`, not `string`. (Former
-   successful matches should keep working, though former failures will not.)
-
-- `options.UnpackError` is no longer a ref type and inherits from `system.Defect`
-  instead of `system.ValueError`.
-
-- `system.ValueError` now inherits from `system.CatchableError` instead of `system.Defect`.
-
-- The procs `parseutils.parseBiggestInt`, `parseutils.parseInt`,
-  `parseutils.parseBiggestUInt` and `parseutils.parseUInt` now raise a
-  `ValueError` when the parsed integer is outside of the valid range.
-  Previously they sometimes raised an `OverflowError` and sometimes they
-  returned `0`.
-
-- The procs `parseutils.parseBin`, `parseutils.parseOct` and `parseutils.parseHex`
-  were not clearing their `var` parameter `number` and used to push its value to
-  the left when storing the parsed string into it. Now they always set the value
-  of the parameter to `0` before storing the result of the parsing, unless the
-  string to parse is not valid (then the value of `number` is not changed).
-
-- `streams.StreamObject` now restricts its fields to only raise `system.Defect`,
-  `system.IOError` and `system.OSError`.
-  This change only affects custom stream implementations.
-
-- nre's `RegexMatch.{captureBounds,captures}[]`  no longer return `Option` or
-  `nil`/`""`, respectively. Use the newly added `n in p.captures` method to
-  check if a group is captured, otherwise you'll receive an exception.
-
-- nre's `RegexMatch.{captureBounds,captures}.toTable` no longer accept a
-  default parameter. Instead uncaptured entries are left empty. Use
-  `Table.getOrDefault()` if you need defaults.
-
-- nre's `RegexMatch.captures.{items,toSeq}` now returns an `Option[string]`
-  instead of a `string`. With the removal of `nil` strings, this is the only
-  way to indicate a missing match. Inside your loops, instead
-  of `capture == ""` or `capture == nil`, use `capture.isSome` to check if a capture is
-  present, and `capture.get` to get its value.
-
-- nre's `replace()` no longer throws `ValueError` when the replacement string
-  has missing captures. It instead throws `KeyError` for named captures, and
-  `IndexError` for unnamed captures. This is consistent with
-  `RegexMatch.{captureBounds,captures}[]`.
-
-- `splitFile` now correctly handles edge cases, see #10047.
-
-- `isNil` is no longer false for undefined in the JavaScript backend:
-  now it's true for both nil and undefined.
-  Use `isNull` or `isUndefined` if you need exact equality:
-  `isNil` is consistent with `===`, `isNull` and `isUndefined` with `==`.
-
-- several deprecated modules were removed: `ssl`, `matchers`, `httpserver`,
-  `unsigned`, `actors`, `parseurl`
-
-- two poorly documented and not used modules (`subexes`, `scgi`) were moved to
-  graveyard (they are available as Nimble packages)
-
-- procs `string.add(int)` and `string.add(float)` which implicitly convert
-  ints and floats to string have been deprecated.
-  Use `string.addInt(int)` and `string.addFloat(float)` instead.
-
-- ``case object`` branch transitions via ``system.reset`` are deprecated.
-  Compile your code with ``-d:nimOldCaseObjects`` for a transition period.
-
-- base64 module: The default parameter `newLine` for the `encode` procs
-  was changed from `"\13\10"` to the empty string `""`.
-
-
-#### Breaking changes in the compiler
-
-- The compiler now implements the "generic symbol prepass" for `when` statements
-  in generics, see bug #8603. This means that code like this does not compile
-  anymore:
-
-```nim
-proc enumToString*(enums: openArray[enum]): string =
-  # typo: 'e' instead 'enums'
-  when e.low.ord >= 0 and e.high.ord < 256:
-    result = newString(enums.len)
-  else:
-    result = newString(enums.len * 2)
-```
-
-- ``discard x`` is now illegal when `x` is a function symbol.
-
-- Implicit imports via ``--import: module`` in a config file are now restricted
-  to the main package.
-
-
-### Library additions
-
-- There is a new stdlib module `std/editdistance` as a replacement for the
-  deprecated `strutils.editDistance`.
-
-- There is a new stdlib module `std/wordwrap` as a replacement for the
-  deprecated `strutils.wordwrap`.
-
-- Added `split`, `splitWhitespace`, `size`, `alignLeft`, `align`,
-  `strip`, `repeat` procs and iterators to `unicode.nim`.
-
-- Added `or` for `NimNode` in `macros`.
-
-- Added `system.typeof` for more control over how `type` expressions
-  can be deduced.
-
-- Added `macros.isInstantiationOf` for checking if the proc symbol
-  is instantiation of generic proc symbol.
-
-- Added the parameter ``isSorted`` for the ``sequtils.deduplicate`` proc.
-
-- There is a new stdlib module `std/diff` to compute the famous "diff"
-  of two texts by line.
-
-- Added `os.relativePath`.
-
-- Added `parseopt.remainingArgs`.
-
-- Added `os.getCurrentCompilerExe` (implemented as `getAppFilename` at CT),
-  can be used to retrieve the currently executing compiler.
-
-- Added `xmltree.toXmlAttributes`.
-
-- Added ``std/sums`` module for fast summation functions.
-
-- Added `Rusage`, `getrusage`, `wait4` to the posix interface.
-
-- Added the `posix_utils` module.
-
-- Added `system.default`.
-
-- Added `sequtils.items` for closure iterators, allows closure iterators
-  to be used by the `mapIt`, `filterIt`, `allIt`, `anyIt`, etc.
-
-
-### Library changes
-
-- The string output of `macros.lispRepr` proc has been tweaked
-  slightly. The `dumpLisp` macro in this module now outputs an
-  indented proper Lisp, devoid of commas.
-
-- Added `macros.signatureHash` that returns a stable identifier
-  derived from the signature of a symbol.
-
-- In `strutils` empty strings now no longer matched as substrings
-  anymore.
-
-- The `Complex` type is now a generic object and not a tuple anymore.
-
-- The `ospaths` module is now deprecated, use `os` instead. Note that
-  `os` is available in a NimScript environment but unsupported
-  operations produce a compile-time error.
-
-- The `parseopt` module now supports a new flag `allowWhitespaceAfterColon`
-  (default value: true) that can be set to `false` for better Posix
-  interoperability. (Bug #9619.)
-
-- `os.joinPath` and `os.normalizePath` handle edge cases like ``"a/b/../../.."``
-  differently.
-
-- `securehash` was moved to `lib/deprecated`.
-
-- The switch ``-d:useWinAnsi`` is not supported anymore.
-
-- In `times` module, procs `format` and `parse` accept a new optional
-  `DateTimeLocale` argument for formatting/parsing dates in other languages.
-
-
-### Language additions
-
-- Vm support for float32<->int32 and float64<->int64 casts was added.
-- There is a new pragma block `noSideEffect` that works like
-  the `gcsafe` pragma block.
-- added `os.getCurrentProcessId`.
-- User defined pragmas are now allowed in the pragma blocks.
-- Pragma blocks are no longer eliminated from the typed AST tree to preserve
-  pragmas for further analysis by macros.
-- Custom pragmas are now supported for `var` and `let` symbols.
-- Tuple unpacking is now supported for constants and for loop variables.
-- Case object branches can be initialized with a runtime discriminator if
-  possible discriminator values are constrained within a case statement.
-
-### Language changes
-
-- The standard extension for SCF (source code filters) files was changed from
-  ``.tmpl`` to ``.nimf``,
-  it's more recognizable and allows tools like Github to recognize it as Nim,
-  see [#9647](https://github.com/nim-lang/Nim/issues/9647).
-  The previous extension will continue to work.
-
-- Pragma syntax is now consistent. Previous syntax where type pragmas did not
-  follow the type name is now deprecated. Also pragma before generic parameter
-  list is deprecated to be consistent with how pragmas are used with a proc. See
-  [#8514](https://github.com/nim-lang/Nim/issues/8514) and
-  [#1872](https://github.com/nim-lang/Nim/issues/1872) for further details.
-
-- Hash sets and tables are initialized by default. The explicit `initHashSet`,
-  `initTable`, etc. are not needed anymore.
-
-
-### Tool changes
-
-- `jsondoc` now includes a `moduleDescription` field with the module
-  description. `jsondoc0` shows comments as its own objects as shown in the
-  documentation.
-- `nimpretty`: --backup now defaults to `off` instead of `on` and the flag was
-  undocumented; use `git` instead of relying on backup files.
-- `koch` now defaults to build the latest *stable* Nimble version unless you
-  explicitly ask for the latest master version via `--latest`.
-
-
-### Compiler changes
-
-- The deprecated `fmod` proc is now unavailable on the VM.
-- A new `--outdir` option was added.
-- The compiled JavaScript file for the project produced by executing `nim js`
-  will no longer be placed in the nimcache directory.
-- The `--hotCodeReloading` has been implemented for the native targets.
-  The compiler also provides a new more flexible API for handling the
-  hot code reloading events in the code.
-- The compiler now supports a ``--expandMacro:macroNameHere`` switch
-  for easy introspection into what a macro expands into.
-- The `-d:release` switch now does not disable runtime checks anymore.
-  For a release build that also disables runtime checks
-  use `-d:release -d:danger` or simply `-d:danger`.
-
-
-### Bugfixes

changelogs/changelog_0_20_2.md

@@ -1,55 +0,0 @@
-# v0.20.2 - 2019-07-17
-
-
-## Changes affecting backwards compatibility
-
-- All `strutils.rfind` procs now take `start` and `last` like `strutils.find`
-  with the same data slice/index meaning. This is backwards compatible for
-  calls *not* changing the `rfind` `start` parameter from its default. (#11487)
-
-  In the unlikely case that you were using `rfind X, start=N`, or `rfind X, N`,
-  then you need to change that to `rfind X, last=N` or `rfind X, 0, N`. (This
-  should minimize gotchas porting code from other languages like Python or C++.)
-
-- On Windows stderr/stdout/stdin are not opened as binary files anymore. Use the switch
-  `-d:nimBinaryStdFiles` for a transition period.
-
-### Breaking changes in the standard library
-
-- Mac OS X / BSD: TSa_Family is now the ``uint8`` type, so type
-  conversions like ``x.sin_family = uint16 toInt(nativesockets.AF_INET)``
-  need to be changed into ``x.sin_family = TSa_Family toInt(nativesockets.AF_INET)``.
-
-
-### Breaking changes in the compiler
-
-
-## Library additions
-
-- `toOpenArray` is now available for the JS target.
-
-## Library changes
-
-- Fix async IO operations stalling even after socket is closed. (#11232)
-
-- More informative error message for `streams.openFileStream`. (#11438)
-
-
-## Language additions
-
-
-## Language changes
-
-
-### Tool changes
-
-
-### Compiler changes
-
-- Better error message for IndexError for empty containers. (#11476)
-
-- Fix regression in semfold for old right shift. (#11477)
-
-- Fix for passing tuples as static params to macros. (#11423)
-
-## Bugfixes

changelogs/changelog_X_XX_X.md

@@ -1,33 +1,27 @@
-# vx.xx.x - yyyy-mm-dd
+## v0.XX.0 - XX/XX/2018
+
+### Changes affecting backwards compatibility
+
+- Example item: ``Foo`` changed to ``Bar``.
+
+#### Breaking changes in the standard library
 
 
-## Changes affecting backwards compatibility
+#### Breaking changes in the compiler
 
-- Example item: `Foo` changed to `Bar`.
+### Library additions
+
+### Library changes
 
 
-### Breaking changes in the standard library
+### Language additions
 
 
-### Breaking changes in the compiler
-
-
-## Library additions
-
-
-## Library changes
-
-
-## Language additions
-
-
-## Language changes
+### Language changes
 
 
 ### Tool changes
 
-
 ### Compiler changes
 
-
-## Bugfixes
+### Bugfixes

compiler.nimble

@@ -4,6 +4,6 @@ author = "Andreas Rumpf"
 description = "Compiler package providing the compiler sources as a library."
 license = "MIT"
 
-installDirs = @["compiler", "nimsuggest"]
+installDirs = @["compiler"]
 
 requires "nim >= 0.14.0"

compiler/aliases.nim

@@ -10,7 +10,7 @@
 ## Simple alias analysis for the HLO and the code generators.
 
 import
-  ast, astalgo, types, trees, intsets
+  ast, astalgo, types, trees, intsets, msgs
 
 type
   TAnalysisResult* = enum
@@ -22,14 +22,14 @@ proc isPartOfAux(n: PNode, b: PType, marker: var IntSet): TAnalysisResult =
   result = arNo
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       result = isPartOfAux(n.sons[i], b, marker)
       if result == arYes: return
   of nkRecCase:
     assert(n.sons[0].kind == nkSym)
     result = isPartOfAux(n.sons[0], b, marker)
     if result == arYes: return
-    for i in 1 ..< sonsLen(n):
+    for i in countup(1, sonsLen(n) - 1):
       case n.sons[i].kind
       of nkOfBranch, nkElse:
         result = isPartOfAux(lastSon(n.sons[i]), b, marker)
@@ -52,7 +52,7 @@ proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult =
   of tyGenericInst, tyDistinct, tyAlias, tySink:
     result = isPartOfAux(lastSon(a), b, marker)
   of tyArray, tySet, tyTuple:
-    for i in 0 ..< sonsLen(a):
+    for i in countup(0, sonsLen(a) - 1):
       result = isPartOfAux(a.sons[i], b, marker)
       if result == arYes: return
   else: discard
@@ -186,7 +186,7 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
         if res != arNo:
           result = res
           if res == arYes: break
-    of nkCallKinds:
+    of nkCall:
       result = arNo
       for i in 1 ..< b.len:
         let res = isPartOf(a, b[i])

compiler/asciitables.nim

@@ -1,83 +0,0 @@
-#[
-move to std/asciitables.nim once stable, or to a nimble paackage
-once compiler can depend on nimble
-]#
-
-type Cell* = object
-  text*: string
-  width*, row*, col*, ncols*, nrows*: int
-
-iterator parseTableCells*(s: string, delim = '\t'): Cell =
-  ## iterates over all cells in a `delim`-delimited `s`, after a 1st
-  ## pass that computes number of rows, columns, and width of each column.
-  var widths: seq[int]
-  var cell: Cell
-  template update() =
-    if widths.len<=cell.col:
-      widths.setLen cell.col+1
-      widths[cell.col] = cell.width
-    else:
-      widths[cell.col] = max(widths[cell.col], cell.width)
-    cell.width = 0
-
-  for a in s:
-    case a
-    of '\n':
-      update()
-      cell.col = 0
-      cell.row.inc
-    elif a == delim:
-      update()
-      cell.col.inc
-    else:
-      # todo: consider multi-width chars when porting to non-ascii implementation
-      cell.width.inc
-  if s.len > 0 and s[^1] != '\n':
-    update()
-
-  cell.ncols = widths.len
-  cell.nrows = cell.row + 1
-  cell.row = 0
-  cell.col = 0
-  cell.width = 0
-
-  template update2() =
-    cell.width = widths[cell.col]
-    yield cell
-    cell.text = ""
-    cell.width = 0
-    cell.col.inc
-
-  template finishRow() =
-    for col in cell.col..<cell.ncols:
-      cell.col = col
-      update2()
-    cell.col = 0
-
-  for a in s:
-    case a
-    of '\n':
-      finishRow()
-      cell.row.inc
-    elif a == delim:
-      update2()
-    else:
-      cell.width.inc
-      cell.text.add a
-
-  if s.len > 0 and s[^1] != '\n':
-    finishRow()
-
-proc alignTable*(s: string, delim = '\t', fill = ' ', sep = " "): string =
-  ## formats a `delim`-delimited `s` representing a table; each cell is aligned
-  ## to a width that's computed for each column; consecutive columns are
-  ## delimted by `sep`, and alignment space is filled using `fill`.
-  ## More customized formatting can be done by calling `parseTableCells` directly.
-  for cell in parseTableCells(s, delim):
-    result.add cell.text
-    for i in cell.text.len..<cell.width:
-      result.add fill
-    if cell.col < cell.ncols-1:
-      result.add sep
-    if cell.col == cell.ncols-1 and cell.row < cell.nrows - 1:
-      result.add '\n'

compiler/ast.nim

@@ -10,7 +10,8 @@
 # abstract syntax tree + symbol table
 
 import
-  lineinfos, hashes, options, ropes, idents, idgen
+  lineinfos, hashes, nversion, options, strutils, std / sha1, ropes, idents,
+  intsets, idgen
 
 type
   TCallingConvention* = enum
@@ -215,7 +216,7 @@ type
     nkEnumFieldDef,       # `ident = expr` in an enumeration
     nkArgList,            # argument list
     nkPattern,            # a special pattern; used for matching
-    nkHiddenTryStmt,      # token used for interpretation
+    nkReturnToken,        # token used for interpretation
     nkClosure,            # (prc, env)-pair (internally used for code gen)
     nkGotoState,          # used for the state machine (for iterators)
     nkState,              # give a label to a code section (for iterators)
@@ -226,7 +227,7 @@ type
   TNodeKinds* = set[TNodeKind]
 
 type
-  TSymFlag* = enum    # already 36 flags!
+  TSymFlag* = enum    # already 33 flags!
     sfUsed,           # read access of sym (for warnings) or simply used
     sfExported,       # symbol is exported from module
     sfFromGeneric,    # symbol is instantiation of a generic; this is needed
@@ -270,27 +271,16 @@ type
                       # language; for interfacing with Objective C
     sfDiscardable,    # returned value may be discarded implicitly
     sfOverriden,      # proc is overriden
-    sfCallsite        # A flag for template symbols to tell the
-                      # compiler it should use line information from
-                      # the calling side of the macro, not from the
-                      # implementation.
     sfGenSym          # symbol is 'gensym'ed; do not add to symbol table
-    sfNonReloadable   # symbol will be left as-is when hot code reloading is on -
-                      # meaning that it won't be renamed and/or changed in any way
-    sfGeneratedOp     # proc is a generated '='; do not inject destructors in it
-                      # variable is generated closure environment; requires early
-                      # destruction for --newruntime.
-
 
   TSymFlags* = set[TSymFlag]
 
 const
   sfNoInit* = sfMainModule       # don't generate code to init the variable
 
-  sfCursor* = sfDispatcher
-    # local variable has been computed to be a "cursor".
-    # see cursors.nim for details about what that means.
-
+  sfImmediate* = sfDispatcher
+    # macro or template is immediately expanded
+    # without considering any possible overloads
   sfAllUntyped* = sfVolatile # macro or template is immediately expanded \
     # in a generic context
 
@@ -344,7 +334,7 @@ type
                      # (apparently something with bootstrapping)
                      # if you need to add a type, they can apparently be reused
     tyNone, tyBool, tyChar,
-    tyEmpty, tyAlias, tyNil, tyUntyped, tyTyped, tyTypeDesc,
+    tyEmpty, tyAlias, tyNil, tyExpr, tyStmt, tyTypeDesc,
     tyGenericInvocation, # ``T[a, b]`` for types to invoke
     tyGenericBody,       # ``T[a, b, body]`` last parameter is the body
     tyGenericInst,       # ``T[a, b, realInstance]`` instantiated generic type
@@ -371,11 +361,9 @@ type
     tyInt, tyInt8, tyInt16, tyInt32, tyInt64, # signed integers
     tyFloat, tyFloat32, tyFloat64, tyFloat128,
     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64,
-    tyOwned, tySink, tyLent,
+    tyOptAsRef, tySink, tyLent,
     tyVarargs,
-    tyUncheckedArray
-      # An array with boundaries [0,+∞]
-
+    tyUnused,
     tyProxy # used as errornous type (for idetools)
 
     tyBuiltInTypeClass
@@ -445,7 +433,7 @@ const
                     tyUserTypeClass, tyUserTypeClassInst,
                     tyAnd, tyOr, tyNot, tyAnything}
 
-  tyMetaTypes* = {tyGenericParam, tyTypeDesc, tyUntyped} + tyTypeClasses
+  tyMetaTypes* = {tyGenericParam, tyTypeDesc, tyExpr} + tyTypeClasses
   tyUserTypeClasses* = {tyUserTypeClass, tyUserTypeClassInst}
 
 type
@@ -474,16 +462,15 @@ type
     nfDefaultParam # an automatically inserter default parameter
     nfDefaultRefsParam # a default param value references another parameter
                        # the flag is applied to proc default values and to calls
-    nfExecuteOnReload  # A top-level statement that will be executed during reloads
 
   TNodeFlags* = set[TNodeFlag]
-  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: ~38)
+  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: beyond that)
     tfVarargs,        # procedure has C styled varargs
                       # tyArray type represeting a varargs list
     tfNoSideEffect,   # procedure type does not allow side effects
     tfFinal,          # is the object final?
     tfInheritable,    # is the object inheritable?
-    tfHasOwned,       # type contains an 'owned' type and must be moved
+    tfAcyclic,        # type is acyclic (for GC optimization)
     tfEnumHasHoles,   # enum cannot be mapped into a range
     tfShallow,        # type can be shallow copied on assignment
     tfThread,         # proc type is marked as ``thread``; alias for ``gcsafe``
@@ -534,8 +521,6 @@ type
     tfCovariant       # covariant generic param mimicing a ptr type
     tfWeakCovariant   # covariant generic param mimicing a seq/array type
     tfContravariant   # contravariant generic param
-    tfCheckedForDestructor # type was checked for having a destructor.
-                           # If it has one, t.destructor is not nil.
 
   TTypeFlags* = set[TTypeFlag]
 
@@ -578,9 +563,13 @@ const
   routineKinds* = {skProc, skFunc, skMethod, skIterator,
                    skConverter, skMacro, skTemplate}
   tfIncompleteStruct* = tfVarargs
+  tfUncheckedArray* = tfVarargs
   tfUnion* = tfNoSideEffect
   tfGcSafe* = tfThread
   tfObjHasKids* = tfEnumHasHoles
+  tfOldSchoolExprStmt* = tfVarargs # for now used to distinguish \
+    # 'varargs[expr]' from 'varargs[untyped]'. Eventually 'expr' will be
+    # deprecated and this mess can be cleaned up.
   tfReturnsNew* = tfInheritable
   skError* = skUnknown
 
@@ -591,8 +580,7 @@ type
   TMagic* = enum # symbols that require compiler magic:
     mNone,
     mDefined, mDefinedInScope, mCompiles, mArrGet, mArrPut, mAsgn,
-    mLow, mHigh, mSizeOf, mAlignOf, mOffsetOf, mTypeTrait,
-    mIs, mOf, mAddr, mType, mTypeOf,
+    mLow, mHigh, mSizeOf, mTypeTrait, mIs, mOf, mAddr, mType, mTypeOf,
     mRoof, mPlugin, mEcho, mShallowCopy, mSlurp, mStaticExec, mStatic,
     mParseExprToAst, mParseStmtToAst, mExpandToAst, mQuoteAst,
     mUnaryLt, mInc, mDec, mOrd,
@@ -620,6 +608,10 @@ type
     mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot,
     mUnaryPlusI, mBitnotI,
     mUnaryPlusF64, mUnaryMinusF64, mAbsF64,
+    mZe8ToI, mZe8ToI64,
+    mZe16ToI, mZe16ToI64,
+    mZe32ToI64, mZeIToI64,
+    mToU8, mToU16, mToU32,
     mToFloat, mToBiggestFloat,
     mToInt, mToBiggestInt,
     mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
@@ -636,8 +628,8 @@ type
     mIsPartOf, mAstToStr, mParallel,
     mSwap, mIsNil, mArrToSeq, mCopyStr, mCopyStrLast,
     mNewString, mNewStringOfCap, mParseBiggestFloat,
-    mMove, mWasMoved, mDestroy,
-    mDefault, mUnown, mAccessEnv, mReset,
+    mMove, mWasMoved,
+    mReset,
     mArray, mOpenArray, mRange, mSet, mSeq, mOpt, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
     mOrdinal,
@@ -649,6 +641,7 @@ type
     mVoidType, mPNimrodNode, mShared, mGuarded, mLock, mSpawn, mDeepCopy,
     mIsMainModule, mCompileDate, mCompileTime, mProcCall,
     mCpuEndian, mHostOS, mHostCPU, mBuildOS, mBuildCPU, mAppType,
+    mNaN, mInf, mNegInf,
     mCompileOption, mCompileOptionArg,
     mNLen, mNChild, mNSetChild, mNAdd, mNAddMultiple, mNDel,
     mNKind, mNSymKind,
@@ -658,21 +651,20 @@ type
 
     mNIntVal, mNFloatVal, mNSymbol, mNIdent, mNGetType, mNStrVal, mNSetIntVal,
     mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo,
-    mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mNSigHash, mNSizeOf,
+    mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent,
     mNBindSym, mLocals, mNCallSite,
     mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym,
     mNHint, mNWarning, mNError,
     mInstantiationInfo, mGetTypeInfo,
-    mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples,
-    mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf,
-    mSymIsInstantiationOf
+    mNimvm, mIntDefine, mStrDefine, mRunnableExamples,
+    mException, mBuiltinType, mSymOwner
 
 # things that we can evaluate safely at compile time, even if not asked for it:
 const
   ctfeWhitelist* = {mNone, mUnaryLt, mSucc,
     mPred, mInc, mDec, mOrd, mLengthOpenArray,
     mLengthStr, mLengthArray, mLengthSeq, mXLenStr, mXLenSeq,
-    mArrGet, mArrPut, mAsgn, mDestroy,
+    mArrGet, mArrPut, mAsgn,
     mIncl, mExcl, mCard, mChr,
     mAddI, mSubI, mMulI, mDivI, mModI,
     mAddF64, mSubF64, mMulF64, mDivF64,
@@ -690,6 +682,10 @@ const
     mEqRef, mEqProc, mEqUntracedRef, mLePtr, mLtPtr, mEqCString, mXor,
     mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI,
     mUnaryPlusF64, mUnaryMinusF64, mAbsF64,
+    mZe8ToI, mZe8ToI64,
+    mZe16ToI, mZe16ToI64,
+    mZe32ToI64, mZeIToI64,
+    mToU8, mToU16, mToU32,
     mToFloat, mToBiggestFloat,
     mToInt, mToBiggestInt,
     mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
@@ -700,6 +696,11 @@ const
     mConStrStr, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
     mInRange, mInSet, mRepr,
     mCopyStr, mCopyStrLast}
+  # magics that require special semantic checking and
+  # thus cannot be overloaded (also documented in the spec!):
+  SpecialSemMagics* = {
+    mDefined, mDefinedInScope, mCompiles, mLow, mHigh, mSizeOf, mIs, mOf,
+    mShallowCopy, mExpandToAst, mParallel, mSpawn, mAstToStr}
 
 type
   PNode* = ref TNode
@@ -727,9 +728,10 @@ type
       sons*: TNodeSeq
     comment*: string
 
+  TSymSeq* = seq[PSym]
   TStrTable* = object         # a table[PIdent] of PSym
     counter*: int
-    data*: seq[PSym]
+    data*: TSymSeq
 
   # -------------- backend information -------------------------------
   TLocKind* = enum
@@ -756,14 +758,12 @@ type
     lfHeader,                 # include header file for symbol
     lfImportCompilerProc,     # ``importc`` of a compilerproc
     lfSingleUse               # no location yet and will only be used once
-    lfEnforceDeref            # a copyMem is required to dereference if this a
-                              # ptr array due to C array limitations.
-                              # See #1181, #6422, #11171
-    lfPrepareForMutation      # string location is about to be mutated (V2)
   TStorageLoc* = enum
     OnUnknown,                # location is unknown (stack, heap or static)
     OnStatic,                 # in a static section
     OnStack,                  # location is on hardware stack
+    OnStackShadowDup,         # location is on the stack but also replicated
+                              # on the shadow stack
     OnHeap                    # location is on heap or global
                               # (reference counting needed)
   TLocFlags* = set[TLocFlag]
@@ -773,6 +773,7 @@ type
     flags*: TLocFlags         # location's flags
     lode*: PNode              # Node where the location came from; can be faked
     r*: Rope                  # rope value of location (code generators)
+    dup*: Rope                # duplicated location for precise stack scans
 
   # ---------------- end of backend information ------------------------------
 
@@ -812,7 +813,7 @@ type
     of routineKinds:
       procInstCache*: seq[PInstantiation]
       gcUnsafetyReason*: PSym  # for better error messages wrt gcsafe
-      transformedBody*: PNode  # cached body after transf pass
+      #scope*: PScope          # the scope where the proc was defined
     of skModule, skPackage:
       # modules keep track of the generic symbols they use from other modules.
       # this is because in incremental compilation, when a module is about to
@@ -858,7 +859,7 @@ type
     offset*: int              # offset of record field
     loc*: TLoc
     annex*: PLib              # additional fields (seldom used, so we use a
-                              # reference to another object to save space)
+                              # reference to another object to safe space)
     constraint*: PNode        # additional constraints like 'lit|result'; also
                               # misused for the codegenDecl pragma in the hope
                               # it won't cause problems
@@ -869,13 +870,6 @@ type
 
   TTypeSeq* = seq[PType]
   TLockLevel* = distinct int16
-
-  TTypeAttachedOp* = enum ## as usual, order is important here
-    attachedDestructor,
-    attachedAsgn,
-    attachedSink,
-    attachedDeepCopy
-
   TType* {.acyclic.} = object of TIdObj # \
                               # types are identical iff they have the
                               # same id; there may be multiple copies of a type
@@ -896,7 +890,12 @@ type
     owner*: PSym              # the 'owner' of the type
     sym*: PSym                # types have the sym associated with them
                               # it is used for converting types to strings
-    attachedOps*: array[TTypeAttachedOp, PSym] # destructors, etc.
+    destructor*: PSym         # destructor. warning: nil here may not necessary
+                              # mean that there is no destructor.
+                              # see instantiateDestructor in semdestruct.nim
+    deepCopy*: PSym           # overriden 'deepCopy' operation
+    assignment*: PSym         # overriden '=' operation
+    sink*: PSym               # overriden '=sink' operation
     methods*: seq[(int,PSym)] # attached methods
     size*: BiggestInt         # the size of the type in bytes
                               # -1 means that the size is unkwown
@@ -905,8 +904,6 @@ type
     loc*: TLoc
     typeInst*: PType          # for generic instantiations the tyGenericInst that led to this
                               # type.
-    uniqueId*: int            # due to a design mistake, we need to keep the real ID here as it
-                              # required by the --incremental:on mode.
 
   TPair* = object
     key*, val*: RootRef
@@ -984,8 +981,7 @@ const
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
                                       nfDotSetter, nfDotField,
                                       nfIsRef, nfPreventCg, nfLL,
-                                      nfFromTemplate, nfDefaultRefsParam,
-                                      nfExecuteOnReload}
+                                      nfFromTemplate, nfDefaultRefsParam}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2
@@ -1097,13 +1093,9 @@ proc newSym*(symKind: TSymKind, name: PIdent, owner: PSym,
   result.id = getID()
   when debugIds:
     registerId(result)
-
-proc astdef*(s: PSym): PNode =
-  # get only the definition (initializer) portion of the ast
-  if s.ast != nil and s.ast.kind == nkIdentDefs:
-    s.ast[2]
-  else:
-    s.ast
+  #if result.id == 77131:
+  #  writeStacktrace()
+  #  echo name.s
 
 proc isMetaType*(t: PType): bool =
   return t.kind in tyMetaTypes or
@@ -1148,17 +1140,17 @@ const                         # for all kind of hash tables:
 proc copyStrTable*(dest: var TStrTable, src: TStrTable) =
   dest.counter = src.counter
   setLen(dest.data, len(src.data))
-  for i in 0 .. high(src.data): dest.data[i] = src.data[i]
+  for i in countup(0, high(src.data)): dest.data[i] = src.data[i]
 
 proc copyIdTable*(dest: var TIdTable, src: TIdTable) =
   dest.counter = src.counter
   newSeq(dest.data, len(src.data))
-  for i in 0 .. high(src.data): dest.data[i] = src.data[i]
+  for i in countup(0, high(src.data)): dest.data[i] = src.data[i]
 
 proc copyObjectSet*(dest: var TObjectSet, src: TObjectSet) =
   dest.counter = src.counter
   setLen(dest.data, len(src.data))
-  for i in 0 .. high(src.data): dest.data[i] = src.data[i]
+  for i in countup(0, high(src.data)): dest.data[i] = src.data[i]
 
 proc discardSons*(father: PNode) =
   when defined(nimNoNilSeqs):
@@ -1269,27 +1261,19 @@ const
   UnspecifiedLockLevel* = TLockLevel(-1'i16)
   MaxLockLevel* = 1000'i16
   UnknownLockLevel* = TLockLevel(1001'i16)
-  AttachedOpToStr*: array[TTypeAttachedOp, string] = ["=destroy", "=", "=sink", "=deepcopy"]
 
 proc `$`*(x: TLockLevel): string =
   if x.ord == UnspecifiedLockLevel.ord: result = "<unspecified>"
   elif x.ord == UnknownLockLevel.ord: result = "<unknown>"
   else: result = $int16(x)
 
-proc `$`*(s: PSym): string =
-  if s != nil:
-    result = s.name.s & "@" & $s.id
-  else:
-    result = "<nil>"
-
 proc newType*(kind: TTypeKind, owner: PSym): PType =
   new(result)
   result.kind = kind
   result.owner = owner
   result.size = -1
-  result.align = -1            # default alignment
+  result.align = 2            # default alignment
   result.id = getID()
-  result.uniqueId = result.id
   result.lockLevel = UnspecifiedLockLevel
   when debugIds:
     registerId(result)
@@ -1336,7 +1320,10 @@ proc assignType*(dest, src: PType) =
   dest.n = src.n
   dest.size = src.size
   dest.align = src.align
-  dest.attachedOps = src.attachedOps
+  dest.destructor = src.destructor
+  dest.deepCopy = src.deepCopy
+  dest.sink = src.sink
+  dest.assignment = src.assignment
   dest.lockLevel = src.lockLevel
   # this fixes 'type TLock = TSysLock':
   if src.sym != nil:
@@ -1347,7 +1334,7 @@ proc assignType*(dest, src: PType) =
     else:
       dest.sym = src.sym
   newSons(dest, sonsLen(src))
-  for i in 0 ..< sonsLen(src): dest.sons[i] = src.sons[i]
+  for i in countup(0, sonsLen(src) - 1): dest.sons[i] = src.sons[i]
 
 proc copyType*(t: PType, owner: PSym, keepId: bool): PType =
   result = newType(t.kind, owner)
@@ -1360,11 +1347,15 @@ proc copyType*(t: PType, owner: PSym, keepId: bool): PType =
 
 proc exactReplica*(t: PType): PType = copyType(t, t.owner, true)
 
-proc copySym*(s: PSym): PSym =
+proc copySym*(s: PSym, keepId: bool = false): PSym =
   result = newSym(s.kind, s.name, s.owner, s.info, s.options)
   #result.ast = nil            # BUGFIX; was: s.ast which made problems
   result.typ = s.typ
-  when debugIds: registerId(result)
+  if keepId:
+    result.id = s.id
+  else:
+    result.id = getID()
+    when debugIds: registerId(result)
   result.flags = s.flags
   result.magic = s.magic
   if s.kind == skModule:
@@ -1473,13 +1464,6 @@ proc propagateToOwner*(owner, elem: PType) =
       o2.flags.incl tfHasAsgn
       owner.flags.incl tfHasAsgn
 
-  if tfHasOwned in elem.flags:
-    let o2 = owner.skipTypes({tyGenericInst, tyAlias, tySink})
-    if o2.kind in {tyTuple, tyObject, tyArray,
-                   tySequence, tyOpt, tySet, tyDistinct}:
-      o2.flags.incl tfHasOwned
-      owner.flags.incl tfHasOwned
-
   if owner.kind notin {tyProc, tyGenericInst, tyGenericBody,
                        tyGenericInvocation, tyPtr}:
     let elemB = elem.skipTypes({tyGenericInst, tyAlias, tySink})
@@ -1510,7 +1494,7 @@ proc delSon*(father: PNode, idx: int) =
   else:
     if isNil(father.sons): return
   var length = sonsLen(father)
-  for i in idx .. length - 2: father.sons[i] = father.sons[i + 1]
+  for i in countup(idx, length - 2): father.sons[i] = father.sons[i + 1]
   setLen(father.sons, length - 1)
 
 proc copyNode*(src: PNode): PNode =
@@ -1572,17 +1556,17 @@ proc copyTree*(src: PNode): PNode =
   of nkStrLit..nkTripleStrLit: result.strVal = src.strVal
   else:
     newSeq(result.sons, sonsLen(src))
-    for i in 0 ..< sonsLen(src):
+    for i in countup(0, sonsLen(src) - 1):
       result.sons[i] = copyTree(src.sons[i])
 
 proc hasSonWith*(n: PNode, kind: TNodeKind): bool =
-  for i in 0 ..< sonsLen(n):
+  for i in countup(0, sonsLen(n) - 1):
     if n.sons[i].kind == kind:
       return true
   result = false
 
 proc hasNilSon*(n: PNode): bool =
-  for i in 0 ..< safeLen(n):
+  for i in countup(0, safeLen(n) - 1):
     if n.sons[i] == nil:
       return true
     elif hasNilSon(n.sons[i]):
@@ -1594,14 +1578,14 @@ proc containsNode*(n: PNode, kinds: TNodeKinds): bool =
   case n.kind
   of nkEmpty..nkNilLit: result = n.kind in kinds
   else:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       if n.kind in kinds or containsNode(n.sons[i], kinds): return true
 
 proc hasSubnodeWith*(n: PNode, kind: TNodeKind): bool =
   case n.kind
   of nkEmpty..nkNilLit: result = n.kind == kind
   else:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       if (n.sons[i].kind == kind) or hasSubnodeWith(n.sons[i], kind):
         return true
     result = false
@@ -1676,11 +1660,6 @@ proc isCompileTimeProc*(s: PSym): bool {.inline.} =
   result = s.kind == skMacro or
            s.kind == skProc and sfCompileTime in s.flags
 
-proc isRunnableExamples*(n: PNode): bool =
-  # Templates and generics don't perform symbol lookups.
-  result = n.kind == nkSym and n.sym.magic == mRunnableExamples or
-    n.kind == nkIdent and n.ident.s == "runnableExamples"
-
 proc requiredParams*(s: PSym): int =
   # Returns the number of required params (without default values)
   # XXX: Perhaps we can store this in the `offset` field of the
@@ -1704,7 +1683,7 @@ proc isAtom*(n: PNode): bool {.inline.} =
 
 proc isEmptyType*(t: PType): bool {.inline.} =
   ## 'void' and 'stmt' types are often equivalent to 'nil' these days:
-  result = t == nil or t.kind in {tyVoid, tyTyped}
+  result = t == nil or t.kind in {tyVoid, tyStmt}
 
 proc makeStmtList*(n: PNode): PNode =
   if n.kind == nkStmtList:
@@ -1744,15 +1723,28 @@ proc toObject*(typ: PType): PType =
   if t.kind == tyRef: t.lastSon
   else: typ
 
-proc isImportedException*(t: PType; conf: ConfigRef): bool =
-  assert t != nil
+proc isException*(t: PType): bool =
+  # check if `y` is object type and it inherits from Exception
+  assert(t != nil)
 
+  if t.kind notin {tyObject, tyGenericInst}:
+    return false
+
+  var base = t
+  while base != nil and base.kind in {tyRef, tyObject, tyGenericInst}:
+    if base.sym != nil and base.sym.magic == mException:
+      return true
+    base = base.lastSon
+  return false
+
+proc isImportedException*(t: PType; conf: ConfigRef): bool =
+  assert(t != nil)
   if optNoCppExceptions in conf.globalOptions:
     return false
 
   let base = t.skipTypes({tyAlias, tyPtr, tyDistinct, tyGenericInst})
 
-  if base.sym != nil and {sfCompileToCpp, sfImportc} * base.sym.flags != {}:
+  if base.sym != nil and sfCompileToCpp in base.sym.flags:
     result = true
 
 proc isInfixAs*(n: PNode): bool =
@@ -1775,7 +1767,7 @@ when false:
     for i in 0 ..< n.safeLen:
       if n[i].containsNil: return true
 
-template hasDestructor*(t: PType): bool = {tfHasAsgn, tfHasOwned} * t.flags != {}
+template hasDestructor*(t: PType): bool = tfHasAsgn in t.flags
 template incompleteType*(t: PType): bool =
   t.sym != nil and {sfForward, sfNoForward} * t.sym.flags == {sfForward}
 
@@ -1783,36 +1775,3 @@ template typeCompleted*(s: PSym) =
   incl s.flags, sfNoForward
 
 template getBody*(s: PSym): PNode = s.ast[bodyPos]
-
-template detailedInfo*(sym: PSym): string =
-  sym.name.s
-
-proc isInlineIterator*(s: PSym): bool {.inline.} =
-  s.kind == skIterator and s.typ.callConv != ccClosure
-
-proc isClosureIterator*(s: PSym): bool {.inline.} =
-  s.kind == skIterator and s.typ.callConv == ccClosure
-
-proc isSinkParam*(s: PSym): bool {.inline.} =
-  s.kind == skParam and (s.typ.kind == tySink or tfHasOwned in s.typ.flags)
-
-proc isSinkType*(t: PType): bool {.inline.} =
-  t.kind == tySink or tfHasOwned in t.flags
-
-proc newProcType*(info: TLineInfo; owner: PSym): PType =
-  result = newType(tyProc, owner)
-  result.n = newNodeI(nkFormalParams, info)
-  rawAddSon(result, nil) # return type
-  # result.n[0] used to be `nkType`, but now it's `nkEffectList` because
-  # the effects are now stored in there too ... this is a bit hacky, but as
-  # usual we desperately try to save memory:
-  addSon(result.n, newNodeI(nkEffectList, info))
-
-proc addParam*(procType: PType; param: PSym) =
-  param.position = procType.len-1
-  addSon(procType.n, newSymNode(param))
-  rawAddSon(procType, param.typ)
-
-template destructor*(t: PType): PSym = t.attachedOps[attachedDestructor]
-template assignment*(t: PType): PSym = t.attachedOps[attachedAsgn]
-template asink*(t: PType): PSym = t.attachedOps[attachedSink]

compiler/astalgo.nim

@@ -152,14 +152,14 @@ proc lookupInRecord(n: PNode, field: PIdent): PSym =
   result = nil
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       result = lookupInRecord(n.sons[i], field)
       if result != nil: return
   of nkRecCase:
     if (n.sons[0].kind != nkSym): return nil
     result = lookupInRecord(n.sons[0], field)
     if result != nil: return
-    for i in 1 ..< sonsLen(n):
+    for i in countup(1, sonsLen(n) - 1):
       case n.sons[i].kind
       of nkOfBranch, nkElse:
         result = lookupInRecord(lastSon(n.sons[i]), field)
@@ -175,7 +175,7 @@ proc getModule*(s: PSym): PSym =
   while result != nil and result.kind != skModule: result = result.owner
 
 proc getSymFromList(list: PNode, ident: PIdent, start: int = 0): PSym =
-  for i in start ..< sonsLen(list):
+  for i in countup(start, sonsLen(list) - 1):
     if list.sons[i].kind == nkSym:
       result = list.sons[i].sym
       if result.name.id == ident.id: return
@@ -238,68 +238,65 @@ proc symToYamlAux(conf: ConfigRef; n: PSym, marker: var IntSet,
 proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet,
                    indent, maxRecDepth: int): Rope
 
+proc ropeConstr(indent: int, c: openArray[Rope]): Rope =
+  # array of (name, value) pairs
+  var istr = rspaces(indent + 2)
+  result = rope("{")
+  var i = 0
+  while i <= high(c):
+    if i > 0: add(result, ",")
+    addf(result, "$N$1\"$2\": $3", [istr, c[i], c[i + 1]])
+    inc(i, 2)
+  addf(result, "$N$1}", [rspaces(indent)])
+
 proc symToYamlAux(conf: ConfigRef; n: PSym, marker: var IntSet, indent: int,
                   maxRecDepth: int): Rope =
   if n == nil:
     result = rope("null")
   elif containsOrIncl(marker, n.id):
-    result = "\"$1\"" % [rope(n.name.s)]
+    result = "\"$1 @$2\"" % [rope(n.name.s), rope(
+        strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))]
   else:
     var ast = treeToYamlAux(conf, n.ast, marker, indent + 2, maxRecDepth - 1)
-                                 #rope("typ"), typeToYamlAux(conf, n.typ, marker,
-                                 #  indent + 2, maxRecDepth - 1),
-    let istr = rspaces(indent + 2)
-    result = rope("{")
-    addf(result, "$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
-    addf(result, "$N$1\"name\": $2", [istr, makeYamlString(n.name.s)])
-    addf(result, "$N$1\"typ\": $2", [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth - 1)])
-    if conf != nil:
-      # if we don't pass the config, we probably don't care about the line info
-      addf(result, "$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
-    if card(n.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
-    addf(result, "$N$1\"magic\": $2", [istr, makeYamlString($n.magic)])
-    addf(result, "$N$1\"ast\": $2", [istr, ast])
-    addf(result, "$N$1\"options\": $2", [istr, flagsToStr(n.options)])
-    addf(result, "$N$1\"position\": $2", [istr, rope(n.position)])
-    addf(result, "$N$1\"k\": $2", [istr, makeYamlString($n.loc.k)])
-    addf(result, "$N$1\"storage\": $2", [istr, makeYamlString($n.loc.storage)])
-    if card(n.loc.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, makeYamlString($n.loc.flags)])
-    addf(result, "$N$1\"r\": $2", [istr, n.loc.r])
-    addf(result, "$N$1\"lode\": $2", [istr, treeToYamlAux(conf, n.loc.lode, marker, indent + 2, maxRecDepth - 1)])
-    addf(result, "$N$1}", [rspaces(indent)])
+    result = ropeConstr(indent, [rope("kind"),
+                                 makeYamlString($n.kind),
+                                 rope("name"), makeYamlString(n.name.s),
+                                 rope("typ"), typeToYamlAux(conf, n.typ, marker,
+                                   indent + 2, maxRecDepth - 1),
+                                 rope("info"), lineInfoToStr(conf, n.info),
+                                 rope("flags"), flagsToStr(n.flags),
+                                 rope("magic"), makeYamlString($n.magic),
+                                 rope("ast"), ast, rope("options"),
+                                 flagsToStr(n.options), rope("position"),
+                                 rope(n.position)])
 
 proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int,
                    maxRecDepth: int): Rope =
-  var sonsRope: Rope
   if n == nil:
-    sonsRope = rope("null")
+    result = rope("null")
   elif containsOrIncl(marker, n.id):
-    sonsRope = "\"$1 @$2\"" % [rope($n.kind), rope(
+    result = "\"$1 @$2\"" % [rope($n.kind), rope(
         strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))]
   else:
     if sonsLen(n) > 0:
-      sonsRope = rope("[")
-      for i in 0 ..< sonsLen(n):
-        if i > 0: add(sonsRope, ",")
-        addf(sonsRope, "$N$1$2", [rspaces(indent + 4), typeToYamlAux(conf, n.sons[i],
+      result = rope("[")
+      for i in countup(0, sonsLen(n) - 1):
+        if i > 0: add(result, ",")
+        addf(result, "$N$1$2", [rspaces(indent + 4), typeToYamlAux(conf, n.sons[i],
             marker, indent + 4, maxRecDepth - 1)])
-      addf(sonsRope, "$N$1]", [rspaces(indent + 2)])
+      addf(result, "$N$1]", [rspaces(indent + 2)])
     else:
-      sonsRope = rope("null")
-
-    let istr = rspaces(indent + 2)
-    result = rope("{")
-    addf(result, "$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
-    addf(result, "$N$1\"sym\": $2",  [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth - 1)])
-    addf(result, "$N$1\"n\": $2",     [istr, treeToYamlAux(conf, n.n, marker, indent + 2, maxRecDepth - 1)])
-    if card(n.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
-    addf(result, "$N$1\"callconv\": $2", [istr, makeYamlString(CallingConvToStr[n.callConv])])
-    addf(result, "$N$1\"size\": $2", [istr, rope(n.size)])
-    addf(result, "$N$1\"align\": $2", [istr, rope(n.align)])
-    addf(result, "$N$1\"sons\": $2", [istr, sonsRope])
+      result = rope("null")
+    result = ropeConstr(indent, [rope("kind"),
+                                 makeYamlString($n.kind),
+                                 rope("sym"), symToYamlAux(conf, n.sym, marker,
+        indent + 2, maxRecDepth - 1), rope("n"), treeToYamlAux(conf, n.n, marker,
+        indent + 2, maxRecDepth - 1), rope("flags"), flagsToStr(n.flags),
+                                 rope("callconv"),
+                                 makeYamlString(CallingConvToStr[n.callConv]),
+                                 rope("size"), rope(n.size),
+                                 rope("align"), rope(n.align),
+                                 rope("sons"), result])
 
 proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
                    maxRecDepth: int): Rope =
@@ -309,8 +306,7 @@ proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
     var istr = rspaces(indent + 2)
     result = "{$N$1\"kind\": $2" % [istr, makeYamlString($n.kind)]
     if maxRecDepth != 0:
-      if conf != nil:
-        addf(result, ",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
+      addf(result, ",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
       case n.kind
       of nkCharLit..nkInt64Lit:
         addf(result, ",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
@@ -330,7 +326,7 @@ proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
       else:
         if sonsLen(n) > 0:
           addf(result, ",$N$1\"sons\": [", [istr])
-          for i in 0 ..< sonsLen(n):
+          for i in countup(0, sonsLen(n) - 1):
             if i > 0: add(result, ",")
             addf(result, "$N$1$2", [rspaces(indent + 4), treeToYamlAux(conf, n.sons[i],
                 marker, indent + 4, maxRecDepth - 1)])
@@ -351,258 +347,93 @@ proc symToYaml(conf: ConfigRef; n: PSym, indent: int = 0, maxRecDepth: int = - 1
   var marker = initIntSet()
   result = symToYamlAux(conf, n, marker, indent, maxRecDepth)
 
-import tables
-
-const backrefStyle = "\e[90m"
-const enumStyle = "\e[34m"
-const numberStyle = "\e[33m"
-const stringStyle = "\e[32m"
-const resetStyle  = "\e[0m"
-
-type
-  DebugPrinter = object
-    conf: ConfigRef
-    visited: Table[pointer, int]
-    renderSymType: bool
-    indent: int
-    currentLine: int
-    firstItem: bool
-    useColor: bool
-    res: string
-
-proc indentMore(this: var DebugPrinter) =
-  this.indent += 2
-
-proc indentLess(this: var DebugPrinter) =
-  this.indent -= 2
-
-proc newlineAndIndent(this: var DebugPrinter) =
-  this.res.add "\n"
-  this.currentLine += 1
-  for i in 0 ..< this.indent:
-    this.res.add ' '
-
-proc openCurly(this: var DebugPrinter) =
-  this.res.add "{"
-  this.indentMore
-  this.firstItem = true
-
-proc closeCurly(this: var DebugPrinter) =
-  this.indentLess
-  this.newlineAndIndent
-  this.res.add "}"
-
-proc comma(this: var DebugPrinter) =
-  this.res.add ", "
-
-proc openBracket(this: var DebugPrinter) =
-  this.res.add "["
-  #this.indentMore
-
-proc closeBracket(this: var DebugPrinter) =
-  #this.indentLess
-  this.res.add "]"
-
-proc key(this: var DebugPrinter; key: string) =
-  if not this.firstItem:
-    this.res.add ","
-  this.firstItem = false
-
-  this.newlineAndIndent
-  this.res.add "\""
-  this.res.add key
-  this.res.add "\": "
-
-proc value(this: var DebugPrinter; value: string) =
-  if this.useColor:
-    this.res.add stringStyle
-  this.res.add "\""
-  this.res.add value
-  this.res.add "\""
-  if this.useColor:
-    this.res.add resetStyle
-
-proc value(this: var DebugPrinter; value: BiggestInt) =
-  if this.useColor:
-    this.res.add numberStyle
-  this.res.addInt value
-  if this.useColor:
-    this.res.add resetStyle
-
-proc value[T: enum](this: var DebugPrinter; value: T) =
-  if this.useColor:
-    this.res.add enumStyle
-  this.res.add "\""
-  this.res.add $value
-  this.res.add "\""
-  if this.useColor:
-    this.res.add resetStyle
-
-proc value[T: enum](this: var DebugPrinter; value: set[T]) =
-  this.openBracket
-  let high = card(value)-1
-  var i = 0
-  for v in value:
-    this.value v
-    if i != high:
-      this.comma
-    inc i
-  this.closeBracket
-
-template earlyExit(this: var DebugPrinter; n: PType | PNode | PSym) =
+proc debugTree*(conf: ConfigRef; n: PNode, indent: int, maxRecDepth: int; renderType=false): Rope
+proc debugType(conf: ConfigRef; n: PType, maxRecDepth=100): Rope =
   if n == nil:
-    this.res.add "null"
-    return
-  let index = this.visited.getOrDefault(cast[pointer](n), -1)
-  if index < 0:
-    this.visited[cast[pointer](n)] = this.currentLine
+    result = rope("null")
   else:
-    if this.useColor:
-      this.res.add backrefStyle
-    this.res.add "<defined "
-    this.res.addInt(this.currentLine - index)
-    this.res.add " lines upwards>"
-    if this.useColor:
-      this.res.add resetStyle
-    return
+    result = rope($n.kind)
+    if n.sym != nil:
+      add(result, " ")
+      add(result, n.sym.name.s)
+    if n.kind in IntegralTypes and n.n != nil:
+      add(result, ", node: ")
+      add(result, debugTree(conf, n.n, 2, maxRecDepth-1, renderType=true))
+    if (n.kind != tyString) and (sonsLen(n) > 0) and maxRecDepth != 0:
+      add(result, "(")
+      for i in countup(0, sonsLen(n) - 1):
+        if i > 0: add(result, ", ")
+        if n.sons[i] == nil:
+          add(result, "null")
+        else:
+          add(result, debugType(conf, n.sons[i], maxRecDepth-1))
+      if n.kind == tyObject and n.n != nil:
+        add(result, ", node: ")
+        add(result, debugTree(conf, n.n, 2, maxRecDepth-1, renderType=true))
+      add(result, ")")
 
-proc value(this: var DebugPrinter; value: PType)
-proc value(this: var DebugPrinter; value: PNode)
-proc value(this: var DebugPrinter; value: PSym) =
-  earlyExit(this, value)
-
-  this.openCurly
-  this.key("kind")
-  this.value(value.kind)
-  this.key("name")
-  this.value(value.name.s)
-  this.key("id")
-  this.value(value.id)
-  if value.kind in {skField, skEnumField, skParam}:
-    this.key("position")
-    this.value(value.position)
-
-  if card(value.flags) > 0:
-    this.key("flags")
-    this.value(value.flags)
-
-  if this.renderSymType and value.typ != nil:
-    this.key "typ"
-    this.value(value.typ)
-
-  this.closeCurly
-
-proc value(this: var DebugPrinter; value: PType) =
-  earlyExit(this, value)
-
-  this.openCurly
-  this.key "kind"
-  this.value value.kind
-
-  this.key "id"
-  this.value value.id
-
-  if value.sym != nil:
-    this.key "sym"
-    this.value value.sym
-    #this.value value.sym.name.s
-
-  if card(value.flags) > 0:
-    this.key "flags"
-    this.value value.flags
-
-  if value.kind in IntegralTypes and value.n != nil:
-    this.key "n"
-    this.value value.n
-
-  if sonsLen(value) > 0:
-    this.key "sons"
-    this.openBracket
-    for i in 0 ..< sonsLen(value):
-      this.value value.sons[i]
-      if i != sonsLen(value) - 1:
-        this.comma
-    this.closeBracket
-
-  if value.n != nil:
-    this.key "n"
-    this.value value.n
-
-  this.closeCurly
-
-proc value(this: var DebugPrinter; value: PNode) =
-  earlyExit(this, value)
-
-  this.openCurly
-  this.key "kind"
-  this.value  value.kind
-  when defined(useNodeIds):
-    this.key "id"
-    this.value value.id
-  if this.conf != nil:
-    this.key "info"
-    this.value $lineInfoToStr(this.conf, value.info)
-  if card(value.flags) > 0:
-    this.key "flags"
-    this.value value.flags
-
-  case value.kind
-  of nkCharLit..nkUInt64Lit:
-    this.key "intVal"
-    this.value value.intVal
-  of nkFloatLit, nkFloat32Lit, nkFloat64Lit:
-    this.key "floatVal"
-    this.value value.floatVal.toStrMaxPrecision
-  of nkStrLit..nkTripleStrLit:
-    this.key "strVal"
-    this.value value.strVal
-  of nkSym:
-    this.key "sym"
-    this.value value.sym
-    #this.value value.sym.name.s
-  of nkIdent:
-    if value.ident != nil:
-      this.key "ident"
-      this.value value.ident.s
+proc debugTree(conf: ConfigRef; n: PNode, indent: int, maxRecDepth: int;
+               renderType=false): Rope =
+  if n == nil:
+    result = rope("null")
   else:
-    if this.renderSymType and value.typ != nil:
-      this.key "typ"
-      this.value value.typ
-    if sonsLen(value) > 0:
-      this.key "sons"
-      this.openBracket
-      for i in 0 ..< sonsLen(value):
-        this.value value.sons[i]
-        if i != sonsLen(value) - 1:
-          this.comma
-      this.closeBracket
-
-  this.closeCurly
+    var istr = rspaces(indent + 2)
+    result = "{$N$1\"kind\": $2" %
+             [istr, makeYamlString($n.kind)]
+    when defined(useNodeIds):
+      addf(result, ",$N$1\"id\": $2", [istr, rope(n.id)])
+    addf(result, ",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
+    if maxRecDepth != 0:
+      addf(result, ",$N$1\"flags\": $2", [istr, rope($n.flags)])
+      case n.kind
+      of nkCharLit..nkUInt64Lit:
+        addf(result, ",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
+      of nkFloatLit, nkFloat32Lit, nkFloat64Lit:
+        addf(result, ",$N$1\"floatVal\": $2",
+            [istr, rope(n.floatVal.toStrMaxPrecision)])
+      of nkStrLit..nkTripleStrLit:
+        addf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
+      of nkSym:
+        addf(result, ",$N$1\"sym\": $2_$3",
+            [istr, rope(n.sym.name.s), rope(n.sym.id)])
+        #     [istr, symToYaml(n.sym, indent, maxRecDepth),
+        #     rope(n.sym.id)])
+        if renderType and n.sym.typ != nil:
+          addf(result, ",$N$1\"typ\": $2", [istr, debugType(conf, n.sym.typ, 2)])
+      of nkIdent:
+        if n.ident != nil:
+          addf(result, ",$N$1\"ident\": $2", [istr, makeYamlString(n.ident.s)])
+        else:
+          addf(result, ",$N$1\"ident\": null", [istr])
+      else:
+        if renderType and n.typ != nil:
+          addf(result, ",$N$1\"typ\": $2", [istr, debugType(conf, n.typ, 2)])
+        if sonsLen(n) > 0:
+          addf(result, ",$N$1\"sons\": [", [istr])
+          for i in countup(0, sonsLen(n) - 1):
+            if i > 0: add(result, ",")
+            addf(result, "$N$1$2", [rspaces(indent + 4), debugTree(conf, n.sons[i],
+                indent + 4, maxRecDepth - 1, renderType)])
+          addf(result, "$N$1]", [istr])
+    addf(result, "$N$1}", [rspaces(indent)])
 
 when declared(echo):
   proc debug(n: PSym; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
+    if n == nil:
+      echo("null")
+    elif n.kind == skUnknown:
+      echo("skUnknown")
+    else:
+      #writeLine(stdout, $symToYaml(n, 0, 1))
+      echo("$1_$2: $3, $4, $5, $6" % [
+        n.name.s, $n.id, $flagsToStr(n.flags), $flagsToStr(n.loc.flags),
+        $lineInfoToStr(conf, n.info), $n.kind])
 
   proc debug(n: PType; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
+    echo($debugType(conf, n))
 
   proc debug(n: PNode; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    #this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
+    echo($debugTree(conf, n, 0, 100))
 
 proc nextTry(h, maxHash: Hash): Hash =
   result = ((5 * h) + 1) and maxHash
@@ -630,7 +461,7 @@ proc objectSetRawInsert(data: var TObjectSeq, obj: RootRef) =
 proc objectSetEnlarge(t: var TObjectSet) =
   var n: TObjectSeq
   newSeq(n, len(t.data) * GrowthFactor)
-  for i in 0 .. high(t.data):
+  for i in countup(0, high(t.data)):
     if t.data[i] != nil: objectSetRawInsert(n, t.data[i])
   swap(t.data, n)
 
@@ -665,18 +496,35 @@ proc strTableContains*(t: TStrTable, n: PSym): bool =
     h = nextTry(h, high(t.data))
   result = false
 
-proc strTableRawInsert(data: var seq[PSym], n: PSym) =
+proc strTableRawInsert(data: var TSymSeq, n: PSym) =
   var h: Hash = n.name.h and high(data)
-  while data[h] != nil:
-    if data[h] == n:
-      # allowed for 'export' feature:
-      #InternalError(n.info, "StrTableRawInsert: " & n.name.s)
-      return
-    h = nextTry(h, high(data))
-  assert(data[h] == nil)
-  data[h] = n
+  if sfImmediate notin n.flags:
+    # fast path:
+    while data[h] != nil:
+      if data[h] == n:
+        # allowed for 'export' feature:
+        #InternalError(n.info, "StrTableRawInsert: " & n.name.s)
+        return
+      h = nextTry(h, high(data))
+    assert(data[h] == nil)
+    data[h] = n
+  else:
+    # slow path; we have to ensure immediate symbols are preferred for
+    # symbol lookups.
+    # consider the chain: foo (immediate), foo, bar, bar (immediate)
+    # then bar (immediate) gets replaced with foo (immediate) and the non
+    # immediate foo is picked! Thus we need to replace it with the first
+    # slot that has in fact the same identifier stored in it!
+    var favPos = -1
+    while data[h] != nil:
+      if data[h] == n: return
+      if favPos < 0 and data[h].name.id == n.name.id: favPos = h
+      h = nextTry(h, high(data))
+    assert(data[h] == nil)
+    data[h] = n
+    if favPos >= 0: swap data[h], data[favPos]
 
-proc symTabReplaceRaw(data: var seq[PSym], prevSym: PSym, newSym: PSym) =
+proc symTabReplaceRaw(data: var TSymSeq, prevSym: PSym, newSym: PSym) =
   assert prevSym.name.h == newSym.name.h
   var h: Hash = prevSym.name.h and high(data)
   while data[h] != nil:
@@ -690,9 +538,9 @@ proc symTabReplace*(t: var TStrTable, prevSym: PSym, newSym: PSym) =
   symTabReplaceRaw(t.data, prevSym, newSym)
 
 proc strTableEnlarge(t: var TStrTable) =
-  var n: seq[PSym]
+  var n: TSymSeq
   newSeq(n, len(t.data) * GrowthFactor)
-  for i in 0 .. high(t.data):
+  for i in countup(0, high(t.data)):
     if t.data[i] != nil: strTableRawInsert(n, t.data[i])
   swap(t.data, n)
 
@@ -828,7 +676,7 @@ iterator items*(tab: TStrTable): PSym =
     s = nextIter(it, tab)
 
 proc hasEmptySlot(data: TIdPairSeq): bool =
-  for h in 0 .. high(data):
+  for h in countup(0, high(data)):
     if data[h].key == nil:
       return true
   result = false
@@ -883,7 +731,7 @@ proc idTablePut(t: var TIdTable, key: PIdObj, val: RootRef) =
   else:
     if mustRehash(len(t.data), t.counter):
       newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(t.data):
+      for i in countup(0, high(t.data)):
         if t.data[i].key != nil:
           idTableRawInsert(n, t.data[i].key, t.data[i].val)
       assert(hasEmptySlot(n))
@@ -929,7 +777,7 @@ proc idNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) =
     if mustRehash(len(t.data), t.counter):
       var n: TIdNodePairSeq
       newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(t.data):
+      for i in countup(0, high(t.data)):
         if t.data[i].key != nil:
           idNodeTableRawInsert(n, t.data[i].key, t.data[i].val)
       swap(t.data, n)
@@ -943,7 +791,7 @@ iterator pairs*(t: TIdNodeTable): tuple[key: PIdObj, val: PNode] =
 proc initIITable(x: var TIITable) =
   x.counter = 0
   newSeq(x.data, StartSize)
-  for i in 0 ..< StartSize: x.data[i].key = InvalidKey
+  for i in countup(0, StartSize - 1): x.data[i].key = InvalidKey
 
 proc iiTableRawGet(t: TIITable, key: int): int =
   var h: Hash
@@ -977,18 +825,10 @@ proc iiTablePut(t: var TIITable, key, val: int) =
     if mustRehash(len(t.data), t.counter):
       var n: TIIPairSeq
       newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(n): n[i].key = InvalidKey
-      for i in 0 .. high(t.data):
+      for i in countup(0, high(n)): n[i].key = InvalidKey
+      for i in countup(0, high(t.data)):
         if t.data[i].key != InvalidKey:
           iiTableRawInsert(n, t.data[i].key, t.data[i].val)
       swap(t.data, n)
     iiTableRawInsert(t.data, key, val)
     inc(t.counter)
-
-proc isAddrNode*(n: PNode): bool =
-  case n.kind
-    of nkAddr, nkHiddenAddr: true
-    of nkCallKinds:
-      if n[0].kind == nkSym and n[0].sym.magic == mAddr: true
-      else: false
-    else: false

compiler/bitsets.nim

@@ -11,17 +11,12 @@
 # the code here should be reused in the Nim standard library
 
 type
-  ElemType = byte
-  TBitSet* = seq[ElemType]    # we use byte here to avoid issues with
+  TBitSet* = seq[int8]        # we use byte here to avoid issues with
                               # cross-compiling; uint would be more efficient
                               # however
-const
-  ElemSize* = 8
-  One = ElemType(1)
-  Zero = ElemType(0)
 
-template modElemSize(arg: untyped): untyped = arg and 7
-template divElemSize(arg: untyped): untyped = arg shr 3
+const
+  ElemSize* = sizeof(int8) * 8
 
 proc bitSetInit*(b: var TBitSet, length: int)
 proc bitSetUnion*(x: var TBitSet, y: TBitSet)
@@ -37,65 +32,66 @@ proc bitSetCard*(x: TBitSet): BiggestInt
 # implementation
 
 proc bitSetIn(x: TBitSet, e: BiggestInt): bool =
-  result = (x[int(e.divElemSize)] and (One shl e.modElemSize)) != Zero
+  result = (x[int(e div ElemSize)] and toU8(int(1 shl (e mod ElemSize)))) !=
+      toU8(0)
 
 proc bitSetIncl(x: var TBitSet, elem: BiggestInt) =
   assert(elem >= 0)
-  x[int(elem.divElemSize)] = x[int(elem.divElemSize)] or
-      (One shl elem.modElemSize)
+  x[int(elem div ElemSize)] = x[int(elem div ElemSize)] or
+      toU8(int(1 shl (elem mod ElemSize)))
 
 proc bitSetExcl(x: var TBitSet, elem: BiggestInt) =
-  x[int(elem.divElemSize)] = x[int(elem.divElemSize)] and
-      not(One shl elem.modElemSize)
+  x[int(elem div ElemSize)] = x[int(elem div ElemSize)] and
+      not toU8(int(1 shl (elem mod ElemSize)))
 
 proc bitSetInit(b: var TBitSet, length: int) =
   newSeq(b, length)
 
 proc bitSetUnion(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] or y[i]
+  for i in countup(0, high(x)): x[i] = x[i] or y[i]
 
 proc bitSetDiff(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] and not y[i]
+  for i in countup(0, high(x)): x[i] = x[i] and not y[i]
 
 proc bitSetSymDiff(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] xor y[i]
+  for i in countup(0, high(x)): x[i] = x[i] xor y[i]
 
 proc bitSetIntersect(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] and y[i]
+  for i in countup(0, high(x)): x[i] = x[i] and y[i]
 
 proc bitSetEquals(x, y: TBitSet): bool =
-  for i in 0 .. high(x):
+  for i in countup(0, high(x)):
     if x[i] != y[i]:
       return false
   result = true
 
 proc bitSetContains(x, y: TBitSet): bool =
-  for i in 0 .. high(x):
-    if (x[i] and not y[i]) != Zero:
+  for i in countup(0, high(x)):
+    if (x[i] and not y[i]) != int8(0):
       return false
   result = true
 
 # Number of set bits for all values of int8
-const populationCount: array[uint8, uint8] = block:
-    var arr: array[uint8, uint8]
+const populationCount: array[low(int8)..high(int8), int8] = block:
+    var arr: array[low(int8)..high(int8), int8]
 
-    proc countSetBits(x: uint8): uint8 =
+    proc countSetBits(x: int8): int8 =
       return
-        ( x and 0b00000001'u8) +
-        ((x and 0b00000010'u8) shr 1) +
-        ((x and 0b00000100'u8) shr 2) +
-        ((x and 0b00001000'u8) shr 3) +
-        ((x and 0b00010000'u8) shr 4) +
-        ((x and 0b00100000'u8) shr 5) +
-        ((x and 0b01000000'u8) shr 6) +
-        ((x and 0b10000000'u8) shr 7)
+        ( x and 0b00000001'i8) +
+        ((x and 0b00000010'i8) shr 1) +
+        ((x and 0b00000100'i8) shr 2) +
+        ((x and 0b00001000'i8) shr 3) +
+        ((x and 0b00010000'i8) shr 4) +
+        ((x and 0b00100000'i8) shr 5) +
+        ((x and 0b01000000'i8) shr 6) +
+        ((x and 0b10000000'i8) shr 7)
+        
 
-
-    for it in low(uint8)..high(uint8):
-      arr[it] = countSetBits(cast[uint8](it))
+    for it in low(int8)..high(int8):
+      arr[it] = countSetBits(it)
 
     arr
 
 proc bitSetCard(x: TBitSet): BiggestInt =
   for it in x:
-    result.inc int(populationCount[it])
+    result.inc populationCount[it]

compiler/canonicalizer.nim

@@ -120,7 +120,7 @@ proc hashType(c: var MD5Context, t: PType) =
 
   case t.kind
   of tyGenericBody, tyGenericInst, tyGenericInvocation:
-    for i in 0 ..< sonsLen(t)-ord(t.kind != tyGenericInvocation):
+    for i in countup(0, sonsLen(t) -1 -ord(t.kind != tyGenericInvocation)):
       c.hashType t.sons[i]
   of tyUserTypeClass:
     internalAssert t.sym != nil and t.sym.owner != nil
@@ -128,7 +128,7 @@ proc hashType(c: var MD5Context, t: PType) =
   of tyUserTypeClassInst:
     let body = t.base
     c.hashSym body.sym
-    for i in 1 .. sonsLen(t) - 2:
+    for i in countup(1, sonsLen(t) - 2):
       c.hashType t.sons[i]
   of tyFromExpr:
     c.hashTree(t.n)
@@ -138,14 +138,14 @@ proc hashType(c: var MD5Context, t: PType) =
   of tyTuple:
     if t.n != nil:
       assert(sonsLen(t.n) == sonsLen(t))
-      for i in 0 ..< sonsLen(t.n):
+      for i in countup(0, sonsLen(t.n) - 1):
         assert(t.n.sons[i].kind == nkSym)
         c &= t.n.sons[i].sym.name.s
         c &= ":"
         c.hashType(t.sons[i])
         c &= ","
     else:
-      for i in 0 ..< sonsLen(t): c.hashType t.sons[i]
+      for i in countup(0, sonsLen(t) - 1): c.hashType t.sons[i]
   of tyRange:
     c.hashTree(t.n)
     c.hashType(t.sons[0])
@@ -238,7 +238,7 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
     encodeVInt(n.sym.id, result)
     pushSym(w, n.sym)
   else:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       encodeNode(w, n.info, n.sons[i], result)
   add(result, ')')
 
@@ -300,11 +300,11 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t.size != - 1:
     add(result, '/')
     encodeVBiggestInt(t.size, result)
-  if t.align != - 1:
+  if t.align != 2:
     add(result, '=')
     encodeVInt(t.align, result)
   encodeLoc(w, t.loc, result)
-  for i in 0 ..< sonsLen(t):
+  for i in countup(0, sonsLen(t) - 1):
     if t.sons[i] == nil:
       add(result, "^()")
     else:

compiler/ccgcalls.nim

@@ -20,7 +20,6 @@ proc hasNoInit(call: PNode): bool {.inline.} =
 
 proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
                callee, params: Rope) =
-  genLineDir(p, ri)
   var pl = callee & ~"(" & params
   # getUniqueType() is too expensive here:
   var typ = skipTypes(ri.sons[0].typ, abstractInst)
@@ -46,23 +45,13 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
         genAssignment(p, d, tmp, {}) # no need for deep copying
     else:
       add(pl, ~")")
-      if p.module.compileToCpp:
-        if lfSingleUse in d.flags:
-          # do not generate spurious temporaries for C++! For C we're better off
-          # with them to prevent undefined behaviour and because the codegen
-          # is free to emit expressions multiple times!
-          d.k = locCall
-          d.r = pl
-          excl d.flags, lfSingleUse
-        else:
-          if d.k == locNone and p.splitDecls == 0:
-            getTempCpp(p, typ.sons[0], d, pl)
-          else:
-            if d.k == locNone: getTemp(p, typ.sons[0], d)
-            var list: TLoc
-            initLoc(list, locCall, d.lode, OnUnknown)
-            list.r = pl
-            genAssignment(p, d, list, {}) # no need for deep copying
+      if p.module.compileToCpp and lfSingleUse in d.flags:
+        # do not generate spurious temporaries for C++! For C we're better off
+        # with them to prevent undefined behaviour and because the codegen
+        # is free to emit expressions multiple times!
+        d.k = locCall
+        d.r = pl
+        excl d.flags, lfSingleUse
       else:
         if d.k == locNone: getTemp(p, typ.sons[0], d)
         assert(d.t != nil)        # generate an assignment to d:
@@ -74,6 +63,26 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
     add(pl, ~");$n")
     line(p, cpsStmts, pl)
 
+proc isInCurrentFrame(p: BProc, n: PNode): bool =
+  # checks if `n` is an expression that refers to the current frame;
+  # this does not work reliably because of forwarding + inlining can break it
+  case n.kind
+  of nkSym:
+    if n.sym.kind in {skVar, skResult, skTemp, skLet} and p.prc != nil:
+      result = p.prc.id == n.sym.owner.id
+  of nkDotExpr, nkBracketExpr:
+    if skipTypes(n.sons[0].typ, abstractInst).kind notin {tyVar,tyLent,tyPtr,tyRef}:
+      result = isInCurrentFrame(p, n.sons[0])
+  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+    result = isInCurrentFrame(p, n.sons[1])
+  of nkHiddenDeref, nkDerefExpr:
+    # what about: var x = addr(y); callAsOpenArray(x[])?
+    # *shrug* ``addr`` is unsafe anyway.
+    result = false
+  of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
+    result = isInCurrentFrame(p, n.sons[0])
+  else: discard
+
 proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
 
 proc openArrayLoc(p: BProc, n: PNode): Rope =
@@ -97,7 +106,7 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
         result = "($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c)]
       else:
         result = "($1)+(($2)-($4)), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), intLiteral(first)]
-    of tyOpenArray, tyVarargs, tyUncheckedArray:
+    of tyOpenArray, tyVarargs:
       result = "($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c)]
     of tyString, tySequence:
       if skipTypes(n.typ, abstractInst).kind == tyVar and
@@ -146,7 +155,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
     var n = if n.kind != nkHiddenAddr: n else: n.sons[0]
     result = openArrayLoc(p, n)
-  elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
+  elif ccgIntroducedPtr(p.config, param):
     initLocExpr(p, n, a)
     result = addrLoc(p.config, a)
   elif p.module.compileToCpp and param.typ.kind == tyVar and
@@ -156,7 +165,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
     # means '*T'. See posix.nim for lots of examples that do that in the wild.
     let callee = call.sons[0]
     if callee.kind == nkSym and
-        {sfImportc, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportc} and
+        {sfImportC, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportC} and
         {lfHeader, lfNoDecl} * callee.sym.loc.flags != {}:
       result = addrLoc(p.config, a)
     else:
@@ -184,27 +193,19 @@ template genParamLoop(params) {.dirty.} =
     if params != nil: add(params, ~", ")
     add(params, genArgNoParam(p, ri.sons[i]))
 
-proc addActualPrefixForHCR(res: var Rope, module: PSym, sym: PSym) =
-  if sym.flags * {sfImportc, sfNonReloadable} == {} and
-      (sym.typ.callConv == ccInline or sym.owner.id == module.id):
-    res = res & "_actual".rope
-
 proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var op: TLoc
   # this is a hotspot in the compiler
   initLocExpr(p, ri.sons[0], op)
   var params: Rope
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInstOwned)
+  var typ = skipTypes(ri.sons[0].typ, abstractInst)
   assert(typ.kind == tyProc)
   assert(sonsLen(typ) == sonsLen(typ.n))
   var length = sonsLen(ri)
-  for i in 1 ..< length:
+  for i in countup(1, length - 1):
     genParamLoop(params)
-  var callee = rdLoc(op)
-  if p.hcrOn and ri.sons[0].kind == nkSym:
-    callee.addActualPrefixForHCR(p.module.module, ri.sons[0].sym)
-  fixupCall(p, le, ri, d, callee, params)
+  fixupCall(p, le, ri, d, op.r, params)
 
 proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
 
@@ -223,17 +224,15 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var typ = skipTypes(ri.sons[0].typ, abstractInst)
   assert(typ.kind == tyProc)
   var length = sonsLen(ri)
-  for i in 1 ..< length:
+  for i in countup(1, length - 1):
     assert(sonsLen(typ) == sonsLen(typ.n))
     genParamLoop(pl)
 
   template genCallPattern {.dirty.} =
-    if tfIterator in typ.flags:
-      lineF(p, cpsStmts, PatIter & ";$n", [rdLoc(op), pl, pl.addComma, rawProc])
-    else:
-      lineF(p, cpsStmts, PatProc & ";$n", [rdLoc(op), pl, pl.addComma, rawProc])
+    lineF(p, cpsStmts, callPattern & ";$n", [op.r, pl, pl.addComma, rawProc])
 
   let rawProc = getRawProcType(p, typ)
+  let callPattern = if tfIterator in typ.flags: PatIter else: PatProc
   if typ.sons[0] != nil:
     if isInvalidReturnType(p.config, typ.sons[0]):
       if sonsLen(ri) > 1: add(pl, ~", ")
@@ -258,11 +257,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
       initLoc(list, locCall, d.lode, OnUnknown)
-      if tfIterator in typ.flags:
-        list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
-      else:
-        list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
-
+      list.r = callPattern % [op.r, pl, pl.addComma, rawProc]
       genAssignment(p, d, list, {}) # no need for deep copying
   else:
     genCallPattern()
@@ -389,7 +384,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
           result.add genOtherArg(p, ri, k, typ)
       inc i
     of '#':
-      if i+1 < pat.len and pat[i+1] in {'+', '@'}:
+      if pat[i+1] in {'+', '@'}:
         let ri = ri[j]
         if ri.kind in nkCallKinds:
           let typ = skipTypes(ri.sons[0].typ, abstractInst)
@@ -404,10 +399,10 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
         else:
           localError(p.config, ri.info, "call expression expected for C++ pattern")
         inc i
-      elif i+1 < pat.len and pat[i+1] == '.':
+      elif pat[i+1] == '.':
         result.add genThisArg(p, ri, j, typ)
         inc i
-      elif i+1 < pat.len and pat[i+1] == '[':
+      elif pat[i+1] == '[':
         var arg = ri.sons[j].skipAddrDeref
         while arg.kind in {nkAddr, nkHiddenAddr, nkObjDownConv}: arg = arg[0]
         result.add genArgNoParam(p, arg)
@@ -470,7 +465,7 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
       add(pl, genThisArg(p, ri, 1, typ))
     add(pl, op.r)
     var params: Rope
-    for i in 2 ..< length:
+    for i in countup(2, length - 1):
       if params != nil: params.add(~", ")
       assert(sonsLen(typ) == sonsLen(typ.n))
       add(params, genOtherArg(p, ri, i, typ))
@@ -506,7 +501,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
     if length > 2:
       add(pl, ~": ")
       add(pl, genArg(p, ri.sons[2], typ.n.sons[2].sym, ri))
-  for i in start ..< length:
+  for i in countup(start, length-1):
     assert(sonsLen(typ) == sonsLen(typ.n))
     if i >= sonsLen(typ):
       internalError(p.config, ri.info, "varargs for objective C method?")
@@ -547,7 +542,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
     line(p, cpsStmts, pl)
 
 proc genCall(p: BProc, e: PNode, d: var TLoc) =
-  if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
+  if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure:
     genClosureCall(p, nil, e, d)
   elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags:
     genInfixCall(p, nil, e, d)
@@ -558,7 +553,7 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) =
   postStmtActions(p)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
-  if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
+  if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
   elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags:
     genInfixCall(p, le, ri, d)

compiler/ccgliterals.nim

@@ -47,41 +47,38 @@ proc genStringLiteralV1(m: BModule; n: PNode): Rope =
                       [genStringLiteralDataOnlyV1(m, n.strVal)])
     else:
       result = ropecg(m, "((#NimStringDesc*) &$1$2)",
-                      [m.tmpBase, id])
+                      [m.tmpBase, rope(id)])
 
 # ------ Version 2: destructor based strings and seqs -----------------------
 
-proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope) =
+proc genStringLiteralDataOnlyV2(m: BModule, s: string): Rope =
+  result = getTempName(m)
   addf(m.s[cfsData], "static const struct {$n" &
-       "  NI cap; void* allocator; NIM_CHAR data[$2+1];$n" &
+       "  NI cap; void* allocator; NIM_CHAR data[$2];$n" &
        "} $1 = { $2, NIM_NIL, $3 };$n",
        [result, rope(len(s)), makeCString(s)])
 
 proc genStringLiteralV2(m: BModule; n: PNode): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   if id == m.labels:
-    let pureLit = getTempName(m)
-    genStringLiteralDataOnlyV2(m, n.strVal, pureLit)
-    result = getTempName(m)
     discard cgsym(m, "NimStrPayload")
     discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
+    let pureLit = genStringLiteralDataOnlyV2(m, n.strVal)
+    result = getTempName(m)
     addf(m.s[cfsData], "static const NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
           [result, rope(len(n.strVal)), pureLit])
   else:
-    result = getTempName(m)
-    addf(m.s[cfsData], "static const NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [result, rope(len(n.strVal)), m.tmpBase & rope(id)])
+    result = m.tmpBase & rope(id)
 
 proc genStringLiteralV2Const(m: BModule; n: PNode): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   var pureLit: Rope
   if id == m.labels:
-    pureLit = getTempName(m)
     discard cgsym(m, "NimStrPayload")
     discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    genStringLiteralDataOnlyV2(m, n.strVal, pureLit)
+    pureLit = genStringLiteralDataOnlyV2(m, n.strVal)
   else:
     pureLit = m.tmpBase & rope(id)
   result = "{$1, (NimStrPayload*)&$2}" % [rope(len(n.strVal)), pureLit]
@@ -91,12 +88,14 @@ proc genStringLiteralV2Const(m: BModule; n: PNode): Rope =
 proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo): Rope =
   case detectStrVersion(m)
   of 0, 1: result = genStringLiteralDataOnlyV1(m, s)
-  of 2:
-    result = getTempName(m)
-    genStringLiteralDataOnlyV2(m, s, result)
+  of 2: result = genStringLiteralDataOnlyV2(m, s)
   else:
     localError(m.config, info, "cannot determine how to produce code for string literal")
 
+proc genStringLiteralFromData(m: BModule; data: Rope; info: TLineInfo): Rope =
+  result = ropecg(m, "((#NimStringDesc*) &$1)",
+                [data])
+
 proc genNilStringLiteral(m: BModule; info: TLineInfo): Rope =
   result = ropecg(m, "((#NimStringDesc*) NIM_NIL)", [])
 

compiler/ccgmerge.nim

@@ -11,8 +11,8 @@
 ## is needed for incremental compilation.
 
 import
-  ast, ropes, options, strutils, nimlexbase, cgendata, rodutils,
-  intsets, llstream, tables, modulegraphs, pathutils
+  ast, astalgo, ropes, options, strutils, nimlexbase, msgs, cgendata, rodutils,
+  intsets, platform, llstream, tables, sighashes, pathutils
 
 # Careful! Section marks need to contain a tabulator so that they cannot
 # be part of C string literals.
@@ -21,7 +21,6 @@ const
   CFileSectionNames: array[TCFileSection, string] = [
     cfsMergeInfo: "",
     cfsHeaders: "NIM_merge_HEADERS",
-    cfsFrameDefines: "NIM_merge_FRAME_DEFINES",
     cfsForwardTypes: "NIM_merge_FORWARD_TYPES",
     cfsTypes: "NIM_merge_TYPES",
     cfsSeqTypes: "NIM_merge_SEQ_TYPES",
@@ -32,7 +31,6 @@ const
     cfsData: "NIM_merge_DATA",
     cfsProcs: "NIM_merge_PROCS",
     cfsInitProc: "NIM_merge_INIT_PROC",
-    cfsDatInitProc: "NIM_merge_DATINIT_PROC",
     cfsTypeInit1: "NIM_merge_TYPE_INIT1",
     cfsTypeInit2: "NIM_merge_TYPE_INIT2",
     cfsTypeInit3: "NIM_merge_TYPE_INIT3",
@@ -146,34 +144,38 @@ proc atEndMark(buf: cstring, pos: int): bool =
 
 proc readVerbatimSection(L: var TBaseLexer): Rope =
   var pos = L.bufpos
+  var buf = L.buf
   var r = newStringOfCap(30_000)
   while true:
-    case L.buf[pos]
+    case buf[pos]
     of CR:
       pos = nimlexbase.handleCR(L, pos)
+      buf = L.buf
       r.add('\L')
     of LF:
       pos = nimlexbase.handleLF(L, pos)
+      buf = L.buf
       r.add('\L')
     of '\0':
       doAssert(false, "ccgmerge: expected: " & NimMergeEndMark)
       break
     else:
-      if atEndMark(L.buf, pos):
+      if atEndMark(buf, pos):
         inc pos, NimMergeEndMark.len
         break
-      r.add(L.buf[pos])
+      r.add(buf[pos])
       inc pos
   L.bufpos = pos
   result = r.rope
 
 proc readKey(L: var TBaseLexer, result: var string) =
   var pos = L.bufpos
+  var buf = L.buf
   setLen(result, 0)
-  while L.buf[pos] in IdentChars:
-    result.add(L.buf[pos])
+  while buf[pos] in IdentChars:
+    result.add(buf[pos])
     inc pos
-  if L.buf[pos] != ':': doAssert(false, "ccgmerge: ':' expected")
+  if buf[pos] != ':': doAssert(false, "ccgmerge: ':' expected")
   L.bufpos = pos + 1 # skip ':'
 
 proc newFakeType(id: int): PType =

compiler/ccgstmts.nim

@@ -15,21 +15,18 @@ const
   stringCaseThreshold = 8
     # above X strings a hash-switch for strings is generated
 
-proc getTraverseProc(p: BProc, v: PSym): Rope =
+proc registerGcRoot(p: BProc, v: PSym) =
   if p.config.selectedGC in {gcMarkAndSweep, gcDestructors, gcV2, gcRefc} and
-      optNimV2 notin p.config.globalOptions and
       containsGarbageCollectedRef(v.loc.t):
     # we register a specialized marked proc here; this has the advantage
     # that it works out of the box for thread local storage then :-)
-    result = genTraverseProcForGlobal(p.module, v, v.info)
-
-proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) =
-  if sfThread in v.flags:
-    appcg(p.module, p.module.initProc.procSec(cpsInit),
-      "$n\t#nimRegisterThreadLocalMarker($1);$n$n", [traverseProc])
-  else:
-    appcg(p.module, p.module.initProc.procSec(cpsInit),
-      "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc])
+    let prc = genTraverseProcForGlobal(p.module, v, v.info)
+    if sfThread in v.flags:
+      appcg(p.module, p.module.initProc.procSec(cpsInit),
+        "#nimRegisterThreadLocalMarker($1);$n", [prc])
+    else:
+      appcg(p.module, p.module.initProc.procSec(cpsInit),
+        "#nimRegisterGlobalMarker($1);$n", [prc])
 
 proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} =
   if n.kind == nkEmpty: return false
@@ -44,60 +41,28 @@ proc inExceptBlockLen(p: BProc): int =
   for x in p.nestedTryStmts:
     if x.inExcept: result.inc
 
-proc startBlockInternal(p: BProc): int {.discardable.} =
-  inc(p.labels)
-  result = len(p.blocks)
-  setLen(p.blocks, result + 1)
-  p.blocks[result].id = p.labels
-  p.blocks[result].nestedTryStmts = p.nestedTryStmts.len.int16
-  p.blocks[result].nestedExceptStmts = p.inExceptBlockLen.int16
-
-template startBlock(p: BProc, start: FormatStr = "{$n",
-                args: varargs[Rope]): int =
-  lineCg(p, cpsStmts, start, args)
-  startBlockInternal(p)
-
-proc endBlock(p: BProc)
-
 proc genVarTuple(p: BProc, n: PNode) =
   var tup, field: TLoc
   if n.kind != nkVarTuple: internalError(p.config, n.info, "genVarTuple")
   var L = sonsLen(n)
 
   # if we have a something that's been captured, use the lowering instead:
-  for i in 0 .. L-3:
+  for i in countup(0, L-3):
     if n[i].kind != nkSym:
       genStmts(p, lowerTupleUnpacking(p.module.g.graph, n, p.prc))
       return
 
-  # check only the first son
-  var forHcr = treatGlobalDifferentlyForHCR(p.module, n.sons[0].sym)
-  let hcrCond = if forHcr: getTempName(p.module) else: nil
-  var hcrGlobals: seq[tuple[loc: TLoc, tp: Rope]]
-  # determine if the tuple is constructed at top-level scope or inside of a block (if/while/block)
-  let isGlobalInBlock = forHcr and p.blocks.len > 2
-  # do not close and reopen blocks if this is a 'global' but inside of a block (if/while/block)
-  forHcr = forHcr and not isGlobalInBlock
-
-  if forHcr:
-    # check with the boolean if the initializing code for the tuple should be ran
-    lineCg(p, cpsStmts, "if ($1)$n", [hcrCond])
-    startBlock(p)
-
   genLineDir(p, n)
   initLocExpr(p, n.sons[L-1], tup)
   var t = tup.t.skipTypes(abstractInst)
-  for i in 0 .. L-3:
+  for i in countup(0, L-3):
     let vn = n.sons[i]
     let v = vn.sym
     if sfCompileTime in v.flags: continue
-    var traverseProc: Rope
     if sfGlobal in v.flags:
       assignGlobalVar(p, vn)
       genObjectInit(p, cpsInit, v.typ, v.loc, true)
-      traverseProc = getTraverseProc(p, v)
-      if traverseProc != nil and not p.hcrOn:
-        registerTraverseProc(p, v, traverseProc)
+      registerGcRoot(p, v)
     else:
       assignLocalVar(p, vn)
       initLocalVar(p, v, immediateAsgn=isAssignedImmediately(p.config, n[L-1]))
@@ -106,29 +71,16 @@ proc genVarTuple(p: BProc, n: PNode) =
       field.r = "$1.Field$2" % [rdLoc(tup), rope(i)]
     else:
       if t.n.sons[i].kind != nkSym: internalError(p.config, n.info, "genVarTuple")
-      field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(p.module, t.n.sons[i].sym)]
+      field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(p.module, t.n.sons[i].sym, t)]
     putLocIntoDest(p, v.loc, field)
-    if forHcr or isGlobalInBlock:
-      hcrGlobals.add((loc: v.loc, tp: if traverseProc == nil: ~"NULL" else: traverseProc))
-
-  if forHcr:
-    # end the block where the tuple gets initialized
-    endBlock(p)
-  if forHcr or isGlobalInBlock:
-    # insert the registration of the globals for the different parts of the tuple at the
-    # start of the current scope (after they have been iterated) and init a boolean to
-    # check if any of them is newly introduced and the initializing code has to be ran
-    lineCg(p, cpsLocals, "NIM_BOOL $1 = NIM_FALSE;$n", [hcrCond])
-    for curr in hcrGlobals:
-      lineCg(p, cpsLocals, "$1 |= hcrRegisterGlobal($4, \"$2\", sizeof($3), $5, (void**)&$2);$N",
-              [hcrCond, curr.loc.r, rdLoc(curr.loc), getModuleDllPath(p.module, n.sons[0].sym), curr.tp])
 
+proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false)
 
 proc loadInto(p: BProc, le, ri: PNode, a: var TLoc) {.inline.} =
   if ri.kind in nkCallKinds and (ri.sons[0].kind != nkSym or
                                  ri.sons[0].sym.magic == mNone):
     genAsgnCall(p, le, ri, a)
-  else:
+  elif ri.kind in {nkDerefExpr, nkHiddenDeref}:
     # this is a hacky way to fix #1181 (tmissingderef)::
     #
     #  var arr1 = cast[ptr array[4, int8]](addr foo)[]
@@ -136,9 +88,20 @@ proc loadInto(p: BProc, le, ri: PNode, a: var TLoc) {.inline.} =
     # However, fixing this properly really requires modelling 'array' as
     # a 'struct' in C to preserve dereferencing semantics completely. Not
     # worth the effort until version 1.0 is out.
-    a.flags.incl(lfEnforceDeref)
+    genDeref(p, ri, a, enforceDeref=true)
+  else:
     expr(p, ri, a)
 
+proc startBlock(p: BProc, start: FormatStr = "{$n",
+                args: varargs[Rope]): int {.discardable.} =
+  lineCg(p, cpsStmts, start, args)
+  inc(p.labels)
+  result = len(p.blocks)
+  setLen(p.blocks, result + 1)
+  p.blocks[result].id = p.labels
+  p.blocks[result].nestedTryStmts = p.nestedTryStmts.len.int16
+  p.blocks[result].nestedExceptStmts = p.inExceptBlockLen.int16
+
 proc assignLabel(b: var TBlock): Rope {.inline.} =
   b.label = "LA" & b.id.rope
   result = b.label
@@ -161,14 +124,13 @@ proc endBlock(p: BProc, blockEnd: Rope) =
 
 proc endBlock(p: BProc) =
   let topBlock = p.blocks.len - 1
+  var blockEnd = if p.blocks[topBlock].label != nil:
+      ropecg(p.module, "} $1: ;$n", p.blocks[topBlock].label)
+    else:
+      ~"}$n"
   let frameLen = p.blocks[topBlock].frameLen
-  var blockEnd: Rope
   if frameLen > 0:
     blockEnd.addf("FR_.len-=$1;$n", [frameLen.rope])
-  if p.blocks[topBlock].label != nil:
-    blockEnd.addf("} $1: ;$n", [p.blocks[topBlock].label])
-  else:
-    blockEnd.addf("}$n", [])
   endBlock(p, blockEnd)
 
 proc genSimpleBlock(p: BProc, stmts: PNode) {.inline.} =
@@ -191,9 +153,9 @@ proc genState(p: BProc, n: PNode) =
   let n0 = n[0]
   if n0.kind == nkIntLit:
     let idx = n.sons[0].intVal
-    linefmt(p, cpsStmts, "STATE$1: ;$n", [idx])
+    linefmt(p, cpsStmts, "STATE$1: ;$n", idx.rope)
   elif n0.kind == nkStrLit:
-    linefmt(p, cpsStmts, "$1: ;$n", [n0.strVal])
+    linefmt(p, cpsStmts, "$1: ;$n", n0.strVal.rope)
 
 proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
   # Called by return and break stmts.
@@ -201,12 +163,12 @@ proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
 
   var stack = newSeq[tuple[n: PNode, inExcept: bool]](0)
 
-  for i in 1 .. howManyTrys:
+  for i in countup(1, howManyTrys):
     let tryStmt = p.nestedTryStmts.pop
     if not p.module.compileToCpp or optNoCppExceptions in p.config.globalOptions:
       # Pop safe points generated by try
       if not tryStmt.inExcept:
-        linefmt(p, cpsStmts, "#popSafePoint();$n", [])
+        linefmt(p, cpsStmts, "#popSafePoint();$n")
 
     # Pop this try-stmt of the list of nested trys
     # so we don't infinite recurse on it in the next step.
@@ -225,9 +187,8 @@ proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
   if not p.module.compileToCpp or optNoCppExceptions in p.config.globalOptions:
     # Pop exceptions that was handled by the
     # except-blocks we are in
-    if not p.noSafePoints:
-      for i in countdown(howManyExcepts-1, 0):
-        linefmt(p, cpsStmts, "#popCurrentException();$n", [])
+    for i in countdown(howManyExcepts-1, 0):
+      linefmt(p, cpsStmts, "#popCurrentException();$n")
 
 proc genGotoState(p: BProc, n: PNode) =
   # we resist the temptation to translate it into duff's device as it later
@@ -280,7 +241,6 @@ proc genSingleVar(p: BProc, a: PNode) =
     genGotoVar(p, a.sons[2])
     return
   var targetProc = p
-  var traverseProc: Rope
   if sfGlobal in v.flags:
     if v.flags * {sfImportc, sfExportc} == {sfImportc} and
         a.sons[2].kind == nkEmpty and
@@ -309,9 +269,7 @@ proc genSingleVar(p: BProc, a: PNode) =
     # if sfImportc notin v.flags: constructLoc(p.module.preInitProc, v.loc)
     if sfExportc in v.flags and p.module.g.generatedHeader != nil:
       genVarPrototype(p.module.g.generatedHeader, vn)
-    traverseProc = getTraverseProc(p, v)
-    if traverseProc != nil and not p.hcrOn:
-      registerTraverseProc(p, v, traverseProc)
+    registerGcRoot(p, v)
   else:
     let value = a.sons[2]
     let imm = isAssignedImmediately(p.config, value)
@@ -343,31 +301,9 @@ proc genSingleVar(p: BProc, a: PNode) =
     assignLocalVar(p, vn)
     initLocalVar(p, v, imm)
 
-  if traverseProc == nil: traverseProc = ~"NULL"
-  # If the var is in a block (control flow like if/while or a block) in global scope just
-  # register the so called "global" so it can be used later on. There is no need to close
-  # and reopen of if (nim_hcr_do_init_) blocks because we are in one already anyway.
-  var forHcr = treatGlobalDifferentlyForHCR(p.module, v)
-  if forHcr and targetProc.blocks.len > 3 and v.owner.kind == skModule:
-    # put it in the locals section - mainly because of loops which
-    # use the var in a call to resetLoc() in the statements section
-    lineCg(targetProc, cpsLocals, "hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1);$n",
-           [v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc])
-    # nothing special left to do later on - let's avoid closing and reopening blocks
-    forHcr = false
-
-  # we close and reopen the global if (nim_hcr_do_init_) blocks in the main Init function
-  # for the module so we can have globals and top-level code be interleaved and still
-  # be able to re-run it but without the top level code - just the init of globals
-  if forHcr:
-    lineCg(targetProc, cpsStmts, "if (hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1))$N",
-           [v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc])
-    startBlock(targetProc)
   if a.sons[2].kind != nkEmpty:
     genLineDir(targetProc, a)
     loadInto(targetProc, a.sons[0], a.sons[2], v.loc)
-  if forHcr:
-    endBlock(targetProc)
 
 proc genClosureVar(p: BProc, a: PNode) =
   var immediateAsgn = a.sons[2].kind != nkEmpty
@@ -446,11 +382,11 @@ proc genReturnStmt(p: BProc, t: PNode) =
   blockLeaveActions(p,
     howManyTrys    = p.nestedTryStmts.len,
     howManyExcepts = p.inExceptBlockLen)
-  if (p.finallySafePoints.len > 0) and not p.noSafePoints:
+  if (p.finallySafePoints.len > 0):
     # If we're in a finally block, and we came here by exception
     # consume it before we return.
     var safePoint = p.finallySafePoints[p.finallySafePoints.len-1]
-    linefmt(p, cpsStmts, "if ($1.status != 0) #popCurrentException();$n", [safePoint])
+    linefmt(p, cpsStmts, "if ($1.status != 0) #popCurrentException();$n", safePoint)
   lineF(p, cpsStmts, "goto BeforeRet_;$n", [])
 
 proc genGotoForCase(p: BProc; caseStmt: PNode) =
@@ -466,16 +402,6 @@ proc genGotoForCase(p: BProc; caseStmt: PNode) =
     genStmts(p, it.lastSon)
     endBlock(p)
 
-
-iterator fieldValuePairs(n: PNode): tuple[memberSym, valueSym: PNode] =
-  assert(n.kind in {nkLetSection, nkVarSection})
-  for identDefs in n:
-    if identDefs.kind == nkIdentDefs:
-      let valueSym = identDefs[^1]
-      for i in 0 ..< identDefs.len-2:
-        let memberSym = identDefs[i]
-        yield((memberSym: memberSym, valueSym: valueSym))
-
 proc genComputedGoto(p: BProc; n: PNode) =
   # first pass: Generate array of computed labels:
   var casePos = -1
@@ -505,12 +431,22 @@ proc genComputedGoto(p: BProc; n: PNode) =
   let tmp = "TMP$1_" % [id.rope]
   var gotoArray = "static void* $#[$#] = {" % [tmp, arraySize.rope]
   for i in 1..arraySize-1:
-    gotoArray.addf("&&TMP$#_, ", [rope(id+i)])
-  gotoArray.addf("&&TMP$#_};$n", [rope(id+arraySize)])
+    gotoArray.addf("&&TMP$#_, ", [(id+i).rope])
+  gotoArray.addf("&&TMP$#_};$n", [(id+arraySize).rope])
   line(p, cpsLocals, gotoArray)
 
-  for j in 0 ..< casePos:
-    genStmts(p, n.sons[j])
+  let topBlock = p.blocks.len-1
+  let oldBody = p.blocks[topBlock].sections[cpsStmts]
+  p.blocks[topBlock].sections[cpsStmts] = nil
+
+  for j in casePos+1 ..< n.len: genStmts(p, n.sons[j])
+  let tailB = p.blocks[topBlock].sections[cpsStmts]
+
+  p.blocks[topBlock].sections[cpsStmts] = nil
+  for j in 0 .. casePos-1: genStmts(p, n.sons[j])
+  let tailA = p.blocks[topBlock].sections[cpsStmts]
+
+  p.blocks[topBlock].sections[cpsStmts] = oldBody & tailA
 
   let caseStmt = n.sons[casePos]
   var a: TLoc
@@ -525,40 +461,19 @@ proc genComputedGoto(p: BProc; n: PNode) =
       if it.sons[j].kind == nkRange:
         localError(p.config, it.info, "range notation not available for computed goto")
         return
-
       let val = getOrdValue(it.sons[j])
       lineF(p, cpsStmts, "TMP$#_:$n", [intLiteral(val+id+1)])
-
     genStmts(p, it.lastSon)
-
-    for j in casePos+1 ..< n.sons.len:
-      genStmts(p, n.sons[j])
-
-    for j in 0 ..< casePos:
-      # prevent new local declarations
-      # compile declarations as assignments
-      let it = n.sons[j]
-      if it.kind in {nkLetSection, nkVarSection}:
-        let asgn = copyNode(it)
-        asgn.kind = nkAsgn
-        asgn.sons.setLen 2
-        for sym, value in it.fieldValuePairs:
-          if value.kind != nkEmpty:
-            asgn.sons[0] = sym
-            asgn.sons[1] = value
-            genStmts(p, asgn)
-      else:
-        genStmts(p, it)
+    #for j in casePos+1 ..< n.len: genStmts(p, n.sons[j]) # tailB
+    #for j in 0 .. casePos-1: genStmts(p, n.sons[j])  # tailA
+    add(p.s(cpsStmts), tailB)
+    add(p.s(cpsStmts), tailA)
 
     var a: TLoc
     initLocExpr(p, caseStmt.sons[0], a)
     lineF(p, cpsStmts, "goto *$#[$#];$n", [tmp, a.rdLoc])
     endBlock(p)
 
-  for j in casePos+1 ..< n.sons.len:
-    genStmts(p, n.sons[j])
-
-
 proc genWhileStmt(p: BProc, t: PNode) =
   # we don't generate labels here as for example GCC would produce
   # significantly worse code
@@ -569,36 +484,34 @@ proc genWhileStmt(p: BProc, t: PNode) =
   genLineDir(p, t)
 
   preserveBreakIdx:
+    p.breakIdx = startBlock(p, "while (1) {$n")
+    p.blocks[p.breakIdx].isLoop = true
+    initLocExpr(p, t.sons[0], a)
+    if (t.sons[0].kind != nkIntLit) or (t.sons[0].intVal == 0):
+      let label = assignLabel(p.blocks[p.breakIdx])
+      lineF(p, cpsStmts, "if (!$1) goto $2;$n", [rdLoc(a), label])
     var loopBody = t.sons[1]
     if loopBody.stmtsContainPragma(wComputedGoto) and
-       hasComputedGoto in CC[p.config.cCompiler].props:
-         # for closure support weird loop bodies are generated:
+        hasComputedGoto in CC[p.config.cCompiler].props:
+      # for closure support weird loop bodies are generated:
       if loopBody.len == 2 and loopBody.sons[0].kind == nkEmpty:
         loopBody = loopBody.sons[1]
       genComputedGoto(p, loopBody)
     else:
-      p.breakIdx = startBlock(p, "while (1) {$n")
-      p.blocks[p.breakIdx].isLoop = true
-      initLocExpr(p, t.sons[0], a)
-      if (t.sons[0].kind != nkIntLit) or (t.sons[0].intVal == 0):
-        let label = assignLabel(p.blocks[p.breakIdx])
-        lineF(p, cpsStmts, "if (!$1) goto $2;$n", [rdLoc(a), label])
       genStmts(p, loopBody)
 
-      if optProfiler in p.options:
-        # invoke at loop body exit:
-        linefmt(p, cpsStmts, "#nimProfile();$n", [])
-      endBlock(p)
+    if optProfiler in p.options:
+      # invoke at loop body exit:
+      linefmt(p, cpsStmts, "#nimProfile();$n")
+    endBlock(p)
 
   dec(p.withinLoop)
 
 proc genBlock(p: BProc, n: PNode, d: var TLoc) =
-  if not isEmptyType(n.typ):
-    # bug #4505: allocate the temp in the outer scope
-    # so that it can escape the generated {}:
-    if d.k == locNone:
-      getTemp(p, n.typ, d)
-    d.flags.incl(lfEnforceDeref)
+  # bug #4505: allocate the temp in the outer scope
+  # so that it can escape the generated {}:
+  if not isEmptyType(n.typ) and d.k == locNone:
+    getTemp(p, n.typ, d)
   preserveBreakIdx:
     p.breakIdx = startBlock(p)
     if n.sons[0].kind != nkEmpty:
@@ -622,25 +535,14 @@ proc genParForStmt(p: BProc, t: PNode) =
     #initLoc(forLoopVar.loc, locLocalVar, forLoopVar.typ, onStack)
     #discard mangleName(forLoopVar)
     let call = t.sons[1]
-    assert(sonsLen(call) in {4, 5})
     initLocExpr(p, call.sons[1], rangeA)
     initLocExpr(p, call.sons[2], rangeB)
 
-    # $n at the beginning because of #9710
-    if call.sonsLen == 4: # `||`(a, b, annotation)
-      lineF(p, cpsStmts, "$n#pragma omp $4$n" &
-                          "for ($1 = $2; $1 <= $3; ++$1)",
-                          [forLoopVar.loc.rdLoc,
-                          rangeA.rdLoc, rangeB.rdLoc,
-                          call.sons[3].getStr.rope])
-    else: # `||`(a, b, step, annotation)
-      var step: TLoc
-      initLocExpr(p, call.sons[3], step)
-      lineF(p, cpsStmts, "$n#pragma omp $5$n" &
-                    "for ($1 = $2; $1 <= $3; $1 += $4)",
-                    [forLoopVar.loc.rdLoc,
-                    rangeA.rdLoc, rangeB.rdLoc, step.rdLoc,
-                    call.sons[4].getStr.rope])
+    lineF(p, cpsStmts, "#pragma omp parallel for $4$n" &
+                        "for ($1 = $2; $1 <= $3; ++$1)",
+                        [forLoopVar.loc.rdLoc,
+                        rangeA.rdLoc, rangeB.rdLoc,
+                        call.sons[3].getStr.rope])
 
     p.breakIdx = startBlock(p)
     p.blocks[p.breakIdx].isLoop = true
@@ -687,26 +589,22 @@ proc genRaiseStmt(p: BProc, t: PNode) =
     if isImportedException(typ, p.config):
       lineF(p, cpsStmts, "throw $1;$n", [e])
     else:
-      lineCg(p, cpsStmts, "#raiseExceptionEx((#Exception*)$1, $2, $3, $4, $5);$n",
-          [e, makeCString(typ.sym.name.s),
-          makeCString(if p.prc != nil: p.prc.name.s else: p.module.module.name.s),
-          quotedFilename(p.config, t.info), toLinenumber(t.info)])
-      if optNimV2 in p.config.globalOptions:
-        lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [e])
+      lineCg(p, cpsStmts, "#raiseException((#Exception*)$1, $2);$n",
+          [e, makeCString(typ.sym.name.s)])
   else:
     genLineDir(p, t)
     # reraise the last exception:
     if p.module.compileToCpp and optNoCppExceptions notin p.config.globalOptions:
       line(p, cpsStmts, ~"throw;$n")
     else:
-      linefmt(p, cpsStmts, "#reraiseException();$n", [])
+      linefmt(p, cpsStmts, "#reraiseException();$n")
 
-template genCaseGenericBranch(p: BProc, b: PNode, e: TLoc,
+proc genCaseGenericBranch(p: BProc, b: PNode, e: TLoc,
                           rangeFormat, eqFormat: FormatStr, labl: TLabel) =
   var
     x, y: TLoc
   var length = sonsLen(b)
-  for i in 0 .. length - 2:
+  for i in countup(0, length - 2):
     if b.sons[i].kind == nkRange:
       initLocExpr(p, b.sons[i].sons[0], x)
       initLocExpr(p, b.sons[i].sons[1], y)
@@ -731,11 +629,10 @@ proc genCaseSecondPass(p: BProc, t: PNode, d: var TLoc,
       exprBlock(p, t.sons[i].sons[0], d)
   result = lend
 
-template genIfForCaseUntil(p: BProc, t: PNode, d: var TLoc,
+proc genIfForCaseUntil(p: BProc, t: PNode, d: var TLoc,
                        rangeFormat, eqFormat: FormatStr,
                        until: int, a: TLoc): TLabel =
   # generate a C-if statement for a Nim case statement
-  var res: TLabel
   var labId = p.labels
   for i in 1..until:
     inc(p.labels)
@@ -748,13 +645,12 @@ template genIfForCaseUntil(p: BProc, t: PNode, d: var TLoc,
     inc(p.labels)
     var gotoTarget = p.labels
     lineF(p, cpsStmts, "goto LA$1_;$n", [rope(gotoTarget)])
-    res = genCaseSecondPass(p, t, d, labId, until)
+    result = genCaseSecondPass(p, t, d, labId, until)
     lineF(p, cpsStmts, "LA$1_: ;$n", [rope(gotoTarget)])
   else:
-    res = genCaseSecondPass(p, t, d, labId, until)
-  res
+    result = genCaseSecondPass(p, t, d, labId, until)
 
-template genCaseGeneric(p: BProc, t: PNode, d: var TLoc,
+proc genCaseGeneric(p: BProc, t: PNode, d: var TLoc,
                     rangeFormat, eqFormat: FormatStr) =
   var a: TLoc
   initLocExpr(p, t.sons[0], a)
@@ -765,7 +661,7 @@ proc genCaseStringBranch(p: BProc, b: PNode, e: TLoc, labl: TLabel,
                          branches: var openArray[Rope]) =
   var x: TLoc
   var length = sonsLen(b)
-  for i in 0 .. length - 2:
+  for i in countup(0, length - 2):
     assert(b.sons[i].kind != nkRange)
     initLocExpr(p, b.sons[i], x)
     assert(b.sons[i].kind in {nkStrLit..nkTripleStrLit})
@@ -776,7 +672,7 @@ proc genCaseStringBranch(p: BProc, b: PNode, e: TLoc, labl: TLabel,
 proc genStringCase(p: BProc, t: PNode, d: var TLoc) =
   # count how many constant strings there are in the case:
   var strings = 0
-  for i in 1 ..< sonsLen(t):
+  for i in countup(1, sonsLen(t) - 1):
     if t.sons[i].kind == nkOfBranch: inc(strings, sonsLen(t.sons[i]) - 1)
   if strings > stringCaseThreshold:
     var bitMask = math.nextPowerOfTwo(strings) - 1
@@ -785,7 +681,7 @@ proc genStringCase(p: BProc, t: PNode, d: var TLoc) =
     var a: TLoc
     initLocExpr(p, t.sons[0], a) # fist pass: gnerate ifs+goto:
     var labId = p.labels
-    for i in 1 ..< sonsLen(t):
+    for i in countup(1, sonsLen(t) - 1):
       inc(p.labels)
       if t.sons[i].kind == nkOfBranch:
         genCaseStringBranch(p, t.sons[i], a, "LA" & rope(p.labels) & "_",
@@ -795,8 +691,8 @@ proc genStringCase(p: BProc, t: PNode, d: var TLoc) =
         # but we reserved a label, which we use later
         discard
     linefmt(p, cpsStmts, "switch (#hashString($1) & $2) {$n",
-            [rdLoc(a), bitMask])
-    for j in 0 .. high(branches):
+            rdLoc(a), rope(bitMask))
+    for j in countup(0, high(branches)):
       if branches[j] != nil:
         lineF(p, cpsStmts, "case $1: $n$2break;$n",
              [intLiteral(j), branches[j]])
@@ -810,7 +706,7 @@ proc genStringCase(p: BProc, t: PNode, d: var TLoc) =
     genCaseGeneric(p, t, d, "", "if (#eqStrings($1, $2)) goto $3;$n")
 
 proc branchHasTooBigRange(b: PNode): bool =
-  for i in 0 .. sonsLen(b)-2:
+  for i in countup(0, sonsLen(b)-2):
     # last son is block
     if (b.sons[i].kind == nkRange) and
         b.sons[i].sons[1].intVal - b.sons[i].sons[0].intVal > RangeExpandLimit:
@@ -897,7 +793,7 @@ proc genRestoreFrameAfterException(p: BProc) =
       p.hasCurFramePointer = true
       p.procSec(cpsLocals).add(ropecg(p.module, "\tTFrame* _nimCurFrame;$n", []))
       p.procSec(cpsInit).add(ropecg(p.module, "\t_nimCurFrame = #getFrame();$n", []))
-    linefmt(p, cpsStmts, "#setFrame(_nimCurFrame);$n", [])
+    linefmt(p, cpsStmts, "#setFrame(_nimCurFrame);$n")
 
 proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   # code to generate:
@@ -957,15 +853,14 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
 
   discard pop(p.nestedTryStmts)
 
-  if t[^1].kind == nkFinally:
-    # c++ does not have finally, therefore code needs to be generated twice
-    if not catchAllPresent:
-      # finally requires catch all presence
-      startBlock(p, "catch (...) {$n")
-      genStmts(p, t[^1][0])
-      line(p, cpsStmts, ~"throw;$n")
-      endBlock(p)
+  if not catchAllPresent and t[^1].kind == nkFinally:
+    # finally requires catch all presence
+    startBlock(p, "catch (...) {$n")
+    genSimpleBlock(p, t[^1][0])
+    line(p, cpsStmts, ~"throw;$n")
+    endBlock(p)
 
+  if t[^1].kind == nkFinally:
     genSimpleBlock(p, t[^1][0])
 
 proc genTry(p: BProc, t: PNode, d: var TLoc) =
@@ -999,41 +894,29 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
   #
   if not isEmptyType(t.typ) and d.k == locNone:
     getTemp(p, t.typ, d)
-  let quirkyExceptions = isDefined(p.config, "nimQuirky") or
-      (t.kind == nkHiddenTryStmt and sfSystemModule in p.module.module.flags)
-  if not quirkyExceptions:
-    p.module.includeHeader("<setjmp.h>")
-  else:
-    p.noSafePoints = true
+  p.module.includeHeader("<setjmp.h>")
   genLineDir(p, t)
+  var safePoint = getTempName(p.module)
   discard cgsym(p.module, "Exception")
-  var safePoint: Rope
-  if not quirkyExceptions:
-    safePoint = getTempName(p.module)
-    linefmt(p, cpsLocals, "#TSafePoint $1;$n", [safePoint])
-    linefmt(p, cpsStmts, "#pushSafePoint(&$1);$n", [safePoint])
-    if isDefined(p.config, "nimStdSetjmp"):
-      linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
-    elif isDefined(p.config, "nimSigSetjmp"):
-      linefmt(p, cpsStmts, "$1.status = sigsetjmp($1.context, 0);$n", [safePoint])
-    elif isDefined(p.config, "nimRawSetjmp"):
-      linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint])
-    else:
-      linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
-    startBlock(p, "if ($1.status == 0) {$n", [safePoint])
-  var length = sonsLen(t)
-  add(p.nestedTryStmts, (t, quirkyExceptions))
-  expr(p, t.sons[0], d)
-  if not quirkyExceptions:
-    linefmt(p, cpsStmts, "#popSafePoint();$n", [])
-    endBlock(p)
-    startBlock(p, "else {$n")
-    linefmt(p, cpsStmts, "#popSafePoint();$n", [])
-    genRestoreFrameAfterException(p)
-  elif 1 < length and t.sons[1].kind == nkExceptBranch:
-    startBlock(p, "if (#getCurrentException()) {$n")
+  linefmt(p, cpsLocals, "#TSafePoint $1;$n", safePoint)
+  linefmt(p, cpsStmts, "#pushSafePoint(&$1);$n", safePoint)
+  if isDefined(p.config, "nimStdSetjmp"):
+    linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", safePoint)
+  elif isDefined(p.config, "nimSigSetjmp"):
+    linefmt(p, cpsStmts, "$1.status = sigsetjmp($1.context, 0);$n", safePoint)
+  elif isDefined(p.config, "nimRawSetjmp"):
+    linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", safePoint)
   else:
-    startBlock(p)
+    linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", safePoint)
+  startBlock(p, "if ($1.status == 0) {$n", [safePoint])
+  var length = sonsLen(t)
+  add(p.nestedTryStmts, (t, false))
+  expr(p, t.sons[0], d)
+  linefmt(p, cpsStmts, "#popSafePoint();$n")
+  endBlock(p)
+  startBlock(p, "else {$n")
+  linefmt(p, cpsStmts, "#popSafePoint();$n")
+  genRestoreFrameAfterException(p)
   p.nestedTryStmts[^1].inExcept = true
   var i = 1
   while (i < length) and (t.sons[i].kind == nkExceptBranch):
@@ -1044,29 +927,25 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
       # general except section:
       if i > 1: lineF(p, cpsStmts, "else", [])
       startBlock(p)
-      if not quirkyExceptions:
-        linefmt(p, cpsStmts, "$1.status = 0;$n", [safePoint])
+      linefmt(p, cpsStmts, "$1.status = 0;$n", safePoint)
       expr(p, t.sons[i].sons[0], d)
-      linefmt(p, cpsStmts, "#popCurrentException();$n", [])
+      linefmt(p, cpsStmts, "#popCurrentException();$n")
       endBlock(p)
     else:
       var orExpr: Rope = nil
-      for j in 0 .. blen - 2:
+      for j in countup(0, blen - 2):
         assert(t.sons[i].sons[j].kind == nkType)
         if orExpr != nil: add(orExpr, "||")
-        let checkFor = if optNimV2 in p.config.globalOptions:
-          genTypeInfo2Name(p.module, t[i][j].typ)
-        else:
-          genTypeInfo(p.module, t[i][j].typ, t[i][j].info)
-        let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
-        appcg(p.module, orExpr, "#isObj(#getCurrentException()->$1, $2)", [memberName, checkFor])
-
+        let isObjFormat = if not p.module.compileToCpp:
+          "#isObj(#getCurrentException()->Sup.m_type, $1)"
+          else: "#isObj(#getCurrentException()->m_type, $1)"
+        appcg(p.module, orExpr, isObjFormat,
+              [genTypeInfo(p.module, t[i][j].typ, t[i][j].info)])
       if i > 1: line(p, cpsStmts, "else ")
       startBlock(p, "if ($1) {$n", [orExpr])
-      if not quirkyExceptions:
-        linefmt(p, cpsStmts, "$1.status = 0;$n", [safePoint])
+      linefmt(p, cpsStmts, "$1.status = 0;$n", safePoint)
       expr(p, t.sons[i].sons[blen-1], d)
-      linefmt(p, cpsStmts, "#popCurrentException();$n", [])
+      linefmt(p, cpsStmts, "#popCurrentException();$n")
       endBlock(p)
     inc(i)
   discard pop(p.nestedTryStmts)
@@ -1075,8 +954,7 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
     p.finallySafePoints.add(safePoint)
     genSimpleBlock(p, t.sons[i].sons[0])
     discard pop(p.finallySafePoints)
-  if not quirkyExceptions:
-    linefmt(p, cpsStmts, "if ($1.status != 0) #reraiseException();$n", [safePoint])
+  linefmt(p, cpsStmts, "if ($1.status != 0) #reraiseException();$n", safePoint)
 
 proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
   var res = ""
@@ -1136,9 +1014,9 @@ proc genAsmStmt(p: BProc, t: PNode) =
   # work:
   if p.prc == nil:
     # top level asm statement?
-    add(p.module.s[cfsProcHeaders], runtimeFormat(CC[p.config.cCompiler].asmStmtFrmt, [s]))
+    addf(p.module.s[cfsProcHeaders], CC[p.config.cCompiler].asmStmtFrmt, [s])
   else:
-    add(p.s(cpsStmts), indentLine(p, runtimeFormat(CC[p.config.cCompiler].asmStmtFrmt, [s])))
+    lineF(p, cpsStmts, CC[p.config.cCompiler].asmStmtFrmt, [s])
 
 proc determineSection(n: PNode): TCFileSection =
   result = cfsProcHeaders
@@ -1171,7 +1049,7 @@ proc genBreakPoint(p: BProc, t: PNode) =
     genLineDir(p, t)          # BUGFIX
     appcg(p.module, p.module.g.breakpoints,
          "#dbgRegisterBreakpoint($1, (NCSTRING)$2, (NCSTRING)$3);$n", [
-        toLinenumber(t.info), makeCString(toFilename(p.config, t.info)),
+        rope(toLinenumber(t.info)), makeCString(toFilename(p.config, t.info)),
         makeCString(name)])
 
 proc genWatchpoint(p: BProc, n: PNode) =
@@ -1214,27 +1092,12 @@ proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
   var L = lengthOrd(p.config, field.typ)
   if not containsOrIncl(p.module.declaredThings, field.id):
     appcg(p.module, cfsVars, "extern $1",
-          [discriminatorTableDecl(p.module, t, field)])
+          discriminatorTableDecl(p.module, t, field))
   lineCg(p, cpsStmts,
         "#FieldDiscriminantCheck((NI)(NU)($1), (NI)(NU)($2), $3, $4);$n",
         [rdLoc(a), rdLoc(tmp), discriminatorTableName(p.module, t, field),
          intLiteral(L+1)])
 
-proc genCaseObjDiscMapping(p: BProc, e: PNode, t: PType, field: PSym; d: var TLoc) =
-  const ObjDiscMappingProcSlot = -5
-  var theProc: PSym = nil
-  for idx, p in items(t.methods):
-    if idx == ObjDiscMappingProcSlot:
-      theProc = p
-      break
-  if theProc == nil:
-    theProc = genCaseObjDiscMapping(t, field, e.info, p.module.g.graph)
-    t.methods.add((ObjDiscMappingProcSlot, theProc))
-  var call = newNodeIT(nkCall, e.info, getSysType(p.module.g.graph, e.info, tyUInt8))
-  call.add newSymNode(theProc)
-  call.add e
-  expr(p, call, d)
-
 proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   var a, tmp: TLoc
   var dotExpr = e.sons[0]
@@ -1242,41 +1105,54 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   initLocExpr(p, e.sons[0], a)
   getTemp(p, a.t, tmp)
   expr(p, e.sons[1], tmp)
-  let field = dotExpr.sons[1].sym
-  if optNimV2 in p.config.globalOptions:
-    let t = dotExpr[0].typ.skipTypes(abstractInst)
-    var oldVal, newVal: TLoc
-    genCaseObjDiscMapping(p, e[0], t, field, oldVal)
-    genCaseObjDiscMapping(p, e[1], t, field, newVal)
-    lineCg(p, cpsStmts,
-          "#nimFieldDiscriminantCheckV2($1, $2);$n",
-          [rdLoc(oldVal), rdLoc(newVal)])
-  else:
-    genDiscriminantCheck(p, a, tmp, dotExpr.sons[0].typ, field)
+  genDiscriminantCheck(p, a, tmp, dotExpr.sons[0].typ, dotExpr.sons[1].sym)
   genAssignment(p, a, tmp, {})
 
+proc patchAsgnStmtListExpr(father, orig, n: PNode) =
+  case n.kind
+  of nkDerefExpr, nkHiddenDeref:
+    let asgn = copyNode(orig)
+    asgn.add orig[0]
+    asgn.add n
+    father.add asgn
+  of nkStmtList, nkStmtListExpr:
+    for x in n:
+      patchAsgnStmtListExpr(father, orig, x)
+  else:
+    father.add n
+
 proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
   if e.sons[0].kind == nkSym and sfGoto in e.sons[0].sym.flags:
     genLineDir(p, e)
     genGotoVar(p, e.sons[1])
   elif not fieldDiscriminantCheckNeeded(p, e):
+    # this fixes bug #6422 but we really need to change the representation of
+    # arrays in the backend...
     let le = e[0]
     let ri = e[1]
+    var needsRepair = false
+    var it = ri
+    while it.kind in {nkStmtList, nkStmtListExpr}:
+      it = it.lastSon
+      needsRepair = true
+    if it.kind in {nkDerefExpr, nkHiddenDeref} and needsRepair:
+      var patchedTree = newNodeI(nkStmtList, e.info)
+      patchAsgnStmtListExpr(patchedTree, e, ri)
+      genStmts(p, patchedTree)
+      return
     var a: TLoc
     discard getTypeDesc(p.module, le.typ.skipTypes(skipPtrs))
-    initLoc(a, locNone, le, OnUnknown)
-    a.flags.incl(lfEnforceDeref)
-    a.flags.incl(lfPrepareForMutation)
-    expr(p, le, a)
-    a.flags.excl(lfPrepareForMutation)
+    if le.kind in {nkDerefExpr, nkHiddenDeref}:
+      genDeref(p, le, a, enforceDeref=true)
+    else:
+      initLocExpr(p, le, a)
     if fastAsgn: incl(a.flags, lfNoDeepCopy)
     assert(a.t != nil)
     genLineDir(p, ri)
-    loadInto(p, le, ri, a)
+    loadInto(p, e.sons[0], ri, a)
   else:
     genLineDir(p, e)
     asgnFieldDiscriminant(p, e)
-    message(p.config, e.info, warnCaseTransition)
 
 proc genStmts(p: BProc, t: PNode) =
   var a: TLoc

compiler/ccgthreadvars.nim

@@ -21,7 +21,7 @@ proc accessThreadLocalVar(p: BProc, s: PSym) =
     incl p.module.flags, usesThreadVars
     addf(p.procSec(cpsLocals), "\tNimThreadVars* NimTV_;$n", [])
     add(p.procSec(cpsInit),
-      ropecg(p.module, "\tNimTV_ = (NimThreadVars*) #GetThreadLocalVars();$n", []))
+      ropecg(p.module, "\tNimTV_ = (NimThreadVars*) #GetThreadLocalVars();$n"))
 
 proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
   if emulatedThreadVars(m.config):

compiler/ccgtrav.nim

@@ -16,9 +16,6 @@ type
     p: BProc
     visitorFrmt: string
 
-const
-  visitorFrmt  = "#nimGCvisit((void*)$1, $2);$n"
-
 proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType)
 proc genCaseRange(p: BProc, branch: PNode)
 proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false)
@@ -28,7 +25,7 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
   if n == nil: return
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       genTraverseProc(c, accessor, n.sons[i], typ)
   of nkRecCase:
     if (n.sons[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
@@ -38,7 +35,7 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
     if disc.loc.t == nil:
       internalError(c.p.config, n.info, "genTraverseProc()")
     lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
-    for i in 1 ..< sonsLen(n):
+    for i in countup(1, sonsLen(n) - 1):
       let branch = n.sons[i]
       assert branch.kind in {nkOfBranch, nkElse}
       if branch.kind == nkOfBranch:
@@ -70,7 +67,7 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType) =
   var p = c.p
   case typ.kind
   of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred,
-     tySink, tyOwned:
+     tySink:
     genTraverseProc(c, accessor, lastSon(typ))
   of tyArray:
     let arraySize = lengthOrd(c.p.config, typ.sons[0])
@@ -78,39 +75,39 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType) =
     getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo(), tyInt), i)
     let oldCode = p.s(cpsStmts)
     linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
-            [i.r, arraySize])
+            i.r, arraySize.rope)
     let oldLen = p.s(cpsStmts).len
-    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", [accessor, i.r]), typ.sons[1])
+    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", accessor, i.r), typ.sons[1])
     if p.s(cpsStmts).len == oldLen:
       # do not emit dummy long loops for faster debug builds:
       p.s(cpsStmts) = oldCode
     else:
       lineF(p, cpsStmts, "}$n", [])
   of tyObject:
-    for i in 0 ..< sonsLen(typ):
+    for i in countup(0, sonsLen(typ) - 1):
       var x = typ.sons[i]
       if x != nil: x = x.skipTypes(skipPtrs)
       genTraverseProc(c, accessor.parentObj(c.p.module), x)
     if typ.n != nil: genTraverseProc(c, accessor, typ.n, typ)
   of tyTuple:
     let typ = getUniqueType(typ)
-    for i in 0 ..< sonsLen(typ):
-      genTraverseProc(c, ropecg(c.p.module, "$1.Field$2", [accessor, i]), typ.sons[i])
+    for i in countup(0, sonsLen(typ) - 1):
+      genTraverseProc(c, ropecg(c.p.module, "$1.Field$2", accessor, i.rope), typ.sons[i])
   of tyRef:
-    lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
+    lineCg(p, cpsStmts, c.visitorFrmt, accessor)
   of tySequence:
     if tfHasAsgn notin typ.flags:
-      lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
+      lineCg(p, cpsStmts, c.visitorFrmt, accessor)
     elif containsGarbageCollectedRef(typ.lastSon):
       # destructor based seqs are themselves not traced but their data is, if
       # they contain a GC'ed type:
       genTraverseProcSeq(c, accessor, typ)
   of tyString:
     if tfHasAsgn notin typ.flags:
-      lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
+      lineCg(p, cpsStmts, c.visitorFrmt, accessor)
   of tyProc:
     if typ.callConv == ccClosure:
-      lineCg(p, cpsStmts, visitorFrmt, [ropecg(c.p.module, "$1.ClE_0", [accessor]), c.visitorFrmt])
+      lineCg(p, cpsStmts, c.visitorFrmt, ropecg(c.p.module, "$1.ClE_0", accessor))
   else:
     discard
 
@@ -137,53 +134,46 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope =
   var c: TTraversalClosure
   var p = newProc(nil, m)
   result = "Marker_" & getTypeName(m, origTyp, sig)
-  let
-    hcrOn = m.hcrOn
-    typ = origTyp.skipTypes(abstractInstOwned)
-    markerName = if hcrOn: result & "_actual" else: result
-    header = "static N_NIMCALL(void, $1)(void* p, NI op)" % [markerName]
-    t = getTypeDesc(m, typ)
+  var typ = origTyp.skipTypes(abstractInst)
 
+  let header = "static N_NIMCALL(void, $1)(void* p, NI op)" % [result]
+
+  let t = getTypeDesc(m, typ)
   lineF(p, cpsLocals, "$1 a;$n", [t])
   lineF(p, cpsInit, "a = ($1)p;$n", [t])
 
   c.p = p
-  c.visitorFrmt = "op" # "#nimGCvisit((void*)$1, op);$n"
+  c.visitorFrmt = "#nimGCvisit((void*)$1, op);$n"
 
   assert typ.kind != tyTypeDesc
   if typ.kind == tySequence:
     genTraverseProcSeq(c, "a".rope, typ)
   else:
-    if skipTypes(typ.sons[0], typedescInst+{tyOwned}).kind == tyArray:
+    if skipTypes(typ.sons[0], typedescInst).kind == tyArray:
       # C's arrays are broken beyond repair:
       genTraverseProc(c, "a".rope, typ.sons[0])
     else:
       genTraverseProc(c, "(*a)".rope, typ.sons[0])
 
-  let generatedProc = "$1 {$n$2$3$4}\n" %
+  let generatedProc = "$1 {$n$2$3$4}$n" %
         [header, p.s(cpsLocals), p.s(cpsInit), p.s(cpsStmts)]
 
-  m.s[cfsProcHeaders].addf("$1;\n", [header])
+  m.s[cfsProcHeaders].addf("$1;$n", [header])
   m.s[cfsProcs].add(generatedProc)
 
-  if hcrOn:
-    addf(m.s[cfsProcHeaders], "N_NIMCALL_PTR(void, $1)(void*, NI);\n", [result])
-    addf(m.s[cfsDynLibInit], "\t$1 = (N_NIMCALL_PTR(void, )(void*, NI)) hcrRegisterProc($3, \"$1\", (void*)$2);\n",
-         [result, markerName, getModuleDllPath(m)])
-
 proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope =
   discard genTypeInfo(m, s.loc.t, info)
 
   var c: TTraversalClosure
   var p = newProc(nil, m)
-  var sLoc = rdLoc(s.loc)
+  var sLoc = s.loc.r
   result = getTempName(m)
 
   if sfThread in s.flags and emulatedThreadVars(m.config):
     accessThreadLocalVar(p, s)
     sLoc = "NimTV_->" & sLoc
 
-  c.visitorFrmt = "0" # "#nimGCvisit((void*)$1, 0);$n"
+  c.visitorFrmt = "#nimGCvisit((void*)$1, 0);$n"
   c.p = p
   let header = "static N_NIMCALL(void, $1)(void)" % [result]
   genTraverseProc(c, sLoc, s.loc.t)

compiler/ccgtypes.nim

@@ -11,10 +11,10 @@
 
 # ------------------------- Name Mangling --------------------------------
 
-import sighashes, modulegraphs
+import sighashes
 from lowerings import createObj
 
-proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope
+proc genProcHeader(m: BModule, prc: PSym): Rope
 
 proc isKeyword(w: PIdent): bool =
   # Nim and C++ share some keywords
@@ -59,23 +59,7 @@ proc mangleParamName(m: BModule; s: PSym): Rope =
   result = s.loc.r
   if result == nil:
     var res = s.name.s.mangle
-    # Take into account if HCR is on because of the following scenario:
-    #   if a module gets imported and it has some more importc symbols in it,
-    # some param names might recieve the "_0" suffix to distinguish from what
-    # is newly available. That might lead to changes in the C code in nimcache
-    # that contain only a parameter name change, but that is enough to mandate
-    # recompilation of that source file and thus a new shared object will be
-    # relinked. That may lead to a module getting reloaded which wasn't intended
-    # and that may be fatal when parts of the current active callstack when
-    # performCodeReload() was called are from the module being reloaded
-    # unintentionally - example (3 modules which import one another):
-    #   main => proxy => reloadable
-    # we call performCodeReload() in proxy to reload only changes in reloadable
-    # but there is a new import which introduces an importc symbol `socket`
-    # and a function called in main or proxy uses `socket` as a parameter name.
-    # That would lead to either needing to reload `proxy` or to overwrite the
-    # executable file for the main module, which is running (or both!) -> error.
-    if m.hcrOn or isKeyword(s.name) or m.g.config.cppDefines.contains(res):
+    if isKeyword(s.name) or m.g.config.cppDefines.contains(res):
       res.add "_0"
     result = res.rope
     s.loc.r = result
@@ -110,8 +94,7 @@ proc scopeMangledParam(p: BProc; param: PSym) =
 
 const
   irrelevantForBackend = {tyGenericBody, tyGenericInst, tyGenericInvocation,
-                          tyDistinct, tyRange, tyStatic, tyAlias, tySink,
-                          tyInferred, tyOwned}
+                          tyDistinct, tyRange, tyStatic, tyAlias, tySink, tyInferred}
 
 proc typeName(typ: PType): Rope =
   let typ = typ.skipTypes(irrelevantForBackend)
@@ -131,7 +114,7 @@ proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope =
       t = t.lastSon
     else:
       break
-  let typ = if typ.kind in {tyAlias, tySink, tyOwned}: typ.lastSon else: typ
+  let typ = if typ.kind in {tyAlias, tySink}: typ.lastSon else: typ
   if typ.loc.r == nil:
     typ.loc.r = typ.typeName & $sig
   else:
@@ -152,17 +135,17 @@ proc mapSetType(conf: ConfigRef; typ: PType): TCTypeKind =
 proc mapType(conf: ConfigRef; typ: PType): TCTypeKind =
   ## Maps a Nim type to a C type
   case typ.kind
-  of tyNone, tyTyped: result = ctVoid
+  of tyNone, tyStmt: result = ctVoid
   of tyBool: result = ctBool
   of tyChar: result = ctChar
   of tySet: result = mapSetType(conf, typ)
-  of tyOpenArray, tyArray, tyVarargs, tyUncheckedArray: result = ctArray
+  of tyOpenArray, tyArray, tyVarargs: result = ctArray
   of tyObject, tyTuple: result = ctStruct
   of tyUserTypeClasses:
     doAssert typ.isResolvedUserTypeClass
     return mapType(conf, typ.lastSon)
   of tyGenericBody, tyGenericInst, tyGenericParam, tyDistinct, tyOrdinal,
-     tyTypeDesc, tyAlias, tySink, tyInferred, tyOwned:
+     tyTypeDesc, tyAlias, tySink, tyInferred:
     result = mapType(conf, lastSon(typ))
   of tyEnum:
     if firstOrd(conf, typ) < 0:
@@ -175,10 +158,10 @@ proc mapType(conf: ConfigRef; typ: PType): TCTypeKind =
       of 8: result = ctInt64
       else: result = ctInt32
   of tyRange: result = mapType(conf, typ.sons[0])
-  of tyPtr, tyVar, tyLent, tyRef:
+  of tyPtr, tyVar, tyLent, tyRef, tyOptAsRef:
     var base = skipTypes(typ.lastSon, typedescInst)
     case base.kind
-    of tyOpenArray, tyArray, tyVarargs, tyUncheckedArray: result = ctPtrToArray
+    of tyOpenArray, tyArray, tyVarargs: result = ctPtrToArray
     of tySet:
       if mapSetType(conf, base) == ctArray: result = ctPtrToArray
       else: result = ctPtr
@@ -209,6 +192,8 @@ proc isImportedCppType(t: PType): bool =
            (x.sym != nil and sfInfixCall in x.sym.flags)
 
 proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope
+proc needsComplexAssignment(typ: PType): bool =
+  result = containsGarbageCollectedRef(typ)
 
 proc isObjLackingTypeField(typ: PType): bool {.inline.} =
   result = (typ.kind == tyObject) and ((tfFinal in typ.flags) and
@@ -228,7 +213,7 @@ proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool =
     of ctStruct:
       let t = skipTypes(rettype, typedescInst)
       if rettype.isImportedCppType or t.isImportedCppType: return false
-      result = containsGarbageCollectedRef(t) or
+      result = needsComplexAssignment(t) or
           (t.kind == tyObject and not isObjLackingTypeField(t))
     else: result = false
 
@@ -246,13 +231,12 @@ proc cacheGetType(tab: TypeCache; sig: SigHash): Rope =
   result = tab.getOrDefault(sig)
 
 proc addAbiCheck(m: BModule, t: PType, name: Rope) =
-  if isDefined(m.config, "checkabi") and (let size = getSize(m.config, t); size != szUnknownSize):
-    addf(m.s[cfsTypeInfo], "NIM_CHECK_SIZE($1, $2);$n", [name, rope(size)])
+  if isDefined(m.config, "checkabi"):
+    addf(m.s[cfsTypeInfo], "NIM_CHECK_SIZE($1, $2);$n", [name, rope(getSize(m.config, t))])
 
-proc ccgIntroducedPtr(conf: ConfigRef; s: PSym, retType: PType): bool =
+proc ccgIntroducedPtr(conf: ConfigRef; s: PSym): bool =
   var pt = skipTypes(s.typ, typedescInst)
   assert skResult != s.kind
-
   if tfByRef in pt.flags: return true
   elif tfByCopy in pt.flags: return false
   case pt.kind
@@ -262,18 +246,13 @@ proc ccgIntroducedPtr(conf: ConfigRef; s: PSym, retType: PType): bool =
       result = true
     elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
       result = true           # requested anyway
-    elif retType != nil and retType.kind == tyLent:
-      result = true
     elif (tfFinal in pt.flags) and (pt.sons[0] == nil):
       result = false          # no need, because no subtyping possible
     else:
       result = true           # ordinary objects are always passed by reference,
                               # otherwise casting doesn't work
   of tyTuple:
-    if retType != nil and retType.kind == tyLent:
-      result = true
-    else:
-      result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options)
+    result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options)
   else: result = false
 
 proc fillResult(conf: ConfigRef; param: PNode) =
@@ -302,7 +281,6 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
   of tyString:
     case detectStrVersion(m)
     of 2:
-      discard cgsym(m, "NimStrPayload")
       discard cgsym(m, "NimStringV2")
       result = typeNameOrLiteral(m, typ, "NimStringV2")
     else:
@@ -311,14 +289,14 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
   of tyCString: result = typeNameOrLiteral(m, typ, "NCSTRING")
   of tyBool: result = typeNameOrLiteral(m, typ, "NIM_BOOL")
   of tyChar: result = typeNameOrLiteral(m, typ, "NIM_CHAR")
-  of tyNil: result = typeNameOrLiteral(m, typ, "void*")
+  of tyNil: result = typeNameOrLiteral(m, typ, "0")
   of tyInt..tyUInt64:
     result = typeNameOrLiteral(m, typ, NumericalTypeToStr[typ.kind])
   of tyDistinct, tyRange, tyOrdinal: result = getSimpleTypeDesc(m, typ.sons[0])
   of tyStatic:
     if typ.n != nil: result = getSimpleTypeDesc(m, lastSon typ)
     else: internalError(m.config, "tyStatic for getSimpleTypeDesc")
-  of tyGenericInst, tyAlias, tySink, tyOwned:
+  of tyGenericInst, tyAlias, tySink:
     result = getSimpleTypeDesc(m, lastSon typ)
   else: result = nil
 
@@ -329,9 +307,6 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
       addAbiCheck(m, typ, result)
 
 proc pushType(m: BModule, typ: PType) =
-  for i in 0 .. high(m.typeStack):
-    # pointer equality is good enough here:
-    if m.typeStack[i] == typ: return
   add(m.typeStack, typ)
 
 proc getTypePre(m: BModule, typ: PType; sig: SigHash): Rope =
@@ -341,17 +316,12 @@ proc getTypePre(m: BModule, typ: PType; sig: SigHash): Rope =
     if result == nil: result = cacheGetType(m.typeCache, sig)
 
 proc structOrUnion(t: PType): Rope =
-  let cachedUnion {.global.} = rope("union")
-  let cachedStruct {.global.} = rope("struct")
   let t = t.skipTypes({tyAlias, tySink})
-  if tfUnion in t.flags: cachedUnion
-  else: cachedStruct
+  (if tfUnion in t.flags: rope("union") else: rope("struct"))
 
-proc addForwardStructFormat(m: BModule, structOrUnion: Rope, typename: Rope) =
-  if m.compileToCpp:
-    m.s[cfsForwardTypes].addf "$1 $2;$n", [structOrUnion, typename]
-  else:
-    m.s[cfsForwardTypes].addf "typedef $1 $2 $2;$n", [structOrUnion, typename]
+proc getForwardStructFormat(m: BModule): string =
+  if m.compileToCpp: result = "$1 $2;$n"
+  else: result = "typedef $1 $2 $2;$n"
 
 proc seqStar(m: BModule): string =
   if m.config.selectedGC == gcDestructors: result = ""
@@ -368,7 +338,8 @@ proc getTypeForward(m: BModule, typ: PType; sig: SigHash): Rope =
     result = getTypeName(m, typ, sig)
     m.forwTypeCache[sig] = result
     if not isImportedType(concrete):
-      addForwardStructFormat(m, structOrUnion(typ), result)
+      addf(m.s[cfsForwardTypes], getForwardStructFormat(m),
+          [structOrUnion(typ), result])
     else:
       pushType(m, concrete)
     doAssert m.forwTypeCache[sig] == result
@@ -387,53 +358,14 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet): Rope =
       result = getTypeForward(m, t, hashType(t))
       pushType(m, t)
   of tySequence:
-    let sig = hashType(t)
     if m.config.selectedGC == gcDestructors:
-      if skipTypes(etB.sons[0], typedescInst).kind == tyEmpty:
-        internalError(m.config, "cannot map the empty seq type to a C type")
-
-      result = cacheGetType(m.forwTypeCache, sig)
-      if result == nil:
-        result = getTypeName(m, t, sig)
-        if not isImportedType(t):
-          m.forwTypeCache[sig] = result
-          addForwardStructFormat(m, rope"struct", result)
-          let payload = result & "_Content"
-          addForwardStructFormat(m, rope"struct", payload)
-
-      if cacheGetType(m.typeCache, sig) == nil:
-        m.typeCache[sig] = result
-        #echo "adding ", sig, " ", typeToString(t), " ", m.module.name.s
-        appcg(m, m.s[cfsTypes],
-          "struct $1 {$N" &
-          "  NI len; $1_Content* p;$N" &
-          "};$N", [result])
+      result = getTypeDescAux(m, t, check)
     else:
-      result = getTypeForward(m, t, sig) & seqStar(m)
-    pushType(m, t)
+      result = getTypeForward(m, t, hashType(t)) & seqStar(m)
+      pushType(m, t)
   else:
     result = getTypeDescAux(m, t, check)
 
-proc getSeqPayloadType(m: BModule; t: PType): Rope =
-  var check = initIntSet()
-  result = getTypeDescWeak(m, t, check) & "_Content"
-  #result = getTypeForward(m, t, hashType(t)) & "_Content"
-
-proc seqV2ContentType(m: BModule; t: PType; check: var IntSet) =
-  let sig = hashType(t)
-  let result = cacheGetType(m.typeCache, sig)
-  if result == nil:
-    discard getTypeDescAux(m, t, check)
-  else:
-    # little hack for now to prevent multiple definitions of the same
-    # Seq_Content:
-    appcg(m, m.s[cfsTypes], """$N
-$3ifndef $2_Content_PP
-$3define $2_Content_PP
-struct $2_Content { NI cap;#AllocatorObj* allocator;$1 data[SEQ_DECL_SIZE];};
-$3endif$N
-      """, [getTypeDescAux(m, t.skipTypes(abstractInst).sons[0], check), result, rope"#"])
-
 proc paramStorageLoc(param: PSym): TStorageLoc =
   if param.typ.skipTypes({tyVar, tyLent, tyTypeDesc}).kind notin {
           tyArray, tyOpenArray, tyVarargs}:
@@ -449,14 +381,14 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
     rettype = ~"void"
   else:
     rettype = getTypeDescAux(m, t.sons[0], check)
-  for i in 1 ..< sonsLen(t.n):
+  for i in countup(1, sonsLen(t.n) - 1):
     if t.n.sons[i].kind != nkSym: internalError(m.config, t.n.info, "genProcParams")
     var param = t.n.sons[i].sym
     if isCompileTimeOnly(param.typ): continue
     if params != nil: add(params, ~", ")
     fillLoc(param.loc, locParam, t.n.sons[i], mangleParamName(m, param),
             param.paramStorageLoc)
-    if ccgIntroducedPtr(m.config, param, t.sons[0]):
+    if ccgIntroducedPtr(m.config, param):
       add(params, getTypeDescWeak(m, param.typ, check))
       add(params, ~"*")
       incl(param.loc.flags, lfIndirect)
@@ -497,33 +429,39 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
   else: add(params, ")")
   params = "(" & params
 
-proc mangleRecFieldName(m: BModule; field: PSym): Rope =
-  if {sfImportc, sfExportc} * field.flags != {}:
+proc mangleRecFieldName(m: BModule; field: PSym, rectype: PType): Rope =
+  if (rectype.sym != nil) and
+      ({sfImportc, sfExportc} * rectype.sym.flags != {}):
     result = field.loc.r
   else:
     result = rope(mangleField(m, field.name))
   if result == nil: internalError(m.config, field.info, "mangleRecFieldName")
 
 proc genRecordFieldsAux(m: BModule, n: PNode,
-                        rectype: PType,
+                        accessExpr: Rope, rectype: PType,
                         check: var IntSet): Rope =
   result = nil
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      add(result, genRecordFieldsAux(m, n.sons[i], rectype, check))
+    for i in countup(0, sonsLen(n) - 1):
+      add(result, genRecordFieldsAux(m, n.sons[i], accessExpr, rectype, check))
   of nkRecCase:
     if n.sons[0].kind != nkSym: internalError(m.config, n.info, "genRecordFieldsAux")
-    add(result, genRecordFieldsAux(m, n.sons[0], rectype, check))
+    add(result, genRecordFieldsAux(m, n.sons[0], accessExpr, rectype, check))
     # prefix mangled name with "_U" to avoid clashes with other field names,
     # since identifiers are not allowed to start with '_'
+    let uname = rope("_U" & mangle(n.sons[0].sym.name.s))
+    let ae = if accessExpr != nil: "$1.$2" % [accessExpr, uname]
+             else: uname
     var unionBody: Rope = nil
-    for i in 1 ..< sonsLen(n):
+    for i in countup(1, sonsLen(n) - 1):
       case n.sons[i].kind
       of nkOfBranch, nkElse:
         let k = lastSon(n.sons[i])
         if k.kind != nkSym:
-          let a = genRecordFieldsAux(m, k, rectype, check)
+          let sname = "S" & rope(i)
+          let a = genRecordFieldsAux(m, k, "$1.$2" % [ae, sname], rectype,
+                                     check)
           if a != nil:
             if tfPacked notin rectype.flags:
               add(unionBody, "struct {")
@@ -533,29 +471,31 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
               else:
                 addf(unionBody, "#pragma pack(push, 1)$nstruct{", [])
             add(unionBody, a)
-            addf(unionBody, "};$n", [])
+            addf(unionBody, "} $1;$n", [sname])
             if tfPacked in rectype.flags and hasAttribute notin CC[m.config.cCompiler].props:
               addf(unionBody, "#pragma pack(pop)$n", [])
         else:
-          add(unionBody, genRecordFieldsAux(m, k, rectype, check))
+          add(unionBody, genRecordFieldsAux(m, k, ae, rectype, check))
       else: internalError(m.config, "genRecordFieldsAux(record case branch)")
     if unionBody != nil:
-      addf(result, "union{$n$1};$n", [unionBody])
+      addf(result, "union{$n$1} $2;$n", [unionBody, uname])
   of nkSym:
     let field = n.sym
     if field.typ.kind == tyVoid: return
     #assert(field.ast == nil)
-    let sname = mangleRecFieldName(m, field)
-    fillLoc(field.loc, locField, n, sname, OnUnknown)
+    let sname = mangleRecFieldName(m, field, rectype)
+    let ae = if accessExpr != nil: "$1.$2" % [accessExpr, sname]
+             else: sname
+    fillLoc(field.loc, locField, n, ae, OnUnknown)
     # for importcpp'ed objects, we only need to set field.loc, but don't
     # have to recurse via 'getTypeDescAux'. And not doing so prevents problems
     # with heavily templatized C++ code:
     if not isImportedCppType(rectype):
       let fieldType = field.loc.lode.typ.skipTypes(abstractInst)
-      if fieldType.kind == tyUncheckedArray:
+      if fieldType.kind == tyArray and tfUncheckedArray in fieldType.flags:
         addf(result, "$1 $2[SEQ_DECL_SIZE];$n",
             [getTypeDescAux(m, fieldType.elemType, check), sname])
-      elif fieldType.kind == tySequence:
+      elif fieldType.kind == tySequence and m.config.selectedGC != gcDestructors:
         # we need to use a weak dependency here for trecursive_table.
         addf(result, "$1 $2;$n", [getTypeDescWeak(m, field.loc.t, check), sname])
       elif field.bitsize != 0:
@@ -567,7 +507,7 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
   else: internalError(m.config, n.info, "genRecordFieldsAux()")
 
 proc getRecordFields(m: BModule, typ: PType, check: var IntSet): Rope =
-  result = genRecordFieldsAux(m, typ.n, typ, check)
+  result = genRecordFieldsAux(m, typ.n, nil, typ, check)
 
 proc fillObjectFields*(m: BModule; typ: PType) =
   # sometimes generic objects are not consistently merged. We patch over
@@ -575,8 +515,6 @@ proc fillObjectFields*(m: BModule; typ: PType) =
   var check = initIntSet()
   discard getRecordFields(m, typ, check)
 
-proc mangleDynLibProc(sym: PSym): Rope
-
 proc getRecordDesc(m: BModule, typ: PType, name: Rope,
                    check: var IntSet): Rope =
   # declare the record:
@@ -605,13 +543,15 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
       appcg(m, result, " : public $1 {$n",
                       [getTypeDescAux(m, typ.sons[0].skipTypes(skipPtrs), check)])
       if typ.isException:
-        appcg(m, result, "virtual void raise() { throw *this; }$n", []) # required for polymorphic exceptions
+        appcg(m, result, "virtual void raise() {throw *this;}$n") # required for polymorphic exceptions
         if typ.sym.magic == mException:
           # Add cleanup destructor to Exception base class
-          appcg(m, result, "~$1();$n", [name])
-          # define it out of the class body and into the procs section so we don't have to
-          # artificially forward-declare popCurrentExceptionEx (very VERY troublesome for HCR)
-          appcg(m, cfsProcs, "inline $1::~$1() {if(this->raiseId) #popCurrentExceptionEx(this->raiseId);}$n", [name])
+          appcg(m, result, "~$1() {if(this->raise_id) popCurrentExceptionEx(this->raise_id);}$n", [name])
+          # hack: forward declare popCurrentExceptionEx() on top of type description,
+          # proper request to generate popCurrentExceptionEx not possible for 2 reasons:
+          # generated function will be below declared Exception type and circular dependency
+          # between Exception and popCurrentExceptionEx function
+          result = genProcHeader(m, magicsys.getCompilerProc(m.g.graph, "popCurrentExceptionEx")) & ";" & result
       hasField = true
     else:
       appcg(m, result, " {$n  $1 Sup;$n",
@@ -633,7 +573,7 @@ proc getTupleDesc(m: BModule, typ: PType, name: Rope,
                   check: var IntSet): Rope =
   result = "$1 $2 {$n" % [structOrUnion(typ), name]
   var desc: Rope = nil
-  for i in 0 ..< sonsLen(typ):
+  for i in countup(0, sonsLen(typ) - 1):
     addf(desc, "$1 Field$2;$n",
          [getTypeDescAux(m, typ.sons[i], check), rope(i)])
   if desc == nil: add(result, "char dummy;\L")
@@ -669,9 +609,18 @@ proc resolveStarsInCppType(typ: PType, idx, stars: int): PType =
       result = if result.kind == tyGenericInst: result.sons[1]
                else: result.elemType
 
+proc getSeqPayloadType(m: BModule; t: PType): Rope =
+  result = getTypeForward(m, t, hashType(t)) & "_Content"
+  when false:
+    var check = initIntSet()
+    # XXX remove this duplication:
+    appcg(m, m.s[cfsSeqTypes],
+      "struct $2_Content { NI cap; void* allocator; $1 data[SEQ_DECL_SIZE]; };$n",
+      [getTypeDescAux(m, t.sons[0], check), result])
+
 proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
   # returns only the type's name
-  var t = origTyp.skipTypes(irrelevantForBackend-{tyOwned})
+  var t = origTyp.skipTypes(irrelevantForBackend)
   if containsOrIncl(check, t.id):
     if not (isImportedCppType(origTyp) or isImportedCppType(t)):
       internalError(m.config, "cannot generate C type for: " & typeToString(origTyp))
@@ -686,7 +635,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
     excl(check, t.id)
     return
   case t.kind
-  of tyRef, tyPtr, tyVar, tyLent:
+  of tyRef, tyOptAsRef, tyPtr, tyVar, tyLent:
     var star = if t.kind == tyVar and tfVarIsPtr notin origTyp.flags and
                     compileToCpp(m): "&" else: "*"
     var et = origTyp.skipTypes(abstractInst).lastSon
@@ -708,15 +657,11 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
         result = name & star
         m.typeCache[sig] = result
     of tySequence:
-      if m.config.selectedGC == gcDestructors:
-        result = getTypeDescWeak(m, et, check) & star
-        m.typeCache[sig] = result
-      else:
-        # no restriction! We have a forward declaration for structs
-        let name = getTypeForward(m, et, hashType et)
-        result = name & seqStar(m) & star
-        m.typeCache[sig] = result
-        pushType(m, et)
+      # no restriction! We have a forward declaration for structs
+      let name = getTypeForward(m, et, hashType et)
+      result = name & seqStar(m) & star
+      m.typeCache[sig] = result
+      pushType(m, et)
     else:
       # else we have a strong dependency  :-(
       result = getTypeDescAux(m, et, check) & star
@@ -747,7 +692,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
           let owner = hashOwner(t.sym)
           if not gDebugInfo.hasEnum(t.sym.name.s, t.sym.info.line, owner):
             var vals: seq[(string, int)] = @[]
-            for i in 0 ..< t.n.len:
+            for i in countup(0, t.n.len - 1):
               assert(t.n.sons[i].kind == nkSym)
               let field = t.n.sons[i].sym
               vals.add((field.name.s, field.position.int))
@@ -768,42 +713,39 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
             "void* ClE_0;$n} $1;$n",
              [result, rettype, desc])
   of tySequence:
-    if m.config.selectedGC == gcDestructors:
-      result = getTypeDescWeak(m, t, check)
-    else:
-      # we cannot use getTypeForward here because then t would be associated
-      # with the name of the struct, not with the pointer to the struct:
-      result = cacheGetType(m.forwTypeCache, sig)
-      if result == nil:
-        result = getTypeName(m, origTyp, sig)
-        if not isImportedType(t):
-          addForwardStructFormat(m, structOrUnion(t), result)
-        m.forwTypeCache[sig] = result
-      assert(cacheGetType(m.typeCache, sig) == nil)
-      m.typeCache[sig] = result & seqStar(m)
+    # we cannot use getTypeForward here because then t would be associated
+    # with the name of the struct, not with the pointer to the struct:
+    result = cacheGetType(m.forwTypeCache, sig)
+    if result == nil:
+      result = getTypeName(m, origTyp, sig)
       if not isImportedType(t):
-        if skipTypes(t.sons[0], typedescInst).kind != tyEmpty:
-          const
-            cppSeq = "struct $2 : #TGenericSeq {$n"
-            cSeq = "struct $2 {$n" &
-                  "  #TGenericSeq Sup;$n"
-          if m.compileToCpp:
-            appcg(m, m.s[cfsSeqTypes],
-                cppSeq & "  $1 data[SEQ_DECL_SIZE];$n" &
-                "};$n", [getTypeDescAux(m, t.sons[0], check), result])
-          else:
-            appcg(m, m.s[cfsSeqTypes],
-                cSeq & "  $1 data[SEQ_DECL_SIZE];$n" &
-                "};$n", [getTypeDescAux(m, t.sons[0], check), result])
-        else:
-          result = rope("TGenericSeq")
-      add(result, seqStar(m))
-  of tyUncheckedArray:
-    result = getTypeName(m, origTyp, sig)
-    m.typeCache[sig] = result
+        addf(m.s[cfsForwardTypes], getForwardStructFormat(m),
+            [structOrUnion(t), result])
+      m.forwTypeCache[sig] = result
+    assert(cacheGetType(m.typeCache, sig) == nil)
+    m.typeCache[sig] = result & seqStar(m)
     if not isImportedType(t):
-      let foo = getTypeDescAux(m, t.sons[0], check)
-      addf(m.s[cfsTypes], "typedef $1 $2[1];$n", [foo, result])
+      if skipTypes(t.sons[0], typedescInst).kind != tyEmpty:
+        const
+          cppSeq = "struct $2 : #TGenericSeq {$n"
+          cSeq = "struct $2 {$n" &
+                 "  #TGenericSeq Sup;$n"
+        if m.config.selectedGC == gcDestructors:
+          appcg(m, m.s[cfsTypes],
+            "typedef struct{ NI cap;void* allocator;$1 data[SEQ_DECL_SIZE];}$2_Content;$n" &
+            "struct $2 {$n" &
+            "  NI len; $2_Content* p;$n" &
+            "};$n", [getTypeDescAux(m, t.sons[0], check), result])
+        else:
+          appcg(m, m.s[cfsSeqTypes],
+              (if m.compileToCpp: cppSeq else: cSeq) &
+              "  $1 data[SEQ_DECL_SIZE];$n" &
+              "};$n", [getTypeDescAux(m, t.sons[0], check), result])
+      elif m.config.selectedGC == gcDestructors:
+        internalError(m.config, "cannot map the empty seq type to a C type")
+      else:
+        result = rope("TGenericSeq")
+    add(result, seqStar(m))
   of tyArray:
     var n: BiggestInt = lengthOrd(m.config, t)
     if n <= 0: n = 1   # make an array of at least one element
@@ -861,12 +803,26 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
       m.typeCache[sig] = typedefName
       result = typedefName
     else:
+      when false:
+        if t.sym != nil and t.sym.name.s == "KeyValuePair":
+          if t == origTyp:
+            echo "wtf: came here"
+            writeStackTrace()
+            quit 1
       result = cacheGetType(m.forwTypeCache, sig)
       if result == nil:
+        when false:
+          if t.sym != nil and t.sym.name.s == "KeyValuePair":
+            # or {sfImportc, sfExportc} * t.sym.flags == {}:
+            if t.loc.r != nil:
+              echo t.kind, " ", hashType t
+              echo origTyp.kind, " ", sig
+            assert t.loc.r == nil
         result = getTypeName(m, origTyp, sig)
         m.forwTypeCache[sig] = result
         if not isImportedType(t):
-          addForwardStructFormat(m, structOrUnion(t), result)
+          addf(m.s[cfsForwardTypes], getForwardStructFormat(m),
+             [structOrUnion(t), result])
         assert m.forwTypeCache[sig] == result
       m.typeCache[sig] = result # always call for sideeffects:
       if not incompleteType(t):
@@ -884,7 +840,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
       of 1, 2, 4, 8: addf(m.s[cfsTypes], "typedef NU$2 $1;$n", [result, rope(s*8)])
       else: addf(m.s[cfsTypes], "typedef NU8 $1[$2];$n",
              [result, rope(getSize(m.config, t))])
-  of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tySink, tyOwned,
+  of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tySink,
      tyUserTypeClass, tyUserTypeClassInst, tyInferred:
     result = getTypeDescAux(m, lastSon(t), check)
   else:
@@ -921,55 +877,37 @@ proc getClosureType(m: BModule, t: PType, kind: TClosureTypeKind): Rope =
 
 proc finishTypeDescriptions(m: BModule) =
   var i = 0
-  var check = initIntSet()
   while i < len(m.typeStack):
-    let t = m.typeStack[i]
-    if m.config.selectedGC == gcDestructors and t.skipTypes(abstractInst).kind == tySequence:
-      seqV2ContentType(m, t, check)
-    else:
-      discard getTypeDescAux(m, t, check)
+    discard getTypeDesc(m, m.typeStack[i])
     inc(i)
 
-template cgDeclFrmt*(s: PSym): string =
-  s.constraint.strVal
+template cgDeclFrmt*(s: PSym): string = s.constraint.strVal
 
-proc isReloadable(m: BModule, prc: PSym): bool =
-  return m.hcrOn and sfNonReloadable notin prc.flags
-
-proc isNonReloadable(m: BModule, prc: PSym): bool =
-  return m.hcrOn and sfNonReloadable in prc.flags
-
-proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope =
+proc genProcHeader(m: BModule, prc: PSym): Rope =
   var
     rettype, params: Rope
+  genCLineDir(result, prc.info, m.config)
   # using static is needed for inline procs
   if lfExportLib in prc.loc.flags:
     if isHeaderFile in m.flags:
       result.add "N_LIB_IMPORT "
     else:
       result.add "N_LIB_EXPORT "
-  elif prc.typ.callConv == ccInline or asPtr or isNonReloadable(m, prc):
+  elif prc.typ.callConv == ccInline:
     result.add "static "
   elif {sfImportc, sfExportc} * prc.flags == {}:
     result.add "N_LIB_PRIVATE "
   var check = initIntSet()
   fillLoc(prc.loc, locProc, prc.ast[namePos], mangleName(m, prc), OnUnknown)
   genProcParams(m, prc.typ, rettype, params, check)
-  # handle the 2 options for hotcodereloading codegen - function pointer
-  # (instead of forward declaration) or header for function budy with "_actual" postfix
-  let asPtrStr = rope(if asPtr: "_PTR" else: "")
-  var name = prc.loc.r
-  if isReloadable(m, prc) and not asPtr:
-    add(name, "_actual")
   # careful here! don't access ``prc.ast`` as that could reload large parts of
   # the object graph!
   if prc.constraint.isNil:
-    addf(result, "$1$2($3, $4)$5",
-         [rope(CallingConvToStr[prc.typ.callConv]), asPtrStr, rettype, name,
+    addf(result, "$1($2, $3)$4",
+         [rope(CallingConvToStr[prc.typ.callConv]), rettype, prc.loc.r,
          params])
   else:
-    let asPtrStr = if asPtr: (rope("(*") & name & ")") else: name
-    result = runtimeFormat(prc.cgDeclFrmt, [rettype, asPtrStr, params])
+    result = prc.cgDeclFrmt % [rettype, prc.loc.r, params]
 
 # ------------------ type info generation -------------------------------------
 
@@ -978,9 +916,6 @@ proc getNimNode(m: BModule): Rope =
   result = "$1[$2]" % [m.typeNodesName, rope(m.typeNodes)]
   inc(m.typeNodes)
 
-proc tiNameForHcr(m: BModule, name: Rope): Rope =
-  return if m.hcrOn: "(*".rope & name & ")" else: name
-
 proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
                         name, base: Rope; info: TLineInfo) =
   var nimtypeKind: int
@@ -990,21 +925,19 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
   else:
     nimtypeKind = ord(typ.kind)
 
-  let nameHcr = tiNameForHcr(m, name)
-
   var size: Rope
   if tfIncompleteStruct in typ.flags: size = rope"void*"
   else: size = getTypeDesc(m, origType)
   addf(m.s[cfsTypeInit3],
        "$1.size = sizeof($2);$n" & "$1.kind = $3;$n" & "$1.base = $4;$n",
-       [nameHcr, size, rope(nimtypeKind), base])
+       [name, size, rope(nimtypeKind), base])
   # compute type flags for GC optimization
   var flags = 0
   if not containsGarbageCollectedRef(typ): flags = flags or 1
   if not canFormAcycle(typ): flags = flags or 2
   #else MessageOut("can contain a cycle: " & typeToString(typ))
   if flags != 0:
-    addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [nameHcr, rope(flags)])
+    addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)])
   discard cgsym(m, "TNimType")
   if isDefined(m.config, "nimTypeNames"):
     var typename = typeToString(if origType.typeInst != nil: origType.typeInst
@@ -1012,17 +945,11 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
     if typename == "ref object" and origType.skipTypes(skipPtrs).sym != nil:
       typename = "anon ref object from " & m.config$origType.skipTypes(skipPtrs).sym.info
     addf(m.s[cfsTypeInit3], "$1.name = $2;$n",
-        [nameHcr, makeCString typename])
+        [name, makeCstring typename])
     discard cgsym(m, "nimTypeRoot")
     addf(m.s[cfsTypeInit3], "$1.nextType = nimTypeRoot; nimTypeRoot=&$1;$n",
-         [nameHcr])
-
-  if m.hcrOn:
-    addf(m.s[cfsVars], "static TNimType* $1;$n", [name])
-    addf(m.hcrCreateTypeInfosProc, "\thcrRegisterGlobal($2, \"$1\", sizeof(TNimType), NULL, (void**)&$1);$n",
-         [name, getModuleDllPath(m, m.module)])
-  else:
-    addf(m.s[cfsVars], "TNimType $1;$n", [name])
+         [name])
+  addf(m.s[cfsVars], "TNimType $1;$n", [name])
 
 proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope;
                     info: TLineInfo) =
@@ -1052,14 +979,6 @@ proc discriminatorTableDecl(m: BModule, objtype: PType, d: PSym): Rope =
   var tmp = discriminatorTableName(m, objtype, d)
   result = "TNimNode* $1[$2];$n" % [tmp, rope(lengthOrd(m.config, d.typ)+1)]
 
-proc genTNimNodeArray(m: BModule, name: Rope, size: Rope) =
-  if m.hcrOn:
-    addf(m.s[cfsVars], "static TNimNode** $1;$n", [name])
-    addf(m.hcrCreateTypeInfosProc, "\thcrRegisterGlobal($3, \"$1\", sizeof(TNimNode*) * $2, NULL, (void**)&$1);$n",
-         [name, size, getModuleDllPath(m, m.module)])
-  else:
-    addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [name, size])
-
 proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
                      info: TLineInfo) =
   case n.kind
@@ -1068,9 +987,9 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
     if L == 1:
       genObjectFields(m, typ, origType, n.sons[0], expr, info)
     elif L > 0:
-      var tmp = getTempName(m) & "_" & $L
-      genTNimNodeArray(m, tmp, rope(L))
-      for i in 0 ..< L:
+      var tmp = getTempName(m)
+      addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(L)])
+      for i in countup(0, L-1):
         var tmp2 = getNimNode(m)
         addf(m.s[cfsTypeInit3], "$1[$2] = &$3;$n", [tmp, rope(i), tmp2])
         genObjectFields(m, typ, origType, n.sons[i], tmp2, info)
@@ -1095,7 +1014,7 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
                            makeCString(field.name.s),
                            tmp, rope(L)])
     addf(m.s[cfsData], "TNimNode* $1[$2];$n", [tmp, rope(L+1)])
-    for i in 1 ..< sonsLen(n):
+    for i in countup(1, sonsLen(n)-1):
       var b = n.sons[i]           # branch
       var tmp2 = getNimNode(m)
       genObjectFields(m, typ, origType, lastSon(b), tmp2, info)
@@ -1103,7 +1022,7 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
       of nkOfBranch:
         if sonsLen(b) < 2:
           internalError(m.config, b.info, "genObjectFields; nkOfBranch broken")
-        for j in 0 .. sonsLen(b) - 2:
+        for j in countup(0, sonsLen(b) - 2):
           if b.sons[j].kind == nkRange:
             var x = int(getOrdValue(b.sons[j].sons[0]))
             var y = int(getOrdValue(b.sons[j].sons[1]))
@@ -1119,8 +1038,6 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
       else: internalError(m.config, n.info, "genObjectFields(nkRecCase)")
   of nkSym:
     var field = n.sym
-    # Do not produce code for void types
-    if isEmptyType(field.typ): return
     if field.bitsize == 0:
       if field.loc.r == nil: fillObjectFields(m, typ)
       if field.loc.t == nil:
@@ -1142,7 +1059,7 @@ proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo
   var tmp = getNimNode(m)
   if not isImportedType(typ):
     genObjectFields(m, typ, origType, typ.n, tmp, info)
-  addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [tiNameForHcr(m, name), tmp])
+  addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, tmp])
   var t = typ.sons[0]
   while t != nil:
     t = t.skipTypes(skipPtrs)
@@ -1154,9 +1071,9 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo)
   var expr = getNimNode(m)
   var length = sonsLen(typ)
   if length > 0:
-    var tmp = getTempName(m) & "_" & $length
-    genTNimNodeArray(m, tmp, rope(length))
-    for i in 0 ..< length:
+    var tmp = getTempName(m)
+    addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(length)])
+    for i in countup(0, length - 1):
       var a = typ.sons[i]
       var tmp2 = getNimNode(m)
       addf(m.s[cfsTypeInit3], "$1[$2] = &$3;$n", [tmp, rope(i), tmp2])
@@ -1170,7 +1087,7 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo)
   else:
     addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 2;$n",
          [expr, rope(length)])
-  addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [tiNameForHcr(m, name), expr])
+  addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, expr])
 
 proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
   # Type information for enumerations is quite heavy, so we do some
@@ -1178,13 +1095,14 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
   # anyway. We generate a cstring array and a loop over it. Exceptional
   # positions will be reset after the loop.
   genTypeInfoAux(m, typ, typ, name, info)
+  var nodePtrs = getTempName(m)
   var length = sonsLen(typ.n)
-  var nodePtrs = getTempName(m) & "_" & $length
-  genTNimNodeArray(m, nodePtrs, rope(length))
+  addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n",
+       [nodePtrs, rope(length)])
   var enumNames, specialCases: Rope
   var firstNimNode = m.typeNodes
   var hasHoles = false
-  for i in 0 ..< length:
+  for i in countup(0, length - 1):
     assert(typ.n.sons[i].kind == nkSym)
     var field = typ.n.sons[i].sym
     var elemNode = getNimNode(m)
@@ -1209,17 +1127,17 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
   add(m.s[cfsTypeInit3], specialCases)
   addf(m.s[cfsTypeInit3],
        "$1.len = $2; $1.kind = 2; $1.sons = &$3[0];$n$4.node = &$1;$n",
-       [getNimNode(m), rope(length), nodePtrs, tiNameForHcr(m, name)])
+       [getNimNode(m), rope(length), nodePtrs, name])
   if hasHoles:
     # 1 << 2 is {ntfEnumHole}
-    addf(m.s[cfsTypeInit3], "$1.flags = 1<<2;$n", [tiNameForHcr(m, name)])
+    addf(m.s[cfsTypeInit3], "$1.flags = 1<<2;$n", [name])
 
 proc genSetInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
   assert(typ.sons[0] != nil)
   genTypeInfoAux(m, typ, typ, name, info)
   var tmp = getNimNode(m)
   addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 0;$n" & "$3.node = &$1;$n",
-       [tmp, rope(firstOrd(m.config, typ)), tiNameForHcr(m, name)])
+       [tmp, rope(firstOrd(m.config, typ)), name])
 
 proc genArrayInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
   genTypeInfoAuxBase(m, typ, typ, name, genTypeInfo(m, typ.sons[1], info), info)
@@ -1240,79 +1158,23 @@ proc genDeepCopyProc(m: BModule; s: PSym; result: Rope) =
   addf(m.s[cfsTypeInit3], "$1.deepcopy =(void* (N_RAW_NIMCALL*)(void*))$2;$n",
      [result, s.loc.r])
 
-proc declareNimType(m: BModule, str: Rope, ownerModule: PSym) =
-  if m.hcrOn:
-    addf(m.s[cfsVars], "static TNimType* $1;$n", [str])
-    addf(m.s[cfsTypeInit1], "\t$1 = (TNimType*)hcrGetGlobal($2, \"$1\");$n",
-          [str, getModuleDllPath(m, ownerModule)])
-  else:
-    addf(m.s[cfsVars], "extern TNimType $1;$n", [str])
-
-proc genTypeInfo2Name(m: BModule; t: PType): Rope =
-  var res = "|"
-  var it = t
-  while it != nil:
-    it = it.skipTypes(skipPtrs)
-    if it.sym != nil:
-      var m = t.sym.owner
-      while m != nil and m.kind != skModule: m = m.owner
-      if m == nil or sfSystemModule in m.flags:
-        # produce short names for system types:
-        res.add it.sym.name.s
-      else:
-        var p = m.owner
-        if p != nil and p.kind == skPackage:
-          res.add p.name.s & "."
-        res.add m.name.s & "."
-        res.add it.sym.name.s
-    else:
-      res.add $hashType(it)
-    res.add "|"
-    it = it.sons[0]
-  result = makeCString(res)
-
-proc genObjectInfoV2(m: BModule, t, origType: PType, name: Rope; info: TLineInfo) =
-  assert t.kind == tyObject
-  if incompleteType(t):
-    localError(m.config, info, "request for RTTI generation for incomplete object: " &
-                      typeToString(t))
-
-  var d: Rope
-  if t.destructor != nil:
-    # the prototype of a destructor is ``=destroy(x: var T)`` and that of a
-    # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling
-    # convention at least:
-    if t.destructor.typ == nil or t.destructor.typ.callConv != ccDefault:
-      localError(m.config, info,
-        "the destructor that is turned into a finalizer needs " &
-        "to have the 'nimcall' calling convention")
-    genProc(m, t.destructor)
-    d = t.destructor.loc.r
-  else:
-    d = rope("NIM_NIL")
-  addf(m.s[cfsVars], "TNimType $1;$n", [name])
-  addf(m.s[cfsTypeInit3], "$1.destructor = (void*)$2; $1.size = sizeof($3); $1.name = $4;$n", [
-    name, d, getTypeDesc(m, t), genTypeInfo2Name(m, t)])
-
 proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope =
   let origType = t
   var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses)
 
-  let prefixTI = if m.hcrOn: "(" else: "(&"
-
   let sig = hashType(origType)
   result = m.typeInfoMarker.getOrDefault(sig)
   if result != nil:
-    return prefixTI.rope & result & ")".rope
+    return "(&".rope & result & ")".rope
 
-  let marker = m.g.typeInfoMarker.getOrDefault(sig)
-  if marker.str != nil:
+  result = m.g.typeInfoMarker.getOrDefault(sig)
+  if result != nil:
     discard cgsym(m, "TNimType")
     discard cgsym(m, "TNimNode")
-    declareNimType(m, marker.str, marker.owner)
+    addf(m.s[cfsVars], "extern TNimType $1;$n", [result])
     # also store in local type section:
-    m.typeInfoMarker[sig] = marker.str
-    return prefixTI.rope & marker.str & ")".rope
+    m.typeInfoMarker[sig] = result
+    return "(&".rope & result & ")".rope
 
   result = "NTI$1_" % [rope($sig)]
   m.typeInfoMarker[sig] = result
@@ -1320,15 +1182,14 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope =
   let owner = t.skipTypes(typedescPtrs).owner.getModule
   if owner != m.module:
     # make sure the type info is created in the owner module
-    assert m.g.modules[owner.position] != nil
     discard genTypeInfo(m.g.modules[owner.position], origType, info)
     # reference the type info as extern here
     discard cgsym(m, "TNimType")
     discard cgsym(m, "TNimNode")
-    declareNimType(m, result, owner)
-    return prefixTI.rope & result & ")".rope
+    addf(m.s[cfsVars], "extern TNimType $1;$n", [result])
+    return "(&".rope & result & ")".rope
 
-  m.g.typeInfoMarker[sig] = (str: result, owner: owner)
+  m.g.typeInfoMarker[sig] = result
   case t.kind
   of tyEmpty, tyVoid: result = rope"0"
   of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar, tyLent:
@@ -1346,25 +1207,21 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope =
       let x = fakeClosureType(m, t.owner)
       genTupleInfo(m, x, x, result, info)
   of tySequence:
-    genTypeInfoAux(m, t, t, result, info)
     if m.config.selectedGC != gcDestructors:
+      genTypeInfoAux(m, t, t, result, info)
       if m.config.selectedGC >= gcMarkAndSweep:
         let markerProc = genTraverseProc(m, origType, sig)
-        addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc])
-  of tyRef:
+        addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [result, markerProc])
+  of tyRef, tyOptAsRef:
     genTypeInfoAux(m, t, t, result, info)
     if m.config.selectedGC >= gcMarkAndSweep:
       let markerProc = genTraverseProc(m, origType, sig)
-      addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc])
-  of tyPtr, tyRange, tyUncheckedArray: genTypeInfoAux(m, t, t, result, info)
+      addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [result, markerProc])
+  of tyPtr, tyRange: genTypeInfoAux(m, t, t, result, info)
   of tyArray: genArrayInfo(m, t, result, info)
   of tySet: genSetInfo(m, t, result, info)
   of tyEnum: genEnumInfo(m, t, result, info)
-  of tyObject:
-    if optNimV2 in m.config.globalOptions:
-      genObjectInfoV2(m, t, origType, result, info)
-    else:
-      genObjectInfo(m, t, origType, result, info)
+  of tyObject: genObjectInfo(m, t, origType, result, info)
   of tyTuple:
     # if t.n != nil: genObjectInfo(m, t, result)
     # else:
@@ -1372,11 +1229,11 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope =
     # results are not deterministic!
     genTupleInfo(m, t, origType, result, info)
   else: internalError(m.config, "genTypeInfo(" & $t.kind & ')')
-  if t.attachedOps[attachedDeepCopy] != nil:
-    genDeepCopyProc(m, t.attachedOps[attachedDeepCopy], result)
-  elif origType.attachedOps[attachedDeepCopy] != nil:
-    genDeepCopyProc(m, origType.attachedOps[attachedDeepCopy], result)
-  result = prefixTI.rope & result & ")".rope
+  if t.deepCopy != nil:
+    genDeepCopyProc(m, t.deepCopy, result)
+  elif origType.deepCopy != nil:
+    genDeepCopyProc(m, origType.deepCopy, result)
+  result = "(&".rope & result & ")".rope
 
 proc genTypeSection(m: BModule, n: PNode) =
   discard

compiler/ccgutils.nim

@@ -10,7 +10,7 @@
 # This module declares some helpers for the C code generator.
 
 import
-  ast, hashes, strutils, msgs, wordrecg,
+  ast, astalgo, ropes, hashes, strutils, types, msgs, wordrecg,
   platform, trees, options
 
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
@@ -28,29 +28,29 @@ proc stmtsContainPragma*(n: PNode, w: TSpecialWord): bool =
   result = getPragmaStmt(n, w) != nil
 
 proc hashString*(conf: ConfigRef; s: string): BiggestInt =
-  # has to be the same algorithm as strmantle.hashString!
+  # has to be the same algorithm as system.hashString!
   if CPU[conf.target.targetCPU].bit == 64:
     # we have to use the same bitwidth
     # as the target CPU
-    var b = 0'u64
-    for i in 0 ..< len(s):
-      b = b + uint(s[i])
-      b = b + (b shl 10)
-      b = b xor (b shr 6)
-    b = b + (b shl 3)
-    b = b xor (b shr 11)
-    b = b + (b shl 15)
-    result = cast[Hash](b)
+    var b = 0'i64
+    for i in countup(0, len(s) - 1):
+      b = b +% ord(s[i])
+      b = b +% `shl`(b, 10)
+      b = b xor `shr`(b, 6)
+    b = b +% `shl`(b, 3)
+    b = b xor `shr`(b, 11)
+    b = b +% `shl`(b, 15)
+    result = b
   else:
-    var a = 0'u32
-    for i in 0 ..< len(s):
-      a = a + uint32(s[i])
-      a = a + (a shl 10)
-      a = a xor (a shr 6)
-    a = a + (a shl 3)
-    a = a xor (a shr 11)
-    a = a + (a shl 15)
-    result = cast[Hash](a)
+    var a = 0'i32
+    for i in countup(0, len(s) - 1):
+      a = a +% ord(s[i]).int32
+      a = a +% `shl`(a, 10'i32)
+      a = a xor `shr`(a, 6'i32)
+    a = a +% `shl`(a, 3'i32)
+    a = a xor `shr`(a, 11'i32)
+    a = a +% `shl`(a, 15'i32)
+    result = a
 
 template getUniqueType*(key: PType): PType = key
 

compiler/cgendata.nim

@@ -10,15 +10,16 @@
 ## This module contains the data structures for the C code generation phase.
 
 import
-  ast, ropes, options, intsets,
-  tables, ndi, lineinfos, pathutils, modulegraphs
+  ast, astalgo, ropes, passes, options, intsets, platform, sighashes,
+  tables, ndi, lineinfos, pathutils
+
+from modulegraphs import ModuleGraph, PPassContext
 
 type
   TLabel* = Rope              # for the C generator a label is just a rope
   TCFileSection* = enum       # the sections a generated C file consists of
     cfsMergeInfo,             # section containing merge information
     cfsHeaders,               # section for C include file headers
-    cfsFrameDefines           # section for nim frame macros
     cfsForwardTypes,          # section for C forward typedefs
     cfsTypes,                 # section for C typedefs
     cfsSeqTypes,              # section for sequence types only
@@ -31,7 +32,6 @@ type
     cfsData,                  # section for C constant data
     cfsProcs,                 # section for C procs that are not inline
     cfsInitProc,              # section for the C init proc
-    cfsDatInitProc,           # section for the C datInit proc
     cfsTypeInit1,             # section 1 for declarations of type information
     cfsTypeInit2,             # section 2 for init of type information
     cfsTypeInit3,             # section 3 for init of type information
@@ -70,7 +70,6 @@ type
     threadVarAccessed*: bool  # true if the proc already accessed some threadvar
     hasCurFramePointer*: bool # true if _nimCurFrame var needed to recover after
                               # exception is generated
-    noSafePoints*: bool       # the proc doesn't use safe points in exception handling
     lastLineInfo*: TLineInfo  # to avoid generating excessive 'nimln' statements
     currLineInfo*: TLineInfo  # AST codegen will make this superfluous
     nestedTryStmts*: seq[tuple[n: PNode, inExcept: bool]]
@@ -98,9 +97,8 @@ type
 
   TTypeSeq* = seq[PType]
   TypeCache* = Table[SigHash, Rope]
-  TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: PSym]]
 
-  CodegenFlag* = enum
+  Codegenflag* = enum
     preventStackTrace,  # true if stack traces need to be prevented
     usesThreadVars,     # true if the module uses a thread var
     frameDeclared,      # hack for ROD support so that we don't declare
@@ -114,12 +112,11 @@ type
     mainModProcs*, mainModInit*, otherModsInit*, mainDatInit*: Rope
     mapping*: Rope             # the generated mapping file (if requested)
     modules*: seq[BModule]     # list of all compiled modules
-    modulesClosed*: seq[BModule] # list of the same compiled modules, but in the order they were closed
-    forwardedProcs*: seq[PSym] # proc:s that did not yet have a body
+    forwardedProcsCounter*: int
     generatedHeader*: BModule
     breakPointId*: int
     breakpoints*: Rope # later the breakpoints are inserted into the main proc
-    typeInfoMarker*: TypeCacheWithOwner
+    typeInfoMarker*: TypeCache
     config*: ConfigRef
     graph*: ModuleGraph
     strVersion*, seqVersion*: int # version of the string/seq implementation to use
@@ -137,7 +134,7 @@ type
 
   TCGen = object of PPassContext # represents a C source file
     s*: TCFileSections        # sections of the C file
-    flags*: set[CodegenFlag]
+    flags*: set[Codegenflag]
     module*: PSym
     filename*: AbsoluteFile
     cfilename*: AbsoluteFile  # filename of the module (including path,
@@ -151,15 +148,14 @@ type
     typeInfoMarker*: TypeCache # needed for generating type information
     initProc*: BProc          # code for init procedure
     preInitProc*: BProc       # code executed before the init proc
-    hcrCreateTypeInfosProc*: Rope # type info globals are in here when HCR=on
-    inHcrInitGuard*: bool     # We are currently withing a HCR reloading guard.
     typeStack*: TTypeSeq      # used for type generation
     dataCache*: TNodeTable
+    forwardedProcs*: TSymSeq  # keep forwarded procs here
     typeNodes*, nimTypes*: int # used for type info generation
     typeNodesName*, nimTypesName*: Rope # used for type info generation
     labels*: Natural          # for generating unique module-scope names
     extensionLoaders*: array['0'..'9', Rope] # special procs for the
-                                             # OpenGL wrapper
+                                              # OpenGL wrapper
     injectStmt*: Rope
     sigConflicts*: CountTable[SigHash]
     g*: BModuleList
@@ -192,10 +188,12 @@ proc newProc*(prc: PSym, module: BModule): BProc =
   result.sigConflicts = initCountTable[string]()
 
 proc newModuleList*(g: ModuleGraph): BModuleList =
-  BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: PSym]](),
-    config: g.config, graph: g, nimtvDeclared: initIntSet())
+  BModuleList(modules: @[], typeInfoMarker: initTable[SigHash, Rope](), config: g.config,
+    graph: g, nimtvDeps: @[], nimtvDeclared: initIntSet())
 
 iterator cgenModules*(g: BModuleList): BModule =
-  for m in g.modulesClosed:
-    # iterate modules in the order they were closed
-    yield m
+  for i in 0..high(g.modules):
+    # ultimately, we are iterating over the file ids here.
+    # some "files" won't have an associated cgen module (like stdin)
+    # and we must skip over them.
+    if g.modules[i] != nil: yield g.modules[i]

compiler/cgmeth.nim

@@ -10,7 +10,7 @@
 ## This module implements code generation for multi methods.
 
 import
-  intsets, options, ast, msgs, idents, renderer, types, magicsys,
+  intsets, options, ast, astalgo, msgs, idents, renderer, types, magicsys,
   sempass2, strutils, modulegraphs, lineinfos
 
 proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
@@ -48,7 +48,7 @@ proc methodCall*(n: PNode; conf: ConfigRef): PNode =
   if disp != nil:
     result.sons[0].sym = disp
     # change the arguments to up/downcasts to fit the dispatcher's parameters:
-    for i in 1 ..< sonsLen(result):
+    for i in countup(1, sonsLen(result)-1):
       result.sons[i] = genConv(result.sons[i], disp.typ.sons[i], true, conf)
   else:
     localError(conf, n.info, "'" & $result.sons[0] & "' lacks a dispatcher")
@@ -56,12 +56,12 @@ proc methodCall*(n: PNode; conf: ConfigRef): PNode =
 type
   MethodResult = enum No, Invalid, Yes
 
-proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
+proc sameMethodBucket(a, b: PSym): MethodResult =
   if a.name.id != b.name.id: return
   if sonsLen(a.typ) != sonsLen(b.typ):
     return
 
-  for i in 1 ..< sonsLen(a.typ):
+  for i in countup(1, sonsLen(a.typ) - 1):
     var aa = a.typ.sons[i]
     var bb = b.typ.sons[i]
     while true:
@@ -75,7 +75,7 @@ proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
     if sameType(a.typ.sons[i], b.typ.sons[i]):
       if aa.kind == tyObject and result != Invalid:
         result = Yes
-    elif aa.kind == tyObject and bb.kind == tyObject and (i == 1 or multiMethods):
+    elif aa.kind == tyObject and bb.kind == tyObject:
       let diff = inheritanceDiff(bb, aa)
       if diff < 0:
         if result != Invalid:
@@ -91,7 +91,7 @@ proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
   if result == Yes:
     # check for return type:
     if not sameTypeOrNil(a.typ.sons[0], b.typ.sons[0]):
-      if b.typ.sons[0] != nil and b.typ.sons[0].kind == tyUntyped:
+      if b.typ.sons[0] != nil and b.typ.sons[0].kind == tyExpr:
         # infer 'auto' from the base to make it consistent:
         b.typ.sons[0] = a.typ.sons[0]
       else:
@@ -160,9 +160,9 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
 proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
   let L = len(g.methods)
   var witness: PSym
-  for i in 0 ..< L:
+  for i in countup(0, L - 1):
     let disp = g.methods[i].dispatcher
-    case sameMethodBucket(disp, s, multimethods = optMultiMethods in g.config.globalOptions)
+    case sameMethodBucket(disp, s)
     of Yes:
       add(g.methods[i].methods, s)
       attachDispatcher(s, disp.ast[dispatcherPos])
@@ -188,17 +188,17 @@ proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
   elif sfBase notin s.flags:
     message(g.config, s.info, warnUseBase)
 
-proc relevantCol(methods: seq[PSym], col: int): bool =
+proc relevantCol(methods: TSymSeq, col: int): bool =
   # returns true iff the position is relevant
   var t = methods[0].typ.sons[col].skipTypes(skipPtrs)
   if t.kind == tyObject:
-    for i in 1 .. high(methods):
+    for i in countup(1, high(methods)):
       let t2 = skipTypes(methods[i].typ.sons[col], skipPtrs)
       if not sameType(t2, t):
         return true
 
 proc cmpSignatures(a, b: PSym, relevantCols: IntSet): int =
-  for col in 1 ..< sonsLen(a.typ):
+  for col in countup(1, sonsLen(a.typ) - 1):
     if contains(relevantCols, col):
       var aa = skipTypes(a.typ.sons[col], skipPtrs)
       var bb = skipTypes(b.typ.sons[col], skipPtrs)
@@ -206,7 +206,7 @@ proc cmpSignatures(a, b: PSym, relevantCols: IntSet): int =
       if (d != high(int)) and d != 0:
         return d
 
-proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
+proc sortBucket(a: var TSymSeq, relevantCols: IntSet) =
   # we use shellsort here; fast and simple
   var n = len(a)
   var h = 1
@@ -215,7 +215,7 @@ proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
     if h > n: break
   while true:
     h = h div 3
-    for i in h ..< n:
+    for i in countup(h, n - 1):
       var v = a[i]
       var j = i
       while cmpSignatures(a[j - h], v, relevantCols) >= 0:
@@ -225,8 +225,8 @@ proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
       a[j] = v
     if h == 1: break
 
-proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet): PSym =
-  var base = methods[0].ast[dispatcherPos].sym
+proc genDispatcher(g: ModuleGraph; methods: TSymSeq, relevantCols: IntSet): PSym =
+  var base = lastSon(methods[0].ast).sym
   result = base
   var paramLen = sonsLen(base.typ)
   var nilchecks = newNodeI(nkStmtList, base.info)
@@ -234,16 +234,16 @@ proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet): PS
   var ands = getSysMagic(g, unknownLineInfo(), "and", mAnd)
   var iss = getSysMagic(g, unknownLineInfo(), "of", mOf)
   let boolType = getSysType(g, unknownLineInfo(), tyBool)
-  for col in 1 ..< paramLen:
+  for col in countup(1, paramLen - 1):
     if contains(relevantCols, col):
       let param = base.typ.n.sons[col].sym
       if param.typ.skipTypes(abstractInst).kind in {tyRef, tyPtr}:
         addSon(nilchecks, newTree(nkCall,
             newSymNode(getCompilerProc(g, "chckNilDisp")), newSymNode(param)))
-  for meth in 0 .. high(methods):
+  for meth in countup(0, high(methods)):
     var curr = methods[meth]      # generate condition:
     var cond: PNode = nil
-    for col in 1 ..< paramLen:
+    for col in countup(1, paramLen - 1):
       if contains(relevantCols, col):
         var isn = newNodeIT(nkCall, base.info, boolType)
         addSon(isn, newSymNode(iss))
@@ -261,7 +261,7 @@ proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet): PS
     let retTyp = base.typ.sons[0]
     let call = newNodeIT(nkCall, base.info, retTyp)
     addSon(call, newSymNode(curr))
-    for col in 1 ..< paramLen:
+    for col in countup(1, paramLen - 1):
       addSon(call, genConv(newSymNode(base.typ.n.sons[col].sym),
                            curr.typ.sons[col], false, g.config))
     var ret: PNode
@@ -281,18 +281,14 @@ proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet): PS
     else:
       disp = ret
   nilchecks.add disp
-  nilchecks.flags.incl nfTransf # should not be further transformed
   result.ast.sons[bodyPos] = nilchecks
 
 proc generateMethodDispatchers*(g: ModuleGraph): PNode =
   result = newNode(nkStmtList)
-  for bucket in 0 ..< len(g.methods):
+  for bucket in countup(0, len(g.methods) - 1):
     var relevantCols = initIntSet()
-    for col in 1 ..< sonsLen(g.methods[bucket].methods[0].typ):
+    for col in countup(1, sonsLen(g.methods[bucket].methods[0].typ) - 1):
       if relevantCol(g.methods[bucket].methods, col): incl(relevantCols, col)
-      if optMultiMethods notin g.config.globalOptions:
-        # if multi-methods are not enabled, we are interested only in the first field
-        break
     sortBucket(g.methods[bucket].methods, relevantCols)
     addSon(result,
            newSymNode(genDispatcher(g, g.methods[bucket].methods, relevantCols)))

compiler/closureiters.nim

@@ -131,8 +131,8 @@
 #   break :stateLoop
 
 import
-  ast, msgs, idents,
-  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
+  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, idents,
+  renderer, types, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
 
 type
   Ctx = object
@@ -210,7 +210,7 @@ proc newUnrollFinallyAccess(ctx: var Ctx, info: TLineInfo): PNode =
 
 proc newCurExcAccess(ctx: var Ctx): PNode =
   if ctx.curExcSym.isNil:
-    ctx.curExcSym = ctx.newEnvVar(":curExc", ctx.g.callCodegenProc("getCurrentException").typ)
+    ctx.curExcSym = ctx.newEnvVar(":curExc", ctx.g.callCodegenProc("getCurrentException", ctx.g.emptyNode).typ)
   ctx.newEnvVarAccess(ctx.curExcSym)
 
 proc newState(ctx: var Ctx, n, gotoOut: PNode): int =
@@ -324,7 +324,7 @@ proc collectExceptState(ctx: var Ctx, n: PNode): PNode {.inline.} =
           assert(c[i].kind == nkType)
           let nextCond = newTree(nkCall,
             newSymNode(g.getSysMagic(c.info, "of", mOf)),
-            g.callCodegenProc("getCurrentException"),
+            g.callCodegenProc("getCurrentException", ctx.g.emptyNode),
             c[i])
           nextCond.typ = ctx.g.getSysType(c.info, tyBool)
           nextCond.info = c.info
@@ -364,7 +364,7 @@ proc addElseToExcept(ctx: var Ctx, n: PNode) =
     block: # :curExc = getCurrentException()
       branchBody.add(newTree(nkAsgn,
         ctx.newCurExcAccess(),
-        ctx.g.callCodegenProc("getCurrentException")))
+        ctx.g.callCodegenProc("getCurrentException", ctx.g.emptyNode)))
 
     block: # goto nearestFinally
       branchBody.add(newTree(nkGotoState, ctx.g.newIntLit(n.info, ctx.nearestFinally)))
@@ -541,7 +541,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
       if isExpr: result.add(ctx.newEnvVarAccess(tmp))
 
-  of nkTryStmt, nkHiddenTryStmt:
+  of nkTryStmt:
     var ns = false
     for i in 0 ..< n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
@@ -810,7 +810,7 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   let branch = newTree(nkElifBranch, cmp, retStmt)
 
   # The C++ backend requires `getCurrentException` here.
-  let raiseStmt = newTree(nkRaiseStmt, ctx.g.callCodegenProc("getCurrentException"))
+  let raiseStmt = newTree(nkRaiseStmt, ctx.g.callCodegenProc("getCurrentException", ctx.g.emptyNode))
   raiseStmt.info = info
   let elseBranch = newTree(nkElse, raiseStmt)
 
@@ -854,155 +854,156 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
 
 proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode =
   result = n
-  case n.kind
-  of nkSkip: discard
+  case n.kind:
+    of nkSkip:
+      discard
 
-  of nkStmtList, nkStmtListExpr:
-    result = addGotoOut(result, gotoOut)
-    for i in 0 ..< n.len:
-      if n[i].hasYields:
-        # Create a new split
-        let go = newNodeI(nkGotoState, n[i].info)
-        n[i] = ctx.transformClosureIteratorBody(n[i], go)
+    of nkStmtList, nkStmtListExpr:
+      result = addGotoOut(result, gotoOut)
+      for i in 0 ..< n.len:
+        if n[i].hasYields:
+          # Create a new split
+          let go = newNodeI(nkGotoState, n[i].info)
+          n[i] = ctx.transformClosureIteratorBody(n[i], go)
 
-        let s = newNodeI(nkStmtList, n[i + 1].info)
-        for j in i + 1 ..< n.len:
-          s.add(n[j])
+          let s = newNodeI(nkStmtList, n[i + 1].info)
+          for j in i + 1 ..< n.len:
+            s.add(n[j])
 
-        n.sons.setLen(i + 1)
-        discard ctx.newState(s, go)
-        if ctx.transformClosureIteratorBody(s, gotoOut) != s:
-          internalError(ctx.g.config, "transformClosureIteratorBody != s")
-        break
+          n.sons.setLen(i + 1)
+          discard ctx.newState(s, go)
+          if ctx.transformClosureIteratorBody(s, gotoOut) != s:
+            internalError(ctx.g.config, "transformClosureIteratorBody != s")
+          break
 
-  of nkYieldStmt:
-    result = newNodeI(nkStmtList, n.info)
-    result.add(n)
-    result.add(gotoOut)
+    of nkYieldStmt:
+      result = newNodeI(nkStmtList, n.info)
+      result.add(n)
+      result.add(gotoOut)
 
-  of nkElse, nkElseExpr:
-    result[0] = addGotoOut(result[0], gotoOut)
-    result[0] = ctx.transformClosureIteratorBody(result[0], gotoOut)
+    of nkElse, nkElseExpr:
+      result[0] = addGotoOut(result[0], gotoOut)
+      result[0] = ctx.transformClosureIteratorBody(result[0], gotoOut)
 
-  of nkElifBranch, nkElifExpr, nkOfBranch:
-    result[^1] = addGotoOut(result[^1], gotoOut)
-    result[^1] = ctx.transformClosureIteratorBody(result[^1], gotoOut)
+    of nkElifBranch, nkElifExpr, nkOfBranch:
+      result[^1] = addGotoOut(result[^1], gotoOut)
+      result[^1] = ctx.transformClosureIteratorBody(result[^1], gotoOut)
+
+    of nkIfStmt, nkCaseStmt:
+      for i in 0 ..< n.len:
+        n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
+      if n[^1].kind != nkElse:
+        # We don't have an else branch, but every possible branch has to end with
+        # gotoOut, so add else here.
+        let elseBranch = newTree(nkElse, gotoOut)
+        n.add(elseBranch)
+
+    of nkWhileStmt:
+      # while e:
+      #   s
+      # ->
+      # BEGIN_STATE:
+      #   if e:
+      #     s
+      #     goto BEGIN_STATE
+      #   else:
+      #     goto OUT
+
+      result = newNodeI(nkGotoState, n.info)
+
+      let s = newNodeI(nkStmtList, n.info)
+      discard ctx.newState(s, result)
+      let ifNode = newNodeI(nkIfStmt, n.info)
+      let elifBranch = newNodeI(nkElifBranch, n.info)
+      elifBranch.add(n[0])
+
+      var body = addGotoOut(n[1], result)
+
+      body = ctx.transformBreaksAndContinuesInWhile(body, result, gotoOut)
+      body = ctx.transformClosureIteratorBody(body, result)
+
+      elifBranch.add(body)
+      ifNode.add(elifBranch)
 
-  of nkIfStmt, nkCaseStmt:
-    for i in 0 ..< n.len:
-      n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
-    if n[^1].kind != nkElse:
-      # We don't have an else branch, but every possible branch has to end with
-      # gotoOut, so add else here.
       let elseBranch = newTree(nkElse, gotoOut)
-      n.add(elseBranch)
+      ifNode.add(elseBranch)
+      s.add(ifNode)
 
-  of nkWhileStmt:
-    # while e:
-    #   s
-    # ->
-    # BEGIN_STATE:
-    #   if e:
-    #     s
-    #     goto BEGIN_STATE
-    #   else:
-    #     goto OUT
+    of nkBlockStmt:
+      result[1] = addGotoOut(result[1], gotoOut)
+      result[1] = ctx.transformBreaksInBlock(result[1], result[0], gotoOut)
+      result[1] = ctx.transformClosureIteratorBody(result[1], gotoOut)
 
-    result = newNodeI(nkGotoState, n.info)
+    of nkTryStmt:
+      # See explanation above about how this works
+      ctx.hasExceptions = true
 
-    let s = newNodeI(nkStmtList, n.info)
-    discard ctx.newState(s, result)
-    let ifNode = newNodeI(nkIfStmt, n.info)
-    let elifBranch = newNodeI(nkElifBranch, n.info)
-    elifBranch.add(n[0])
-
-    var body = addGotoOut(n[1], result)
-
-    body = ctx.transformBreaksAndContinuesInWhile(body, result, gotoOut)
-    body = ctx.transformClosureIteratorBody(body, result)
-
-    elifBranch.add(body)
-    ifNode.add(elifBranch)
-
-    let elseBranch = newTree(nkElse, gotoOut)
-    ifNode.add(elseBranch)
-    s.add(ifNode)
-
-  of nkBlockStmt:
-    result[1] = addGotoOut(result[1], gotoOut)
-    result[1] = ctx.transformBreaksInBlock(result[1], result[0], gotoOut)
-    result[1] = ctx.transformClosureIteratorBody(result[1], gotoOut)
-
-  of nkTryStmt, nkHiddenTryStmt:
-    # See explanation above about how this works
-    ctx.hasExceptions = true
-
-    result = newNodeI(nkGotoState, n.info)
-    var tryBody = toStmtList(n[0])
-    var exceptBody = ctx.collectExceptState(n)
-    var finallyBody = newTree(nkStmtList, getFinallyNode(ctx, n))
-    finallyBody = ctx.transformReturnsInTry(finallyBody)
-    finallyBody.add(ctx.newEndFinallyNode(finallyBody.info))
-
-    # The following index calculation is based on the knowledge how state
-    # indexes are assigned
-    let tryIdx = ctx.states.len
-    var exceptIdx, finallyIdx: int
-    if exceptBody.kind != nkEmpty:
-      exceptIdx = -(tryIdx + 1)
-      finallyIdx = tryIdx + 2
-    else:
-      exceptIdx = tryIdx + 1
-      finallyIdx = tryIdx + 1
-
-    let outToFinally = newNodeI(nkGotoState, finallyBody.info)
-
-    block: # Create initial states.
-      let oldExcHandlingState = ctx.curExcHandlingState
-      ctx.curExcHandlingState = exceptIdx
-      let realTryIdx = ctx.newState(tryBody, result)
-      assert(realTryIdx == tryIdx)
+      result = newNodeI(nkGotoState, n.info)
+      var tryBody = toStmtList(n[0])
+      var exceptBody = ctx.collectExceptState(n)
+      var finallyBody = newTree(nkStmtList, getFinallyNode(ctx, n))
+      finallyBody = ctx.transformReturnsInTry(finallyBody)
+      finallyBody.add(ctx.newEndFinallyNode(finallyBody.info))
 
+      # The following index calculation is based on the knowledge how state
+      # indexes are assigned
+      let tryIdx = ctx.states.len
+      var exceptIdx, finallyIdx: int
       if exceptBody.kind != nkEmpty:
+        exceptIdx = -(tryIdx + 1)
+        finallyIdx = tryIdx + 2
+      else:
+        exceptIdx = tryIdx + 1
+        finallyIdx = tryIdx + 1
+
+      let outToFinally = newNodeI(nkGotoState, finallyBody.info)
+
+      block: # Create initial states.
+        let oldExcHandlingState = ctx.curExcHandlingState
+        ctx.curExcHandlingState = exceptIdx
+        let realTryIdx = ctx.newState(tryBody, result)
+        assert(realTryIdx == tryIdx)
+
+        if exceptBody.kind != nkEmpty:
+          ctx.curExcHandlingState = finallyIdx
+          let realExceptIdx = ctx.newState(exceptBody, nil)
+          assert(realExceptIdx == -exceptIdx)
+
+        ctx.curExcHandlingState = oldExcHandlingState
+        let realFinallyIdx = ctx.newState(finallyBody, outToFinally)
+        assert(realFinallyIdx == finallyIdx)
+
+      block: # Subdivide the states
+        let oldNearestFinally = ctx.nearestFinally
+        ctx.nearestFinally = finallyIdx
+
+        let oldExcHandlingState = ctx.curExcHandlingState
+
+        ctx.curExcHandlingState = exceptIdx
+
+        if ctx.transformReturnsInTry(tryBody) != tryBody:
+          internalError(ctx.g.config, "transformReturnsInTry != tryBody")
+        if ctx.transformClosureIteratorBody(tryBody, outToFinally) != tryBody:
+          internalError(ctx.g.config, "transformClosureIteratorBody != tryBody")
+
         ctx.curExcHandlingState = finallyIdx
-        let realExceptIdx = ctx.newState(exceptBody, nil)
-        assert(realExceptIdx == -exceptIdx)
+        ctx.addElseToExcept(exceptBody)
+        if ctx.transformReturnsInTry(exceptBody) != exceptBody:
+          internalError(ctx.g.config, "transformReturnsInTry != exceptBody")
+        if ctx.transformClosureIteratorBody(exceptBody, outToFinally) != exceptBody:
+          internalError(ctx.g.config, "transformClosureIteratorBody != exceptBody")
 
-      ctx.curExcHandlingState = oldExcHandlingState
-      let realFinallyIdx = ctx.newState(finallyBody, outToFinally)
-      assert(realFinallyIdx == finallyIdx)
+        ctx.curExcHandlingState = oldExcHandlingState
+        ctx.nearestFinally = oldNearestFinally
+        if ctx.transformClosureIteratorBody(finallyBody, gotoOut) != finallyBody:
+          internalError(ctx.g.config, "transformClosureIteratorBody != finallyBody")
 
-    block: # Subdivide the states
-      let oldNearestFinally = ctx.nearestFinally
-      ctx.nearestFinally = finallyIdx
+    of nkGotoState, nkForStmt:
+      internalError(ctx.g.config, "closure iter " & $n.kind)
 
-      let oldExcHandlingState = ctx.curExcHandlingState
-
-      ctx.curExcHandlingState = exceptIdx
-
-      if ctx.transformReturnsInTry(tryBody) != tryBody:
-        internalError(ctx.g.config, "transformReturnsInTry != tryBody")
-      if ctx.transformClosureIteratorBody(tryBody, outToFinally) != tryBody:
-        internalError(ctx.g.config, "transformClosureIteratorBody != tryBody")
-
-      ctx.curExcHandlingState = finallyIdx
-      ctx.addElseToExcept(exceptBody)
-      if ctx.transformReturnsInTry(exceptBody) != exceptBody:
-        internalError(ctx.g.config, "transformReturnsInTry != exceptBody")
-      if ctx.transformClosureIteratorBody(exceptBody, outToFinally) != exceptBody:
-        internalError(ctx.g.config, "transformClosureIteratorBody != exceptBody")
-
-      ctx.curExcHandlingState = oldExcHandlingState
-      ctx.nearestFinally = oldNearestFinally
-      if ctx.transformClosureIteratorBody(finallyBody, gotoOut) != finallyBody:
-        internalError(ctx.g.config, "transformClosureIteratorBody != finallyBody")
-
-  of nkGotoState, nkForStmt:
-    internalError(ctx.g.config, "closure iter " & $n.kind)
-
-  else:
-    for i in 0 ..< n.len:
-      n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
+    else:
+      for i in 0 ..< n.len:
+        n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
 
 proc stateFromGotoState(n: PNode): int =
   assert(n.kind == nkGotoState)
@@ -1148,6 +1149,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
 
   # :state = exceptionTable[:state]
   block:
+
     # exceptionTable[:state]
     let getNextState = newTree(nkBracketExpr,
       ctx.createExceptionTable(),
@@ -1202,7 +1204,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
   block:
     result.add(newTree(nkAsgn,
       ctx.newCurExcAccess(),
-      ctx.g.callCodegenProc("getCurrentException")))
+      ctx.g.callCodegenProc("getCurrentException", ctx.g.emptyNode)))
 
 proc wrapIntoTryExcept(ctx: var Ctx, n: PNode): PNode {.inline.} =
   let setupExc = newTree(nkCall,

compiler/cmdlinehelper.nim

@@ -48,10 +48,9 @@ proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: Confi
   if self.suggestMode:
     conf.command = "nimsuggest"
 
-  template runNimScriptIfExists(path: AbsoluteFile) =
-    let p = path # eval once
-    if fileExists(p):
-      runNimScript(cache, p, freshDefines = false, conf)
+  proc runNimScriptIfExists(path: AbsoluteFile)=
+    if fileExists(path):
+      runNimScript(cache, path, freshDefines = false, conf)
 
   # Caution: make sure this stays in sync with `loadConfigs`
   if optSkipSystemConfigFile notin conf.globalOptions:

compiler/commands.nim

@@ -17,7 +17,6 @@ template bootSwitch(name, expr, userString) =
   const name = if expr: " " & userString else: ""
 
 bootSwitch(usedRelease, defined(release), "-d:release")
-bootSwitch(usedDanger, defined(danger), "-d:danger")
 bootSwitch(usedGnuReadline, defined(useLinenoise), "-d:useLinenoise")
 bootSwitch(usedBoehm, defined(boehmgc), "--gc:boehm")
 bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
@@ -27,17 +26,15 @@ bootSwitch(usedNoGC, defined(nogc), "--gc:none")
 
 import
   os, msgs, options, nversion, condsyms, strutils, extccomp, platform,
-  wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos,
-  pathutils, strtabs
-
-from incremental import nimIncremental
+  wordrecg, parseutils, nimblecmd, idents, parseopt, sequtils, lineinfos,
+  pathutils
 
 # but some have deps to imported modules. Yay.
 bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
 bootSwitch(usedNativeStacktrace,
   defined(nativeStackTrace) and nativeStackTraceSupported,
   "-d:nativeStackTrace")
-bootSwitch(usedFFI, hasFFI, "-d:nimHasLibFFI")
+bootSwitch(usedFFI, hasFFI, "-d:useFFI")
 
 type
   TCmdLinePass* = enum
@@ -50,16 +47,15 @@ const
       "Compiled at $4\n" &
       "Copyright (c) 2006-" & copyrightYear & " by Andreas Rumpf\n"
 
-proc genFeatureDesc[T: enum](t: typedesc[T]): string {.compileTime.} =
-  var x = ""
-  for f in low(T)..high(T):
-    if x.len > 0: x.add "|"
-    x.add $f
-  x
-
 const
   Usage = slurp"../doc/basicopt.txt".replace(" //", " ")
-  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", " ") % [genFeatureDesc(Feature), genFeatureDesc(LegacyFeature)]
+  FeatureDesc = block:
+    var x = ""
+    for f in low(Feature)..high(Feature):
+      if x.len > 0: x.add "|"
+      x.add $f
+    x
+  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", " ") % FeatureDesc
 
 proc getCommandLineDesc(conf: ConfigRef): string =
   result = (HelpMessage % [VersionAsString, platform.OS[conf.target.hostOS].name,
@@ -100,24 +96,25 @@ proc writeVersionInfo(conf: ConfigRef; pass: TCmdLinePass) =
     when gitHash.len == 40:
       msgWriteln(conf, "git hash: " & gitHash, {msgStdout})
 
-    msgWriteln(conf, "active boot switches:" & usedRelease & usedDanger &
+    msgWriteln(conf, "active boot switches:" & usedRelease &
       usedTinyC & usedGnuReadline & usedNativeStacktrace &
       usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC,
                {msgStdout})
     msgQuit(0)
 
-proc writeCommandLineUsage*(conf: ConfigRef) =
-  msgWriteln(conf, getCommandLineDesc(conf), {msgStdout})
+proc writeCommandLineUsage*(conf: ConfigRef; helpWritten: var bool) =
+  if not helpWritten:
+    msgWriteln(conf, getCommandLineDesc(conf), {msgStdout})
+    helpWritten = true
 
 proc addPrefix(switch: string): string =
-  if len(switch) <= 1: result = "-" & switch
+  if len(switch) == 1: result = "-" & switch
   else: result = "--" & switch
 
 const
   errInvalidCmdLineOption = "invalid command line option: '$1'"
   errOnOrOffExpectedButXFound = "'on' or 'off' expected, but '$1' found"
   errOnOffOrListExpectedButXFound = "'on', 'off' or 'list' expected, but '$1' found"
-  errOffHintsError = "'off', 'hint' or 'error' expected, but '$1' found"
 
 proc invalidCmdLineOption(conf: ConfigRef; pass: TCmdLinePass, switch: string, info: TLineInfo) =
   if switch == " ": localError(conf, info, errInvalidCmdLineOption % "-")
@@ -144,7 +141,7 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
 proc processOnOffSwitch(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass,
                         info: TLineInfo) =
   case arg.normalize
-  of "","on": conf.options = conf.options + op
+  of "on": conf.options = conf.options + op
   of "off": conf.options = conf.options - op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
@@ -160,7 +157,7 @@ proc processOnOffSwitchOrList(conf: ConfigRef; op: TOptions, arg: string, pass:
 proc processOnOffSwitchG(conf: ConfigRef; op: TGlobalOptions, arg: string, pass: TCmdLinePass,
                          info: TLineInfo) =
   case arg.normalize
-  of "", "on": conf.globalOptions = conf.globalOptions + op
+  of "on": conf.globalOptions = conf.globalOptions + op
   of "off": conf.globalOptions = conf.globalOptions - op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
@@ -226,7 +223,7 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
     case arg.normalize
     of "boehm":        result = conf.selectedGC == gcBoehm
     of "refc":         result = conf.selectedGC == gcRefc
-    of "v2":           result = false
+    of "v2":           result = conf.selectedGC == gcV2
     of "markandsweep": result = conf.selectedGC == gcMarkAndSweep
     of "generational": result = false
     of "destructors":  result = conf.selectedGC == gcDestructors
@@ -279,9 +276,8 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "fieldchecks": result = contains(conf.options, optFieldCheck)
   of "rangechecks": result = contains(conf.options, optRangeCheck)
   of "boundchecks": result = contains(conf.options, optBoundsCheck)
-  of "refchecks": result = contains(conf.options, optRefCheck)
   of "overflowchecks": result = contains(conf.options, optOverflowCheck)
-  of "stylechecks": result = contains(conf.options, optStyleCheck)
+  of "movechecks": result = contains(conf.options, optMoveCheck)
   of "linedir": result = contains(conf.options, optLineDir)
   of "assertions", "a": result = contains(conf.options, optAssert)
   of "run", "r": result = contains(conf.globalOptions, optRun)
@@ -291,10 +287,9 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "taintmode": result = contains(conf.globalOptions, optTaintMode)
   of "tlsemulation": result = contains(conf.globalOptions, optTlsEmulation)
   of "implicitstatic": result = contains(conf.options, optImplicitStatic)
-  of "patterns", "trmacros": result = contains(conf.options, optTrMacros)
+  of "patterns": result = contains(conf.options, optPatterns)
   of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace)
   of "nilseqs": result = contains(conf.options, optNilSeqs)
-  of "oldast": result = contains(conf.options, optOldAst)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc processPath(conf: ConfigRef; path: string, info: TLineInfo,
@@ -390,12 +385,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     conf.nimcacheDir = processPath(conf, arg, info, true)
   of "out", "o":
     expectArg(conf, switch, arg, pass, info)
-    let f = splitFile(arg.expandTilde)
-    conf.outFile = RelativeFile f.name & f.ext
-    conf.outDir = toAbsoluteDir f.dir
-  of "outdir":
-    expectArg(conf, switch, arg, pass, info)
-    conf.outDir = toAbsoluteDir arg.expandTilde
+    conf.outFile = AbsoluteFile arg
   of "docseesrcurl":
     expectArg(conf, switch, arg, pass, info)
     conf.docSeeSrcUrl = arg
@@ -422,19 +412,26 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if pass in {passCmd2, passPP}:
       addExternalFileToLink(conf, AbsoluteFile arg)
   of "debuginfo":
-    processOnOffSwitchG(conf, {optCDebug}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optCDebug)
   of "embedsrc":
-    processOnOffSwitchG(conf, {optEmbedOrigSrc}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optEmbedOrigSrc)
   of "compileonly", "c":
-    processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optCompileOnly)
   of "nolinking":
-    processOnOffSwitchG(conf, {optNoLinking}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optNoLinking)
   of "nomain":
-    processOnOffSwitchG(conf, {optNoMain}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optNoMain)
   of "forcebuild", "f":
-    processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optForceFullMake)
   of "project":
-    processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl conf.globalOptions, optWholeProject
   of "gc":
     expectArg(conf, switch, arg, pass, info)
     case arg.normalize
@@ -444,7 +441,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "refc":
       conf.selectedGC = gcRefc
     of "v2":
-      message(conf, info, warnDeprecated, "--gc:v2 is deprecated; using default gc")
+      conf.selectedGC = gcV2
     of "markandsweep":
       conf.selectedGC = gcMarkAndSweep
       defineSymbol(conf.symbols, "gcmarkandsweep")
@@ -482,15 +479,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "native", "gdb":
       incl(conf.globalOptions, optCDebug)
       conf.options = conf.options + {optLineDir} - {optEndb}
-      #defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
+      defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
       undefSymbol(conf.symbols, "endb")
     else:
       localError(conf, info, "expected endb|gdb but found " & arg)
-  of "g": # alias for --debugger:native
-    incl(conf.globalOptions, optCDebug)
-    conf.options = conf.options + {optLineDir} - {optEndb}
-    #defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
-    undefSymbol(conf.symbols, "endb")
   of "profiler":
     processOnOffSwitch(conf, {optProfiler}, arg, pass, info)
     if optProfiler in conf.options: defineSymbol(conf.symbols, "profiler")
@@ -500,20 +492,12 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if optMemTracker in conf.options: defineSymbol(conf.symbols, "memtracker")
     else: undefSymbol(conf.symbols, "memtracker")
   of "hotcodereloading":
-    processOnOffSwitchG(conf, {optHotCodeReloading}, arg, pass, info)
-    if conf.hcrOn:
-      defineSymbol(conf.symbols, "hotcodereloading")
-      defineSymbol(conf.symbols, "useNimRtl")
-      # hardcoded linking with dynamic runtime for MSVC for smaller binaries
-      # should do the same for all compilers (wherever applicable)
-      if isVSCompatible(conf):
-        extccomp.addCompileOptionCmd(conf, "/MD")
-    else:
-      undefSymbol(conf.symbols, "hotcodereloading")
-      undefSymbol(conf.symbols, "useNimRtl")
+    processOnOffSwitch(conf, {optHotCodeReloading}, arg, pass, info)
+    if optHotCodeReloading in conf.options: defineSymbol(conf.symbols, "hotcodereloading")
+    else: undefSymbol(conf.symbols, "hotcodereloading")
   of "oldnewlines":
     case arg.normalize
-    of "","on":
+    of "on":
       conf.oldNewlines = true
       defineSymbol(conf.symbols, "nimOldNewlines")
     of "off":
@@ -523,7 +507,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       localError(conf, info, errOnOrOffExpectedButXFound % arg)
   of "laxstrings": processOnOffSwitch(conf, {optLaxStrings}, arg, pass, info)
   of "nilseqs": processOnOffSwitch(conf, {optNilSeqs}, arg, pass, info)
-  of "oldast": processOnOffSwitch(conf, {optOldAst}, arg, pass, info)
   of "checks", "x": processOnOffSwitch(conf, ChecksOptions, arg, pass, info)
   of "floatchecks":
     processOnOffSwitch(conf, {optNaNCheck, optInfCheck}, arg, pass, info)
@@ -534,9 +517,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "fieldchecks": processOnOffSwitch(conf, {optFieldCheck}, arg, pass, info)
   of "rangechecks": processOnOffSwitch(conf, {optRangeCheck}, arg, pass, info)
   of "boundchecks": processOnOffSwitch(conf, {optBoundsCheck}, arg, pass, info)
-  of "refchecks": processOnOffSwitch(conf, {optRefCheck}, arg, pass, info)
   of "overflowchecks": processOnOffSwitch(conf, {optOverflowCheck}, arg, pass, info)
-  of "stylechecks": processOnOffSwitch(conf, {optStyleCheck}, arg, pass, info)
+  of "movechecks": processOnOffSwitch(conf, {optMoveCheck}, arg, pass, info)
   of "linedir": processOnOffSwitch(conf, {optLineDir}, arg, pass, info)
   of "assertions", "a": processOnOffSwitch(conf, {optAssert}, arg, pass, info)
   of "deadcodeelim": discard # deprecated, dead code elim always on
@@ -547,8 +529,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "taintmode": processOnOffSwitchG(conf, {optTaintMode}, arg, pass, info)
   of "implicitstatic":
     processOnOffSwitch(conf, {optImplicitStatic}, arg, pass, info)
-  of "patterns", "trmacros":
-    processOnOffSwitch(conf, {optTrMacros}, arg, pass, info)
+  of "patterns":
+    processOnOffSwitch(conf, {optPatterns}, arg, pass, info)
   of "opt":
     expectArg(conf, switch, arg, pass, info)
     case arg.normalize
@@ -607,44 +589,33 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitchG(conf, {optGenIndex}, arg, pass, info)
   of "import":
     expectArg(conf, switch, arg, pass, info)
-    if pass in {passCmd2, passPP}:
-      conf.implicitImports.add findModule(conf, arg, toFullPath(conf, info)).string
+    if pass in {passCmd2, passPP}: conf.implicitImports.add arg
   of "include":
     expectArg(conf, switch, arg, pass, info)
-    if pass in {passCmd2, passPP}:
-      conf.implicitIncludes.add findModule(conf, arg, toFullPath(conf, info)).string
+    if pass in {passCmd2, passPP}: conf.implicitIncludes.add arg
   of "listcmd":
-    processOnOffSwitchG(conf, {optListCmd}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optListCmd)
   of "genmapping":
-    processOnOffSwitchG(conf, {optGenMapping}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optGenMapping)
   of "os":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd1, passPP}:
       let theOS = platform.nameToOS(arg)
-      if theOS == osNone:
-        let osList = platform.listOSnames().join(", ")
-        localError(conf, info, "unknown OS: '$1'. Available options are: $2" % [arg, $osList])
+      if theOS == osNone: localError(conf, info, "unknown OS: '$1'" % arg)
       elif theOS != conf.target.hostOS:
         setTarget(conf.target, theOS, conf.target.targetCPU)
   of "cpu":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd1, passPP}:
       let cpu = platform.nameToCPU(arg)
-      if cpu == cpuNone:
-        let cpuList = platform.listCPUnames().join(", ")
-        localError(conf, info, "unknown CPU: '$1'. Available options are: $2" % [ arg, cpuList])
+      if cpu == cpuNone: localError(conf, info, "unknown CPU: '$1'" % arg)
       elif cpu != conf.target.hostCPU:
         setTarget(conf.target, conf.target.targetOS, cpu)
   of "run", "r":
-    processOnOffSwitchG(conf, {optRun}, arg, pass, info)
-  of "errormax":
-    expectArg(conf, switch, arg, pass, info)
-    # Note: `nim check` (etc) can overwrite this.
-    # `0` is meaningless, give it a useful meaning as in clang's -ferror-limit
-    # If user doesn't set this flag and the code doesn't either, it'd
-    # have the same effect as errorMax = 1
-    let ret = parseInt(arg)
-    conf.errorMax = if ret == 0: high(int) else: ret
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optRun)
   of "verbosity":
     expectArg(conf, switch, arg, pass, info)
     let verbosity = parseInt(arg)
@@ -670,30 +641,30 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "help", "h":
     expectNoArg(conf, switch, arg, pass, info)
     helpOnError(conf, pass)
-  of "symbolfiles": discard "ignore for backwards compat"
-  of "incremental":
-    when not nimIncremental:
-      localError(conf, info, "the compiler was not built with " &
-        "incremental compilation features; bootstrap with " &
-        "-d:nimIncremental to enable")
+  of "symbolfiles", "incremental":
     case arg.normalize
     of "on": conf.symbolFiles = v2Sf
     of "off": conf.symbolFiles = disabledSf
     of "writeonly": conf.symbolFiles = writeOnlySf
     of "readonly": conf.symbolFiles = readOnlySf
     of "v2": conf.symbolFiles = v2Sf
-    else: localError(conf, info, "invalid option for --incremental: " & arg)
+    else: localError(conf, info, "invalid option for --symbolFiles: " & arg)
   of "skipcfg":
-    processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optSkipSystemConfigFile)
   of "skipprojcfg":
-    processOnOffSwitchG(conf, {optSkipProjConfigFile}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optSkipProjConfigFile)
   of "skipusercfg":
-    processOnOffSwitchG(conf, {optSkipUserConfigFile}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optSkipUserConfigFile)
   of "skipparentcfg":
-    processOnOffSwitchG(conf, {optSkipParentConfigFiles}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optSkipParentConfigFiles)
   of "genscript", "gendeps":
-    processOnOffSwitchG(conf, {optGenScript}, arg, pass, info)
-    processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optGenScript)
+    incl(conf.globalOptions, optCompileOnly)
   of "colors": processOnOffSwitchG(conf, {optUseColors}, arg, pass, info)
   of "lib":
     expectArg(conf, switch, arg, pass, info)
@@ -727,13 +698,16 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectNoArg(conf, switch, arg, pass, info)
     conf.ideCmd = ideUse
   of "stdout":
-    processOnOffSwitchG(conf, {optStdout}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl(conf.globalOptions, optStdout)
   of "listfullpaths":
-    processOnOffSwitchG(conf, {optListFullPaths}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl conf.globalOptions, optListFullPaths
   of "dynliboverride":
     dynlibOverride(conf, switch, arg, pass, info)
   of "dynliboverrideall":
-    processOnOffSwitchG(conf, {optDynlibOverrideAll}, arg, pass, info)
+    expectNoArg(conf, switch, arg, pass, info)
+    incl conf.globalOptions, optDynlibOverrideAll
   of "cs":
     # only supported for compatibility. Does nothing.
     expectArg(conf, switch, arg, pass, info)
@@ -745,11 +719,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         conf.features.incl parseEnum[Feature](arg)
       except ValueError:
         localError(conf, info, "unknown experimental feature")
-  of "legacy":
-    try:
-      conf.legacyFeatures.incl parseEnum[LegacyFeature](arg)
-    except ValueError:
-      localError(conf, info, "unknown obsolete feature")
   of "nocppexceptions":
     expectNoArg(conf, switch, arg, pass, info)
     incl(conf.globalOptions, optNoCppExceptions)
@@ -760,36 +729,17 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       conf.cppDefine(arg)
   of "newruntime":
     expectNoArg(conf, switch, arg, pass, info)
-    if pass in {passCmd2, passPP}:
-      doAssert(conf != nil)
-      incl(conf.features, destructor)
-      incl(conf.globalOptions, optNimV2)
-      defineSymbol(conf.symbols, "nimV2")
-      conf.selectedGC = gcDestructors
-      defineSymbol(conf.symbols, "gcdestructors")
-  of "stylecheck":
-    case arg.normalize
-    of "off": conf.globalOptions = conf.globalOptions - {optStyleHint, optStyleError}
-    of "hint": conf.globalOptions = conf.globalOptions + {optStyleHint}
-    of "error": conf.globalOptions = conf.globalOptions + {optStyleError}
-    else: localError(conf, info, errOffHintsError % arg)
-  of "showallmismatches":
-    processOnOffSwitchG(conf, {optShowAllMismatches}, arg, pass, info)
+    doAssert(conf != nil)
+    incl(conf.features, destructor)
+    defineSymbol(conf.symbols, "nimNewRuntime")
+  of "nep1":
+    processOnOffSwitchG(conf, {optCheckNep1}, arg, pass, info)
   of "cppcompiletonamespace":
     if arg.len > 0:
       conf.cppCustomNamespace = arg
     else:
       conf.cppCustomNamespace = "Nim"
     defineSymbol(conf.symbols, "cppCompileToNamespace", conf.cppCustomNamespace)
-  of "docinternal":
-    processOnOffSwitchG(conf, {optDocInternal}, arg, pass, info)
-  of "multimethods":
-    processOnOffSwitchG(conf, {optMultiMethods}, arg, pass, info)
-  of "expandmacro":
-    expectArg(conf, switch, arg, pass, info)
-    conf.macrosToExpand[arg] = "T"
-  of "":
-    conf.projectName = "-"
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)
@@ -817,7 +767,7 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser;
                       argsCount: var int; config: ConfigRef): bool =
   if argsCount == 0:
     # nim filename.nims  is the same as "nim e filename.nims":
-    if p.key.endsWith(".nims"):
+    if p.key.endswith(".nims"):
       config.command = "e"
       incl(config.globalOptions, optWasNimscript)
       config.projectName = unixToNativePath(p.key)
@@ -829,8 +779,7 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser;
     if pass == passCmd1: config.commandArgs.add p.key
     if argsCount == 1:
       # support UNIX style filenames everywhere for portable build scripts:
-      if config.projectName.len == 0:
-        config.projectName = unixToNativePath(p.key)
+      config.projectName = unixToNativePath(p.key)
       config.arguments = cmdLineRest(p)
       result = true
   inc argsCount

compiler/condsyms.nim

@@ -10,26 +10,31 @@
 # This module handles the conditional symbols.
 
 import
-  strtabs
+  strtabs, platform, strutils, idents
 
 from options import Feature
 from lineinfos import HintsToStr, WarningsToStr
 
+const
+  catNone = "false"
+
 proc defineSymbol*(symbols: StringTableRef; symbol: string, value: string = "true") =
   symbols[symbol] = value
 
 proc undefSymbol*(symbols: StringTableRef; symbol: string) =
-  symbols.del(symbol)
+  symbols[symbol] = catNone
 
 #proc lookupSymbol*(symbols: StringTableRef; symbol: string): string =
 #  result = if isDefined(symbol): gSymbols[symbol] else: nil
 
 iterator definedSymbolNames*(symbols: StringTableRef): string =
   for key, val in pairs(symbols):
-    yield key
+    if val != catNone: yield key
 
 proc countDefinedSymbols*(symbols: StringTableRef): int =
-  symbols.len
+  result = 0
+  for key, val in pairs(symbols):
+    if val != catNone: inc(result)
 
 proc initDefines*(symbols: StringTableRef) =
   # for bootstrapping purposes and old code:
@@ -53,7 +58,7 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimtypedescfixed")
   defineSymbol("nimKnowsNimvm")
   defineSymbol("nimArrIdx")
-  defineSymbol("nimHasalignOf")
+  defineSymbol("nimImmediateDeprecated")
   defineSymbol("nimNewShiftOps")
   defineSymbol("nimDistros")
   defineSymbol("nimHasCppDefine")
@@ -77,16 +82,8 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimNoNilSeqs")
   defineSymbol("nimNoNilSeqs2")
   defineSymbol("nimHasUserErrors")
-  defineSymbol("nimUncheckedArrayTyp")
-  defineSymbol("nimHasTypeof")
-  defineSymbol("nimErrorProcCanHaveBody")
-  defineSymbol("nimHasInstantiationOfInMacro")
-  defineSymbol("nimHasHotCodeReloading")
+
   defineSymbol("nimHasNilSeqs")
-  defineSymbol("nimHasSignatureHashInMacro")
-  defineSymbol("nimHasDefault")
-  defineSymbol("nimMacrosSizealignof")
-  defineSymbol("nimNoZeroExtendMagic")
   for f in low(Feature)..high(Feature):
     defineSymbol("nimHas" & $f)
 
@@ -94,6 +91,3 @@ proc initDefines*(symbols: StringTableRef) =
     defineSymbol("nimHasWarning" & s)
   for s in HintsToStr:
     defineSymbol("nimHasHint" & s)
-
-  defineSymbol("nimFixedOwned")
-  defineSymbol("nimHasStyleChecks")

compiler/cursors.nim

@@ -1,72 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2019 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-import
-  intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
-  strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
-  lineinfos, parampatterns
-
-##[
-This module implements "cursor" detection. A cursor is a local variable
-that is used for navigation in a datastructure, it does not "own" the
-data it aliases but it might update the underlying datastructure.
-
-Two primary examples for cursors that I have in mind and that are critical
-for optimization:
-
-1. Local string variable introduced by ``for x in a``::
-
-  var i = 0
-  while i < a.len:
-    let cursor = a[i]
-    use cursor
-    inc i
-
-2. Local ``ref`` variable for navigation::
-
-  var cursor = listHead
-  while cursor != nil:
-    use cursor
-    cursor = cursor.next
-
-Cursors are very interesting for the optimizer because they can be copyMem'ed
-and don't need a destructor.
-
-More formally, a cursor is a variable that is set on all paths to
-a *location* or a proc call that produced a ``lent/var`` type. All statements
-that come after these assignments MUST not mutate what the cursor aliases.
-
-Mutations *through* the cursor are allowed if the cursor has ref semantics.
-
-Look at this complex real world example taken from the compiler itself:
-
-.. code-block:: Nim
-
-  proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope =
-    var t = typ
-    while true:
-      if t.sym != nil and {sfImportc, sfExportc} * t.sym.flags != {}:
-        return t.sym.loc.r
-
-      if t.kind in irrelevantForBackend:
-        t = t.lastSon
-      else:
-        break
-    let typ = if typ.kind in {tyAlias, tySink, tyOwned}: typ.lastSon else: typ
-    if typ.loc.r == nil:
-      typ.loc.r = typ.typeName & $sig
-    result = typ.loc.r
-    if result == nil: internalError(m.config, "getTypeName: " & $typ.kind)
-
-Here `t` is a cursor but without a control flow based analysis we are unlikely
-to detect it.
-
-]##
-
-# Araq: I owe you an implementation. For now use the .cursor pragma. :-/

compiler/depends.nim

@@ -10,7 +10,8 @@
 # This module implements a dependency file generator.
 
 import
-  options, ast, ropes, idents, passes, modulepaths, pathutils
+  os, options, ast, astalgo, msgs, ropes, idents, passes, modulepaths,
+  pathutils
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -34,14 +35,14 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode =
   let b = Backend(g.graph.backend)
   case n.kind
   of nkImportStmt:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       var imported = getModuleName(g.config, n.sons[i])
       addDependencyAux(b, g.module.name.s, imported)
   of nkFromStmt, nkImportExceptStmt:
     var imported = getModuleName(g.config, n.sons[0])
     addDependencyAux(b, g.module.name.s, imported)
   of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr:
-    for i in 0 ..< sonsLen(n): discard addDotDependency(c, n.sons[i])
+    for i in countup(0, sonsLen(n) - 1): discard addDotDependency(c, n.sons[i])
   else:
     discard
 

compiler/destroyer.nim

@@ -0,0 +1,483 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Injects destructor calls into Nim code as well as
+## an optimizer that optimizes copies to moves. This is implemented as an
+## AST to AST transformation so that every backend benefits from it.
+
+## Rules for destructor injections:
+##
+## foo(bar(X(), Y()))
+## X and Y get destroyed after bar completes:
+##
+## foo( (tmpX = X(); tmpY = Y(); tmpBar = bar(tmpX, tmpY);
+##       destroy(tmpX); destroy(tmpY);
+##       tmpBar))
+## destroy(tmpBar)
+##
+## var x = f()
+## body
+##
+## is the same as:
+##
+##  var x;
+##  try:
+##    move(x, f())
+##  finally:
+##    destroy(x)
+##
+## But this really just an optimization that tries to avoid to
+## introduce too many temporaries, the 'destroy' is caused by
+## the 'f()' call. No! That is not true for 'result = f()'!
+##
+## x = y where y is read only once
+## is the same as:  move(x, y)
+##
+## Actually the more general rule is: The *last* read of ``y``
+## can become a move if ``y`` is the result of a construction.
+##
+## We also need to keep in mind here that the number of reads is
+## control flow dependent:
+## let x = foo()
+## while true:
+##   y = x  # only one read, but the 2nd iteration will fail!
+## This also affects recursions! Only usages that do not cross
+## a loop boundary (scope) and are not used in function calls
+## are safe.
+##
+##
+## x = f() is the same as:  move(x, f())
+##
+## x = y
+## is the same as:  copy(x, y)
+##
+## Reassignment works under this scheme:
+## var x = f()
+## x = y
+##
+## is the same as:
+##
+##  var x;
+##  try:
+##    move(x, f())
+##    copy(x, y)
+##  finally:
+##    destroy(x)
+##
+##  result = f()  must not destroy 'result'!
+##
+## The produced temporaries clutter up the code and might lead to
+## inefficiencies. A better strategy is to collect all the temporaries
+## in a single object that we put into a single try-finally that
+## surrounds the proc body. This means the code stays quite efficient
+## when compiled to C. In fact, we do the same for variables, so
+## destructors are called when the proc returns, not at scope exit!
+## This makes certains idioms easier to support. (Taking the slice
+## of a temporary object.)
+##
+## foo(bar(X(), Y()))
+## X and Y get destroyed after bar completes:
+##
+## var tmp: object
+## foo( (move tmp.x, X(); move tmp.y, Y(); tmp.bar = bar(tmpX, tmpY);
+##       tmp.bar))
+## destroy(tmp.bar)
+## destroy(tmp.x); destroy(tmp.y)
+##
+
+##[
+From https://github.com/nim-lang/Nim/wiki/Destructors
+
+Rule      Pattern                 Transformed into
+----      -------                 ----------------
+1.1	      var x: T; stmts	        var x: T; try stmts
+                                  finally: `=destroy`(x)
+1.2       var x: sink T; stmts    var x: sink T; stmts; ensureEmpty(x)
+2         x = f()                 `=sink`(x, f())
+3         x = lastReadOf z        `=sink`(x, z); wasMoved(z)
+4.1       y = sinkParam           `=sink`(y, sinkParam)
+4.2       x = y                   `=`(x, y) # a copy
+5.1       f_sink(g())             f_sink(g())
+5.2       f_sink(y)               f_sink(copy y); # copy unless we can see it's the last read
+5.3       f_sink(move y)          f_sink(y); wasMoved(y) # explicit moves empties 'y'
+5.4       f_noSink(g())           var tmp = bitwiseCopy(g()); f(tmp); `=destroy`(tmp)
+
+Remarks: Rule 1.2 is not yet implemented because ``sink`` is currently
+  not allowed as a local variable.
+
+``move`` builtin needs to be implemented.
+]##
+
+import
+  intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
+  strutils, options, dfa, lowerings, tables, modulegraphs,
+  lineinfos
+
+const
+  InterestingSyms = {skVar, skResult, skLet}
+
+type
+  Con = object
+    owner: PSym
+    g: ControlFlowGraph
+    jumpTargets: IntSet
+    tmpObj: PType
+    tmp: PSym
+    destroys, topLevelVars: PNode
+    toDropBit: Table[int, PSym]
+    graph: ModuleGraph
+    emptyNode: PNode
+
+proc getTemp(c: var Con; typ: PType; info: TLineInfo): PNode =
+  # XXX why are temps fields in an object here?
+  let f = newSym(skField, getIdent(c.graph.cache, ":d" & $c.tmpObj.n.len), c.owner, info)
+  f.typ = typ
+  rawAddField c.tmpObj, f
+  result = rawDirectAccess(c.tmp, f)
+
+proc isHarmlessVar*(s: PSym; c: Con): bool =
+  # 's' is harmless if it used only once and its
+  # definition/usage are not split by any labels:
+  #
+  # let s = foo()
+  # while true:
+  #   a[i] = s
+  #
+  # produces:
+  #
+  # def s
+  # L1:
+  #   use s
+  # goto L1
+  #
+  # let s = foo()
+  # if cond:
+  #   a[i] = s
+  # else:
+  #   a[j] = s
+  #
+  # produces:
+  #
+  # def s
+  # fork L2
+  # use s
+  # goto L3
+  # L2:
+  # use s
+  # L3
+  #
+  # So this analysis is for now overly conservative, but correct.
+  var defsite = -1
+  var usages = 0
+  for i in 0..<c.g.len:
+    case c.g[i].kind
+    of def:
+      if c.g[i].sym == s:
+        if defsite < 0: defsite = i
+        else: return false
+    of use, useWithinCall:
+      if c.g[i].sym == s:
+        if defsite < 0: return false
+        for j in defsite .. i:
+          # not within the same basic block?
+          if j in c.jumpTargets: return false
+        # if we want to die after the first 'use':
+        if usages > 1: return false
+        inc usages
+    #of useWithinCall:
+    #  if c.g[i].sym == s: return false
+    of goto, fork:
+      discard "we do not perform an abstract interpretation yet"
+  result = usages <= 1
+
+template interestingSym(s: PSym): bool =
+  s.owner == c.owner and s.kind in InterestingSyms and hasDestructor(s.typ)
+
+proc patchHead(n: PNode) =
+  if n.kind in nkCallKinds and n[0].kind == nkSym and n.len > 1:
+    let s = n[0].sym
+    if s.name.s[0] == '=' and s.name.s in ["=sink", "=", "=destroy"]:
+      if sfFromGeneric in s.flags:
+        excl(s.flags, sfFromGeneric)
+        patchHead(s.getBody)
+      if n[1].typ.isNil:
+        # XXX toptree crashes without this workaround. Figure out why.
+        return
+      let t = n[1].typ.skipTypes({tyVar, tyLent, tyGenericInst, tyAlias, tySink, tyInferred})
+      template patch(op, field) =
+        if s.name.s == op and field != nil and field != s:
+          n.sons[0].sym = field
+      patch "=sink", t.sink
+      patch "=", t.assignment
+      patch "=destroy", t.destructor
+  for x in n:
+    patchHead(x)
+
+proc patchHead(s: PSym) =
+  if sfFromGeneric in s.flags:
+    patchHead(s.ast[bodyPos])
+
+proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
+  var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">"
+  if opname == "=" and ri != nil:
+    m.add "; requires a copy because it's not the last read of '"
+    m.add renderTree(ri)
+    m.add '\''
+  localError(c.graph.config, ri.info, errGenerated, m)
+
+template genOp(opr, opname, ri) =
+  let op = opr
+  if op == nil:
+    globalError(c.graph.config, dest.info, "internal error: '" & opname & "' operator not found for type " & typeToString(t))
+  elif op.ast[genericParamsPos].kind != nkEmpty:
+    globalError(c.graph.config, dest.info, "internal error: '" & opname & "' operator is generic")
+  patchHead op
+  if sfError in op.flags: checkForErrorPragma(c, t, ri, opname)
+  result = newTree(nkCall, newSymNode(op), newTree(nkHiddenAddr, dest))
+
+proc genSink(c: Con; t: PType; dest, ri: PNode): PNode =
+  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
+  genOp(if t.sink != nil: t.sink else: t.assignment, "=sink", ri)
+
+proc genCopy(c: Con; t: PType; dest, ri: PNode): PNode =
+  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
+  genOp(t.assignment, "=", ri)
+
+proc genDestroy(c: Con; t: PType; dest: PNode): PNode =
+  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
+  genOp(t.destructor, "=destroy", nil)
+
+proc addTopVar(c: var Con; v: PNode) =
+  c.topLevelVars.add newTree(nkIdentDefs, v, c.emptyNode, c.emptyNode)
+
+proc dropBit(c: var Con; s: PSym): PSym =
+  result = c.toDropBit.getOrDefault(s.id)
+  assert result != nil
+
+proc registerDropBit(c: var Con; s: PSym) =
+  let result = newSym(skTemp, getIdent(c.graph.cache, s.name.s & "_AliveBit"), c.owner, s.info)
+  result.typ = getSysType(c.graph, s.info, tyBool)
+  let trueVal = newIntTypeNode(nkIntLit, 1, result.typ)
+  c.topLevelVars.add newTree(nkIdentDefs, newSymNode result, c.emptyNode, trueVal)
+  c.toDropBit[s.id] = result
+  # generate:
+  #  if not sinkParam_AliveBit: `=destroy`(sinkParam)
+  let t = s.typ.skipTypes({tyGenericInst, tyAlias, tySink})
+  if t.destructor != nil:
+    c.destroys.add newTree(nkIfStmt,
+      newTree(nkElifBranch, newSymNode result, genDestroy(c, t, newSymNode s)))
+
+proc p(n: PNode; c: var Con): PNode
+
+template recurse(n, dest) =
+  for i in 0..<n.len:
+    dest.add p(n[i], c)
+
+proc isSinkParam(s: PSym): bool {.inline.} =
+  result = s.kind == skParam and s.typ.kind == tySink
+
+const constrExprs = nkCallKinds+{nkObjConstr}
+
+proc destructiveMoveSink(n: PNode; c: var Con): PNode =
+  # generate:  (chckMove(sinkParam_AliveBit); sinkParam_AliveBit = false; sinkParam)
+  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
+  let bit = newSymNode dropBit(c, n.sym)
+  if optMoveCheck in c.owner.options:
+    result.add callCodegenProc(c.graph, "chckMove", bit)
+  result.add newTree(nkAsgn, bit,
+    newIntTypeNode(nkIntLit, 0, getSysType(c.graph, n.info, tyBool)))
+  result.add n
+
+proc genMagicCall(n: PNode; c: var Con; magicname: string; m: TMagic): PNode =
+  result = newNodeI(nkCall, n.info)
+  result.add(newSymNode(createMagic(c.graph, magicname, m)))
+  result.add n
+
+proc moveOrCopy(dest, ri: PNode; c: var Con): PNode =
+  if ri.kind in constrExprs:
+    result = genSink(c, dest.typ, dest, ri)
+    # watch out and no not transform 'ri' twice if it's a call:
+    let ri2 = copyNode(ri)
+    recurse(ri, ri2)
+    result.add ri2
+  elif ri.kind == nkSym and ri.sym.kind != skParam and isHarmlessVar(ri.sym, c):
+    # Rule 3: `=sink`(x, z); wasMoved(z)
+    var snk = genSink(c, dest.typ, dest, ri)
+    snk.add p(ri, c)
+    result = newTree(nkStmtList, snk, genMagicCall(ri, c, "wasMoved", mWasMoved))
+  elif ri.kind == nkSym and isSinkParam(ri.sym):
+    result = genSink(c, dest.typ, dest, ri)
+    result.add destructiveMoveSink(ri, c)
+  else:
+    result = genCopy(c, dest.typ, dest, ri)
+    result.add p(ri, c)
+
+proc passCopyToSink(n: PNode; c: var Con): PNode =
+  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
+  let tmp = getTemp(c, n.typ, n.info)
+  if hasDestructor(n.typ):
+    var m = genCopy(c, n.typ, tmp, n)
+    m.add p(n, c)
+    result.add m
+    message(c.graph.config, n.info, hintPerformance,
+      ("passing '$1' to a sink parameter introduces an implicit copy; " &
+      "use 'move($1)' to prevent it") % $n)
+  else:
+    result.add newTree(nkAsgn, tmp, p(n, c))
+  result.add tmp
+
+proc genWasMoved(n: PNode; c: var Con): PNode =
+  # The mWasMoved builtin does not take the address.
+  result = genMagicCall(n, c, "wasMoved", mWasMoved)
+
+proc destructiveMoveVar(n: PNode; c: var Con): PNode =
+  # generate: (let tmp = v; reset(v); tmp)
+  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
+
+  var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), c.owner, n.info)
+  temp.typ = n.typ
+  var v = newNodeI(nkLetSection, n.info)
+  let tempAsNode = newSymNode(temp)
+
+  var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
+  vpart.sons[0] = tempAsNode
+  vpart.sons[1] = c.emptyNode
+  vpart.sons[2] = n
+  add(v, vpart)
+
+  result.add v
+  result.add genWasMoved(n, c)
+  result.add tempAsNode
+
+proc p(n: PNode; c: var Con): PNode =
+  case n.kind
+  of nkVarSection, nkLetSection:
+    discard "transform; var x = y to  var x; x op y  where op is a move or copy"
+    result = newNodeI(nkStmtList, n.info)
+
+    for i in 0..<n.len:
+      let it = n[i]
+      let L = it.len-1
+      let ri = it[L]
+      if it.kind == nkVarTuple and hasDestructor(ri.typ):
+        let x = lowerTupleUnpacking(c.graph, it, c.owner)
+        result.add p(x, c)
+      elif it.kind == nkIdentDefs and hasDestructor(it[0].typ):
+        for j in 0..L-2:
+          let v = it[j]
+          doAssert v.kind == nkSym
+          # move the variable declaration to the top of the frame:
+          c.addTopVar v
+          # make sure it's destroyed at the end of the proc:
+          c.destroys.add genDestroy(c, v.typ, v)
+          if ri.kind != nkEmpty:
+            let r = moveOrCopy(v, ri, c)
+            result.add r
+      else:
+        # keep it, but transform 'ri':
+        var varSection = copyNode(n)
+        var itCopy = copyNode(it)
+        for j in 0..L-1:
+          itCopy.add it[j]
+        itCopy.add p(ri, c)
+        varSection.add itCopy
+        result.add varSection
+  of nkCallKinds:
+    let parameters = n[0].typ
+    let L = if parameters != nil: parameters.len else: 0
+    for i in 1 ..< n.len:
+      let arg = n[i]
+      if i < L and parameters[i].kind == tySink:
+        if arg.kind in nkCallKinds:
+          # recurse but skip the call expression in order to prevent
+          # destructor injections: Rule 5.1 is different from rule 5.4!
+          let a = copyNode(arg)
+          recurse(arg, a)
+          n.sons[i] = a
+        elif arg.kind in {nkObjConstr, nkCharLit..nkFloat128Lit}:
+          discard "object construction to sink parameter: nothing to do"
+        elif arg.kind == nkSym and isHarmlessVar(arg.sym, c):
+          # if x is a variable and it its last read we eliminate its
+          # destructor invokation, but don't. We need to reset its memory
+          # to disable its destructor which we have not elided:
+          n.sons[i] = destructiveMoveVar(arg, c)
+        elif arg.kind == nkSym and isSinkParam(arg.sym):
+          # mark the sink parameter as used:
+          n.sons[i] = destructiveMoveSink(arg, c)
+        else:
+          # an object that is not temporary but passed to a 'sink' parameter
+          # results in a copy.
+          n.sons[i] = passCopyToSink(arg, c)
+      else:
+        n.sons[i] = p(arg, c)
+
+    if n.typ != nil and hasDestructor(n.typ):
+      discard "produce temp creation"
+      result = newNodeIT(nkStmtListExpr, n.info, n.typ)
+      let tmp = getTemp(c, n.typ, n.info)
+      var sinkExpr = genSink(c, n.typ, tmp, n)
+      sinkExpr.add n
+      result.add sinkExpr
+      result.add tmp
+      c.destroys.add genDestroy(c, n.typ, tmp)
+    else:
+      result = n
+  of nkAsgn, nkFastAsgn:
+    if hasDestructor(n[0].typ):
+      result = moveOrCopy(n[0], n[1], c)
+    else:
+      result = copyNode(n)
+      recurse(n, result)
+  of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef,
+      nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef:
+    result = n
+  else:
+    result = copyNode(n)
+    recurse(n, result)
+
+proc injectDestructorCalls*(g: ModuleGraph; owner: PSym; n: PNode): PNode =
+  when defined(nimDebugDestroys):
+    echo "injecting into ", n
+  var c: Con
+  c.owner = owner
+  c.tmp = newSym(skTemp, getIdent(g.cache, ":d"), owner, n.info)
+  c.tmpObj = createObj(g, owner, n.info)
+  c.tmp.typ = c.tmpObj
+  c.destroys = newNodeI(nkStmtList, n.info)
+  c.topLevelVars = newNodeI(nkVarSection, n.info)
+  c.toDropBit = initTable[int, PSym]()
+  c.graph = g
+  c.emptyNode = newNodeI(nkEmpty, n.info)
+  let cfg = constructCfg(owner, n)
+  shallowCopy(c.g, cfg)
+  c.jumpTargets = initIntSet()
+  for i in 0..<c.g.len:
+    if c.g[i].kind in {goto, fork}:
+      c.jumpTargets.incl(i+c.g[i].dest)
+  if owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter}:
+    let params = owner.typ.n
+    for i in 1 ..< params.len:
+      let param = params[i].sym
+      if param.typ.kind == tySink: registerDropBit(c, param)
+  let body = p(n, c)
+  if c.tmp.typ.n.len > 0:
+    c.addTopVar(newSymNode c.tmp)
+  result = newNodeI(nkStmtList, n.info)
+  if c.topLevelVars.len > 0:
+    result.add c.topLevelVars
+  if c.destroys.len > 0:
+    result.add newTryFinally(body, c.destroys)
+  else:
+    result.add body
+
+  when defined(nimDebugDestroys):
+    if owner.name.s == "main" or true:
+      echo "------------------------------------"
+      echo owner.name.s, " transformed to: "
+      echo result

compiler/dfa.nim

@@ -7,7 +7,9 @@
 #    distribution, for details about the copyright.
 #
 
-## Data flow analysis for Nim.
+## Data flow analysis for Nim. For now the task is to prove that every
+## usage of a local variable 'v' is covered by an initialization to 'v'
+## first.
 ## We transform the AST into a linear list of instructions first to
 ## make this easier to handle: There are only 2 different branching
 ## instructions: 'goto X' is an unconditional goto, 'fork X'
@@ -15,35 +17,26 @@
 ## taken). Exhaustive case statements are translated
 ## so that the last branch is transformed into an 'else' branch.
 ## ``return`` and ``break`` are all covered by 'goto'.
-##
-## Control flow through exception handling:
-## Contrary to popular belief, exception handling doesn't cause
-## many problems for this DFA representation, ``raise`` is a statement
-## that ``goes to`` the outer ``finally`` or ``except`` if there is one,
-## otherwise it is the same as ``return``. Every call is treated as
-## a call that can potentially ``raise``. However, without a surrounding
-## ``try`` we don't emit these ``fork ReturnLabel`` instructions in order
-## to speed up the dataflow analysis passes.
-##
+## The case to detect is ``use v`` that is not dominated by
+## a ``def v``.
 ## The data structures and algorithms used here are inspired by
 ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
 ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
 
-import ast, types, intsets, lineinfos, renderer
-
-from patterns import sameTrees
+import ast, astalgo, types, intsets, tables, msgs, options, lineinfos
 
 type
   InstrKind* = enum
-    goto, fork, join, def, use
+    goto, fork, def, use,
+    useWithinCall # this strange special case is used to get more
+                  # move optimizations out of regular code
+                  # XXX This is still overly pessimistic in
+                  # call(let x = foo; bar(x))
   Instr* = object
     n*: PNode
     case kind*: InstrKind
-    of def, use: sym*: PSym # 'sym' can also be 'nil' and
-                            # then 'n' contains the def/use location.
-                            # This is used so that we can track object
-                            # and tuple field accesses precisely.
-    of goto, fork, join: dest*: int
+    of def, use, useWithinCall: sym*: PSym
+    of goto, fork: dest*: int
 
   ControlFlowGraph* = seq[Instr]
 
@@ -52,13 +45,13 @@ type
     label: PSym
     fixups: seq[TPosition]
 
+  ValueKind = enum
+    undef, value, valueOrUndef
+
   Con = object
     code: ControlFlowGraph
-    inCall, inTryStmt: int
+    inCall: int
     blocks: seq[TBlock]
-    tryStmtFixups: seq[TPosition]
-    forks: seq[TPosition]
-    owner: PSym
 
 proc debugInfo(info: TLineInfo): string =
   result = $info.line #info.toFilename & ":" & $info.line
@@ -69,26 +62,26 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
   var jumpTargets = initIntSet()
   let last = if last < 0: c.len-1 else: min(last, c.len-1)
   for i in start..last:
-    if c[i].kind in {goto, fork, join}:
+    if c[i].kind in {goto, fork}:
       jumpTargets.incl(i+c[i].dest)
   var i = start
   while i <= last:
     if i in jumpTargets: result.add("L" & $i & ":\n")
     result.add "\t"
-    result.add ($i & " " & $c[i].kind)
+    result.add $c[i].kind
     result.add "\t"
     case c[i].kind
-    of def, use:
-      result.add renderTree(c[i].n)
-    of goto, fork, join:
+    of def, use, useWithinCall:
+      result.add c[i].sym.name.s
+    of goto, fork:
       result.add "L"
-      result.addInt c[i].dest+i
+      result.add c[i].dest+i
     result.add("\t#")
     result.add(debugInfo(c[i].n.info))
     result.add("\n")
     inc i
   if i in jumpTargets: result.add("L" & $i & ": End\n")
-  # consider calling `asciitables.alignTable`
+
 
 proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
   ## echos the ControlFlowGraph for debugging purposes.
@@ -100,179 +93,24 @@ proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
 proc forkI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: fork, dest: 0)
-  c.forks.add result
 
 proc gotoI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: goto, dest: 0)
 
-#[
-
-Design of join
-==============
-
-block:
-  if cond: break
-  def(x)
-
-use(x)
-
-Generates:
-
-L0: fork lab1
-  join L0  # patched.
-  goto Louter
-lab1:
-  def x
-  join L0
-Louter:
-  use x
-
-
-block outer:
-  while a:
-    while b:
-      if foo:
-        if bar:
-          break outer # --> we need to 'join' every pushed 'fork' here
-
-
-This works and then our abstract interpretation needs to deal with 'fork'
-differently. It really causes a split in execution. Two threads are
-"spawned" and both need to reach the 'join L' instruction. Afterwards
-the abstract interpretations are joined and execution resumes single
-threaded.
-
-
-Abstract Interpretation
------------------------
-
-proc interpret(pc, state, comesFrom): state =
-  result = state
-  # we need an explicit 'create' instruction (an explicit heap), in order
-  # to deal with 'var x = create(); var y = x; var z = y; destroy(z)'
-  while true:
-    case pc
-    of fork:
-      let a = interpret(pc+1, result, pc)
-      let b = interpret(forkTarget, result, pc)
-      result = a ++ b # ++ is a union operation
-      inc pc
-    of join:
-      if joinTarget == comesFrom: return result
-      else: inc pc
-    of use X:
-      if not result.contains(x):
-        error "variable not initialized " & x
-      inc pc
-    of def X:
-      if not result.contains(x):
-        result.incl X
-      else:
-        error "overwrite of variable causes memory leak " & x
-      inc pc
-    of destroy X:
-      result.excl X
-
-This is correct but still can lead to false positives:
-
-proc p(cond: bool) =
-  if cond:
-    new(x)
-  otherThings()
-  if cond:
-    destroy x
-
-Is not a leak. We should find a way to model *data* flow, not just
-control flow. One solution is to rewrite the 'if' without a fork
-instruction. The unstructured aspect can now be easily dealt with
-the 'goto' and 'join' instructions.
-
-proc p(cond: bool) =
-  L0: fork Lend
-    new(x)
-    # do not 'join' here!
-
-  Lend:
-    otherThings()
-    join L0  # SKIP THIS FOR new(x) SOMEHOW
-  destroy x
-  join L0 # but here.
-
-
-
-But if we follow 'goto Louter' we will never come to the join point.
-We restore the bindings after popping pc from the stack then there
-"no" problem?!
-
-
-while cond:
-  prelude()
-  if not condB: break
-  postlude()
-
---->
-var setFlag = true
-while cond and not setFlag:
-  prelude()
-  if not condB:
-    setFlag = true   # BUT: Dependency
-  if not setFlag:    # HERE
-    postlude()
-
---->
-var setFlag = true
-while cond and not setFlag:
-  prelude()
-  if not condB:
-    postlude()
-    setFlag = true
-
-
--------------------------------------------------
-
-while cond:
-  prelude()
-  if more:
-    if not condB: break
-    stuffHere()
-  postlude()
-
--->
-var setFlag = true
-while cond and not setFlag:
-  prelude()
-  if more:
-    if not condB:
-      setFlag = false
-    else:
-      stuffHere()
-      postlude()
-  else:
-    postlude()
-
-This is getting complicated. Instead we keep the whole 'join' idea but
-duplicate the 'join' instructions on breaks and return exits!
-
-]#
-
-proc joinI(c: var Con; fromFork: TPosition; n: PNode) =
-  let dist = fromFork.int - c.code.len
-  c.code.add Instr(n: n, kind: join, dest: dist)
-
 proc genLabel(c: Con): TPosition =
   result = TPosition(c.code.len)
 
 proc jmpBack(c: var Con, n: PNode, p = TPosition(0)) =
   let dist = p.int - c.code.len
-  doAssert(low(int) div 2 + 1 < dist and dist < high(int) div 2)
+  doAssert(-0x7fff < dist and dist < 0x7fff)
   c.code.add Instr(n: n, kind: goto, dest: dist)
 
 proc patch(c: var Con, p: TPosition) =
   # patch with current index
   let p = p.int
   let diff = c.code.len - p
-  doAssert(low(int) div 2 + 1 < diff and diff < high(int) div 2)
+  doAssert(-0x7fff < diff and diff < 0x7fff)
   c.code[p].dest = diff
 
 proc popBlock(c: var Con; oldLen: int) =
@@ -292,411 +130,134 @@ proc isTrue(n: PNode): bool =
 
 proc gen(c: var Con; n: PNode) # {.noSideEffect.}
 
-when true:
-  proc genWhile(c: var Con; n: PNode) =
-    # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
-    # through the loop. We need to prove this is correct for our
-    # purposes. But Herb Sutter claims it is. (Proof by authority.)
-    #[
-    while cond:
-      body
-
-    Becomes:
-
-    if cond:
-      body
-      if cond:
-        body
-        if cond:
-          body
-
-    We still need to ensure 'break' resolves properly, so an AST to AST
-    translation is impossible.
-
-    So the code to generate is:
-
-      cond
-      fork L4  # F1
-      body
-      cond
-      fork L5  # F2
-      body
-      cond
-      fork L6  # F3
-      body
-    L6:
-      join F3
-    L5:
-      join F2
-    L4:
-      join F1
-    ]#
+proc genWhile(c: var Con; n: PNode) =
+  # L1:
+  #   cond, tmp
+  #   fork tmp, L2
+  #   body
+  #   jmp L1
+  # L2:
+  let L1 = c.genLabel
+  withBlock(nil):
     if isTrue(n.sons[0]):
-      # 'while true' is an idiom in Nim and so we produce
-      # better code for it:
-      for i in 0..2:
-        withBlock(nil):
-          c.gen(n.sons[1])
+      c.gen(n.sons[1])
+      c.jmpBack(n, L1)
     else:
-      let oldForksLen = c.forks.len
-      var endings: array[3, TPosition]
-      for i in 0..2:
-        withBlock(nil):
-          c.gen(n.sons[0])
-          endings[i] = c.forkI(n)
-          c.gen(n.sons[1])
-      for i in countdown(endings.high, 0):
-        let endPos = endings[i]
-        c.patch(endPos)
-        c.joinI(c.forks.pop(), n)
-      doAssert(c.forks.len == oldForksLen)
-
-else:
-
-  proc genWhile(c: var Con; n: PNode) =
-    # lab1:
-    #   cond, tmp
-    #   fork tmp, lab2
-    #   body
-    #   jmp lab1
-    # lab2:
-    let oldForksLen = c.forks.len
-    let lab1 = c.genLabel
-    withBlock(nil):
-      if isTrue(n.sons[0]):
-        c.gen(n.sons[1])
-        c.jmpBack(n, lab1)
-      else:
-        c.gen(n.sons[0])
-        let lab2 = c.forkI(n)
-        c.gen(n.sons[1])
-        c.jmpBack(n, lab1)
-        c.patch(lab2)
-    setLen(c.forks, oldForksLen)
+      c.gen(n.sons[0])
+      let L2 = c.forkI(n)
+      c.gen(n.sons[1])
+      c.jmpBack(n, L1)
+      c.patch(L2)
 
 proc genBlock(c: var Con; n: PNode) =
   withBlock(n.sons[0].sym):
     c.gen(n.sons[1])
 
-proc genJoins(c: var Con; n: PNode) =
-  for i in countdown(c.forks.high, 0): joinI(c, c.forks[i], n)
-
 proc genBreak(c: var Con; n: PNode) =
-  genJoins(c, n)
-  let lab1 = c.gotoI(n)
+  let L1 = c.gotoI(n)
   if n.sons[0].kind == nkSym:
     #echo cast[int](n.sons[0].sym)
     for i in countdown(c.blocks.len-1, 0):
       if c.blocks[i].label == n.sons[0].sym:
-        c.blocks[i].fixups.add lab1
+        c.blocks[i].fixups.add L1
         return
     #globalError(n.info, "VM problem: cannot find 'break' target")
   else:
-    c.blocks[c.blocks.high].fixups.add lab1
-
-template forkT(n, body) =
-  let oldLen = c.forks.len
-  let lab1 = c.forkI(n)
-  body
-  c.patch(lab1)
-  c.joinI(lab1, n)
-  setLen(c.forks, oldLen)
+    c.blocks[c.blocks.high].fixups.add L1
 
 proc genIf(c: var Con, n: PNode) =
-  #[
-
-  if cond:
-    A
-  elif condB:
-    B
-  elif condC:
-    C
-  else:
-    D
-
-  cond
-  fork lab1
-  A
-  goto Lend
-  lab1:
-    condB
-    fork lab2
-    B
-    goto Lend2
-  lab2:
-    condC
-    fork L3
-    C
-    goto Lend3
-  L3:
-    D
-    goto Lend3 # not eliminated to simplify the join generation
-  Lend3:
-    join F3
-  Lend2:
-    join F2
-  Lend:
-    join F1
-
-  ]#
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
-  for i in 0 ..< len(n):
+  for i in countup(0, len(n) - 1):
     var it = n.sons[i]
     c.gen(it.sons[0])
     if it.len == 2:
-      let elsePos = forkI(c, it[1])
+      let elsePos = c.forkI(it.sons[1])
       c.gen(it.sons[1])
-      endings.add(c.gotoI(it.sons[1]))
+      if i < sonsLen(n)-1:
+        endings.add(c.gotoI(it.sons[1]))
       c.patch(elsePos)
-  for i in countdown(endings.high, 0):
-    let endPos = endings[i]
-    c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
+  for endPos in endings: c.patch(endPos)
 
 proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, a
-  #   fork lab1
+  #   fork L1
   #   asgn dest, b
-  # lab1:
-  #   join F1
+  # L1:
   c.gen(n.sons[1])
-  forkT(n):
-    c.gen(n.sons[2])
+  let L1 = c.forkI(n)
+  c.gen(n.sons[2])
+  c.patch(L1)
 
 proc genCase(c: var Con; n: PNode) =
-  #  if (!expr1) goto lab1;
+  #  if (!expr1) goto L1;
   #    thenPart
   #    goto LEnd
-  #  lab1:
-  #  if (!expr2) goto lab2;
+  #  L1:
+  #  if (!expr2) goto L2;
   #    thenPart2
   #    goto LEnd
-  #  lab2:
+  #  L2:
   #    elsePart
   #  Lend:
-  let isExhaustive = skipTypes(n.sons[0].typ,
-    abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
-
   var endings: seq[TPosition] = @[]
-  let oldLen = c.forks.len
   c.gen(n.sons[0])
   for i in 1 ..< n.len:
     let it = n.sons[i]
     if it.len == 1:
       c.gen(it.sons[0])
-    elif i == n.len-1 and isExhaustive:
-      # treat the last branch as 'else' if this is an exhaustive case statement.
-      c.gen(it.lastSon)
     else:
       let elsePos = c.forkI(it.lastSon)
       c.gen(it.lastSon)
-      endings.add(c.gotoI(it.lastSon))
+      if i < sonsLen(n)-1:
+        endings.add(c.gotoI(it.lastSon))
       c.patch(elsePos)
-  for i in countdown(endings.high, 0):
-    let endPos = endings[i]
-    c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
+  for endPos in endings: c.patch(endPos)
 
 proc genTry(c: var Con; n: PNode) =
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
-  inc c.inTryStmt
-  let oldFixups = c.tryStmtFixups.len
-
-  #let elsePos = c.forkI(n)
+  let elsePos = c.forkI(n)
   c.gen(n.sons[0])
-  dec c.inTryStmt
-  for i in oldFixups..c.tryStmtFixups.high:
-    let f = c.tryStmtFixups[i]
-    c.patch(f)
-    # we also need to produce join instructions
-    # for the 'fork' that might preceed the goto instruction
-    if f.int-1 >= 0 and c.code[f.int-1].kind == fork:
-      c.joinI(TPosition(f.int-1), n)
-
-  setLen(c.tryStmtFixups, oldFixups)
-
-  #c.patch(elsePos)
+  c.patch(elsePos)
   for i in 1 ..< n.len:
     let it = n.sons[i]
     if it.kind != nkFinally:
       var blen = len(it)
       let endExcept = c.forkI(it)
       c.gen(it.lastSon)
-      endings.add(c.gotoI(it))
+      if i < sonsLen(n)-1:
+        endings.add(c.gotoI(it))
       c.patch(endExcept)
-  for i in countdown(endings.high, 0):
-    let endPos = endings[i]
-    c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-
-  # join the 'elsePos' forkI instruction:
-  #c.joinI(c.forks.pop(), n)
-
+  for endPos in endings: c.patch(endPos)
   let fin = lastSon(n)
   if fin.kind == nkFinally:
     c.gen(fin.sons[0])
-  doAssert(c.forks.len == oldLen)
-
-template genNoReturn(c: var Con; n: PNode) =
-  # leave the graph
-  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 proc genRaise(c: var Con; n: PNode) =
-  genJoins(c, n)
   gen(c, n.sons[0])
-  if c.inTryStmt > 0:
-    c.tryStmtFixups.add c.gotoI(n)
-  else:
-    genNoReturn(c, n)
-
-proc genImplicitReturn(c: var Con) =
-  if c.owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter} and resultPos < c.owner.ast.len:
-    gen(c, c.owner.ast.sons[resultPos])
+  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 proc genReturn(c: var Con; n: PNode) =
-  genJoins(c, n)
-  if n.sons[0].kind != nkEmpty:
-    gen(c, n.sons[0])
-  else:
-    genImplicitReturn(c)
-  genNoReturn(c, n)
+  if n.sons[0].kind != nkEmpty: gen(c, n.sons[0])
+  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 const
-  InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp}
-  PathKinds0 = {nkDotExpr, nkCheckedFieldExpr,
-                nkBracketExpr, nkDerefExpr, nkHiddenDeref,
-                nkAddr, nkHiddenAddr,
-                nkObjDownConv, nkObjUpConv}
-  PathKinds1 = {nkHiddenStdConv, nkHiddenSubConv}
+  InterestingSyms = {skVar, skResult, skLet}
 
-proc getRoot(n: PNode): PNode =
-  result = n
-  while true:
-    case result.kind
-    of PathKinds0:
-      result = result[0]
-    of PathKinds1:
-      result = result[1]
-    else: break
-
-proc skipConvDfa*(n: PNode): PNode =
-  result = n
-  while true:
-    case result.kind
-    of nkObjDownConv, nkObjUpConv:
-      result = result[0]
-    of PathKinds1:
-      result = result[1]
-    else: break
-
-proc genUse(c: var Con; orig: PNode) =
-  let n = dfa.getRoot(orig)
+proc genUse(c: var Con; n: PNode) =
+  var n = n
+  while n.kind in {nkDotExpr, nkCheckedFieldExpr,
+                   nkBracketExpr, nkDerefExpr, nkHiddenDeref,
+                   nkAddr, nkHiddenAddr}:
+    n = n[0]
   if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: orig, kind: use, sym: if orig != n: nil else: n.sym)
-
-proc aliases(obj, field: PNode): bool =
-  var n = field
-  var obj = obj
-  while obj.kind in {nkHiddenSubConv, nkHiddenStdConv, nkObjDownConv, nkObjUpConv}:
-    obj = obj[0]
-  while true:
-    if sameTrees(obj, n): return true
-    case n.kind
-    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-       nkObjDownConv, nkObjUpConv, nkHiddenDeref, nkDerefExpr:
-      n = n[0]
-    of nkBracketExpr:
-      let x = n[0]
-      if x.typ != nil and x.typ.skipTypes(abstractInst).kind == tyTuple:
-        n = x
-      else:
-        break
+    if c.inCall > 0:
+      c.code.add Instr(n: n, kind: useWithinCall, sym: n.sym)
     else:
-      break
-  return false
-
-proc useInstrTargets*(ins: Instr; loc: PNode): bool =
-  assert ins.kind == use
-  if ins.sym != nil and loc.kind == nkSym:
-    result = ins.sym == loc.sym
-  else:
-    result = ins.n == loc or sameTrees(ins.n, loc)
-  if not result:
-    # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
-    # def x.f; question: does it affect the full 'x'? No.
-    # def x; question: does it affect the 'x.f'? Yes.
-    # use x.f;  question: does it affect the full 'x'? No.
-    # use x; question does it affect 'x.f'? Yes.
-    result = aliases(ins.n, loc) or aliases(loc, ins.n)
-
-proc defInstrTargets*(ins: Instr; loc: PNode): bool =
-  assert ins.kind == def
-  if ins.sym != nil and loc.kind == nkSym:
-    result = ins.sym == loc.sym
-  else:
-    result = ins.n == loc or sameTrees(ins.n, loc)
-  if not result:
-    # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
-    # def x.f; question: does it affect the full 'x'? No.
-    # def x; question: does it affect the 'x.f'? Yes.
-    # use x.f;  question: does it affect the full 'x'? No.
-    # use x; question does it affect 'x.f'? Yes.
-    result = aliases(ins.n, loc)
-
-proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
-  var n = orig
-  while true:
-    case n.kind
-    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-       nkObjDownConv, nkObjUpConv:
-      n = n[0]
-    of nkHiddenDeref, nkDerefExpr:
-      # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
-      # pointer indirection.
-      n = n[0]
-      return n.kind == nkSym and n.sym.owner == owner and (isSinkParam(n.sym) or
-          n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
-    of nkBracketExpr:
-      let x = n[0]
-      if x.typ != nil and x.typ.skipTypes(abstractInst).kind == tyTuple:
-        n = x
-      else:
-        break
-    else:
-      break
-  # XXX Allow closure deref operations here if we know
-  # the owner controlled the closure allocation?
-  result = n.kind == nkSym and n.sym.owner == owner and
-    owner.kind != skModule and
-    (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar)
-  # Note: There is a different move analyzer possible that checks for
-  # consume(param.key); param.key = newValue  for all paths. Then code like
-  #
-  #   let splited = split(move self.root, x)
-  #   self.root = merge(splited.lower, splited.greater)
-  #
-  # could be written without the ``move self.root``. However, this would be
-  # wrong! Then the write barrier for the ``self.root`` assignment would
-  # free the old data and all is lost! Lesson: Don't be too smart, trust the
-  # lower level C++ optimizer to specialize this code.
+      c.code.add Instr(n: n, kind: use, sym: n.sym)
 
 proc genDef(c: var Con; n: PNode) =
   if n.kind == nkSym and n.sym.kind in InterestingSyms:
     c.code.add Instr(n: n, kind: def, sym: n.sym)
-  elif isAnalysableFieldAccess(n, c.owner):
-    c.code.add Instr(n: n, kind: def, sym: nil)
-
-proc canRaise(fn: PNode): bool =
-  const magicsThatCanRaise = {
-    mNone, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst}
-  if fn.kind == nkSym and fn.sym.magic notin magicsThatCanRaise:
-    result = false
-  else:
-    result = true
 
 proc genCall(c: var Con; n: PNode) =
   gen(c, n[0])
@@ -705,23 +266,8 @@ proc genCall(c: var Con; n: PNode) =
   inc c.inCall
   for i in 1..<n.len:
     gen(c, n[i])
-    when false:
-      if t != nil and i < t.len and t.sons[i].kind == tyVar:
-        # This is wrong! Pass by var is a 'might def', not a 'must def'
-        # like the other defs we emit. This is not good enough for a move
-        # optimizer.
-        genDef(c, n[i])
-  # every call can potentially raise:
-  if c.inTryStmt > 0 and canRaise(n[0]):
-    # we generate the instruction sequence:
-    # fork lab1
-    # goto exceptionHandler (except or finally)
-    # lab1:
-    # join F1
-    let endGoto = c.forkI(n)
-    c.tryStmtFixups.add c.gotoI(n)
-    c.patch(endGoto)
-    c.joinI(c.forks.pop(), n)
+    if t != nil and i < t.len and t.sons[i].kind == tyVar:
+      genDef(c, n[i])
   dec c.inCall
 
 proc genMagic(c: var Con; n: PNode; m: TMagic) =
@@ -730,14 +276,16 @@ proc genMagic(c: var Con; n: PNode; m: TMagic) =
   of mNew, mNewFinalize:
     genDef(c, n[1])
     for i in 2..<n.len: gen(c, n[i])
+  of mExit:
+    genCall(c, n)
+    c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
   else:
     genCall(c, n)
 
 proc genVarSection(c: var Con; n: PNode) =
   for a in n:
-    if a.kind == nkCommentStmt:
-      discard
-    elif a.kind == nkVarTuple:
+    if a.kind == nkCommentStmt: continue
+    if a.kind == nkVarTuple:
       gen(c, a.lastSon)
       for i in 0 .. a.len-3: genDef(c, a[i])
     else:
@@ -755,19 +303,15 @@ proc gen(c: var Con; n: PNode) =
         genMagic(c, n, s.magic)
       else:
         genCall(c, n)
-      if sfNoReturn in n.sons[0].sym.flags:
-        genNoReturn(c, n)
     else:
       genCall(c, n)
   of nkCharLit..nkNilLit: discard
   of nkAsgn, nkFastAsgn:
     gen(c, n[1])
-    # watch out: 'obj[i].f2 = value' sets 'f2' but
-    # "uses" 'i'. But we are only talking about builtin array indexing so
-    # it doesn't matter and 'x = 34' is NOT a usage of 'x'.
     genDef(c, n[0])
-  of PathKinds0 - {nkHiddenStdConv, nkHiddenSubConv, nkObjDownConv, nkObjUpConv}:
-    genUse(c, n)
+  of nkDotExpr, nkCheckedFieldExpr, nkBracketExpr,
+     nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr:
+    gen(c, n[0])
   of nkIfStmt, nkIfExpr: genIf(c, n)
   of nkWhenStmt:
     # This is "when nimvm" node. Chose the first branch.
@@ -778,23 +322,125 @@ proc gen(c: var Con; n: PNode) =
   of nkReturnStmt: genReturn(c, n)
   of nkRaiseStmt: genRaise(c, n)
   of nkBreakStmt: genBreak(c, n)
-  of nkTryStmt, nkHiddenTryStmt: genTry(c, n)
+  of nkTryStmt: genTry(c, n)
   of nkStmtList, nkStmtListExpr, nkChckRangeF, nkChckRange64, nkChckRange,
-     nkBracket, nkCurly, nkPar, nkTupleConstr, nkClosure, nkObjConstr, nkYieldStmt:
+     nkBracket, nkCurly, nkPar, nkTupleConstr, nkClosure, nkObjConstr:
     for x in n: gen(c, x)
   of nkPragmaBlock: gen(c, n.lastSon)
-  of nkDiscardStmt, nkObjDownConv, nkObjUpConv: gen(c, n.sons[0])
-  of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast, nkHiddenSubConv, nkHiddenStdConv:
+  of nkDiscardStmt: gen(c, n.sons[0])
+  of nkHiddenStdConv, nkHiddenSubConv, nkConv, nkExprColonExpr, nkExprEqExpr,
+     nkCast:
     gen(c, n.sons[1])
-  of nkStringToCString, nkCStringToString: gen(c, n.sons[0])
+  of nkObjDownConv, nkStringToCString, nkCStringToString: gen(c, n.sons[0])
   of nkVarSection, nkLetSection: genVarSection(c, n)
-  of nkDefer:
-    doAssert false, "dfa construction pass requires the elimination of 'defer'"
   else: discard
 
+proc dfa(code: seq[Instr]; conf: ConfigRef) =
+  var u = newSeq[IntSet](code.len) # usages
+  var d = newSeq[IntSet](code.len) # defs
+  var c = newSeq[IntSet](code.len) # consumed
+  var backrefs = initTable[int, int]()
+  for i in 0..<code.len:
+    u[i] = initIntSet()
+    d[i] = initIntSet()
+    c[i] = initIntSet()
+    case code[i].kind
+    of use, useWithinCall: u[i].incl(code[i].sym.id)
+    of def: d[i].incl(code[i].sym.id)
+    of fork, goto:
+      let d = i+code[i].dest
+      backrefs.add(d, i)
+
+  var w = @[0]
+  var maxIters = 50
+  var someChange = true
+  var takenGotos = initIntSet()
+  var consuming = -1
+  while w.len > 0 and maxIters > 0: # and someChange:
+    dec maxIters
+    var pc = w.pop() # w[^1]
+    var prevPc = -1
+    # this simulates a single linear control flow execution:
+    while pc < code.len:
+      if prevPc >= 0:
+        someChange = false
+        # merge step and test for changes (we compute the fixpoints here):
+        # 'u' needs to be the union of prevPc, pc
+        # 'd' needs to be the intersection of 'pc'
+        for id in u[prevPc]:
+          if not u[pc].containsOrIncl(id):
+            someChange = true
+        # in (a; b) if ``a`` sets ``v`` so does ``b``. The intersection
+        # is only interesting on merge points:
+        for id in d[prevPc]:
+          if not d[pc].containsOrIncl(id):
+            someChange = true
+        # if this is a merge point, we take the intersection of the 'd' sets:
+        if backrefs.hasKey(pc):
+          var intersect = initIntSet()
+          assign(intersect, d[pc])
+          var first = true
+          for prevPc in backrefs.allValues(pc):
+            for def in d[pc]:
+              if def notin d[prevPc]:
+                excl(intersect, def)
+                someChange = true
+                when defined(debugDfa):
+                  echo "Excluding ", pc, " prev ", prevPc
+          assign d[pc], intersect
+      if consuming >= 0:
+        if not c[pc].containsOrIncl(consuming):
+          someChange = true
+        consuming = -1
+
+      # our interpretation ![I!]:
+      prevPc = pc
+      case code[pc].kind
+      of goto:
+        # we must leave endless loops eventually:
+        if not takenGotos.containsOrIncl(pc) or someChange:
+          pc = pc + code[pc].dest
+        else:
+          inc pc
+      of fork:
+        # we follow the next instruction but push the dest onto our "work" stack:
+        #if someChange:
+        w.add pc + code[pc].dest
+        inc pc
+      of use, useWithinCall:
+        #if not d[prevPc].missingOrExcl():
+        # someChange = true
+        consuming = code[pc].sym.id
+        inc pc
+      of def:
+        if not d[pc].containsOrIncl(code[pc].sym.id):
+          someChange = true
+        inc pc
+
+  when defined(useDfa) and defined(debugDfa):
+    for i in 0..<code.len:
+      echo "PC ", i, ": defs: ", d[i], "; uses ", u[i], "; consumes ", c[i]
+
+  # now check the condition we're interested in:
+  for i in 0..<code.len:
+    case code[i].kind
+    of use, useWithinCall:
+      let s = code[i].sym
+      if s.id notin d[i]:
+        localError(conf, code[i].n.info, "usage of uninitialized variable: " & s.name.s)
+      if s.id in c[i]:
+        localError(conf, code[i].n.info, "usage of an already consumed variable: " & s.name.s)
+
+    else: discard
+
+proc dataflowAnalysis*(s: PSym; body: PNode; conf: ConfigRef) =
+  var c = Con(code: @[], blocks: @[])
+  gen(c, body)
+  when defined(useDfa) and defined(debugDfa): echoCfg(c.code)
+  dfa(c.code, conf)
+
 proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
   ## constructs a control flow graph for ``body``.
-  var c = Con(code: @[], blocks: @[], owner: s)
+  var c = Con(code: @[], blocks: @[])
   gen(c, body)
-  genImplicitReturn(c)
   shallowCopy(result, c.code)

compiler/docgen.nim

@@ -15,18 +15,17 @@ import
   ast, strutils, strtabs, options, msgs, os, ropes, idents,
   wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast,
   packages/docutils/rst, packages/docutils/rstgen,
-  json, xmltree, cgi, trees, types,
-  typesrenderer, astalgo, lineinfos, intsets,
-  pathutils, trees
+  packages/docutils/highlite, sempass2, json, xmltree, cgi,
+  typesrenderer, astalgo, modulepaths, lineinfos, sequtils, intsets,
+  pathutils
 
 const
-  exportSection = skField
+  exportSection = skTemp
 
 type
   TSections = array[TSymKind, Rope]
   TDocumentor = object of rstgen.RstGenerator
-    modDesc: Rope       # module description
-    modDeprecationMsg: Rope
+    modDesc: Rope           # module description
     toc, section: TSections
     indexValFilename: string
     analytics: string  # Google Analytics javascript, "" if doesn't exist
@@ -68,7 +67,7 @@ proc attachToType(d: PDoc; p: PSym): PSym =
 
 template declareClosures =
   proc compilerMsgHandler(filename: string, line, col: int,
-                          msgKind: rst.MsgKind, arg: string) {.procvar, gcsafe.} =
+                          msgKind: rst.MsgKind, arg: string) {.procvar.} =
     # translate msg kind:
     var k: TMsgKind
     case msgKind
@@ -82,10 +81,9 @@ template declareClosures =
     of mwUnknownSubstitution: k = warnUnknownSubstitutionX
     of mwUnsupportedLanguage: k = warnLanguageXNotSupported
     of mwUnsupportedField: k = warnFieldXNotSupported
-    {.gcsafe.}:
-      globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg)
+    globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg)
 
-  proc docgenFindFile(s: string): string {.procvar, gcsafe.} =
+  proc docgenFindFile(s: string): string {.procvar.} =
     result = options.findFile(conf, s).string
     if result.len == 0:
       result = getCurrentDir() / s
@@ -102,16 +100,16 @@ proc parseRst(text, filename: string,
 proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
                  ext: string, dir: RelativeDir; guessTarget: bool): AbsoluteFile =
   if optWholeProject in conf.globalOptions:
-    let d = if conf.outDir.isEmpty: conf.projectPath / dir else: conf.outDir
+    # This is correct, for 'nim doc --project' we interpret the '--out' option as an
+    # absolute directory, not as a filename!
+    let d = if conf.outFile.isEmpty: conf.projectPath / dir else: AbsoluteDir(conf.outFile)
     createDir(d)
     result = d / changeFileExt(filename, ext)
   elif guessTarget:
-    let d = if not conf.outDir.isEmpty: conf.outDir
-            else: conf.projectPath
+    let d = if not conf.outFile.isEmpty: splitFile(conf.outFile).dir
+    else: conf.projectPath
     createDir(d)
     result = d / changeFileExt(filename, ext)
-  elif not conf.outFile.isEmpty:
-    result = absOutFile(conf)
   else:
     result = getOutFile(conf, filename, ext)
 
@@ -121,7 +119,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   result.conf = conf
   result.cache = cache
   initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex),
-                   conf.configVars, filename.string, {roSupportRawDirective, roSupportMarkdown},
+                   conf.configVars, filename.string, {roSupportRawDirective},
                    docgenFindFile, compilerMsgHandler)
 
   if conf.configVars.hasKey("doc.googleAnalytics"):
@@ -144,43 +142,20 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   result.id = 100
   result.jArray = newJArray()
   initStrTable result.types
-  result.onTestSnippet =
-    proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) =
-      var d = TDocumentor(gen)
-      var outp: AbsoluteFile
-      if filename.len == 0:
-        inc(d.id)
-        let nameOnly = splitFile(d.filename).name
-        outp = getNimcacheDir(conf) / RelativeDir(nameOnly) /
-               RelativeFile(nameOnly & "_snippet_" & $d.id & ".nim")
-      elif isAbsolute(filename):
-        outp = AbsoluteFile(filename)
-      else:
-        # Nim's convention: every path is relative to the file it was written in:
-        let nameOnly = splitFile(d.filename).name
-        outp = AbsoluteDir(nameOnly) / RelativeFile(filename)
-      # Make sure the destination directory exists
-      createDir(outp.splitFile.dir)
-      # Include the current file if we're parsing a nim file
-      let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")]
-      writeFile(outp, importStmt & content)
-      let c = if cmd.startsWith("nim "): os.getAppFilename() & cmd.substr(3)
-              else: cmd
-      let c2 = c % quoteShell(outp)
-      rawMessage(conf, hintExecuting, c2)
-      if execShellCmd(c2) != status:
-        rawMessage(conf, errGenerated, "executing of external program failed: " & c2)
+  result.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string; status: int; content: string) =
+    localError(conf, newLineInfo(conf, AbsoluteFile d.filename, -1, -1),
+               warnUser, "only 'rst2html' supports the ':test:' attribute")
   result.emitted = initIntSet()
   result.destFile = getOutFile2(conf, relativeTo(filename, conf.projectPath),
                                 outExt, RelativeDir"htmldocs", false)
   result.thisDir = result.destFile.splitFile.dir
 
-template dispA(conf: ConfigRef; dest: var Rope, xml, tex: string, args: openArray[Rope]) =
+proc dispA(conf: ConfigRef; dest: var Rope, xml, tex: string, args: openArray[Rope]) =
   if conf.cmd != cmdRst2tex: addf(dest, xml, args)
   else: addf(dest, tex, args)
 
 proc getVarIdx(varnames: openArray[string], id: string): int =
-  for i in 0 .. high(varnames):
+  for i in countup(0, high(varnames)):
     if cmpIgnoreStyle(varnames[i], id) == 0:
       return i
   result = -1
@@ -248,7 +223,7 @@ proc genComment(d: PDoc, n: PNode): string =
   result = ""
   var dummyHasToc: bool
   if n.comment.len > 0:
-    renderRstToOut(d[], parseRst(n.comment, toFullPath(d.conf, n.info),
+    renderRstToOut(d[], parseRst(n.comment, toFilename(d.conf, n.info),
                                toLinenumber(n.info), toColumn(n.info),
                                dummyHasToc, d.options, d.conf), result)
 
@@ -257,9 +232,9 @@ proc genRecCommentAux(d: PDoc, n: PNode): Rope =
   result = genComment(d, n).rope
   if result == nil:
     if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef,
-                  nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkHiddenStdConv}:
+                  nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn}:
       # notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}:
-      for i in 0 ..< len(n):
+      for i in countup(0, len(n)-1):
         result = genRecCommentAux(d, n.sons[i])
         if result != nil: return
   else:
@@ -287,7 +262,7 @@ proc getPlainDocstring(n: PNode): string =
   if startsWith(n.comment, "##"):
     result = n.comment
   if result.len < 1:
-    for i in 0 ..< safeLen(n):
+    for i in countup(0, safeLen(n)-1):
       result = getPlainDocstring(n.sons[i])
       if result.len > 0: return
 
@@ -304,17 +279,13 @@ proc externalDep(d: PDoc; module: PSym): string =
   else:
     result = extractFilename toFullPath(d.conf, FileIndex module.position)
 
-proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRenderFlags = {};
-                           procLink: Rope) =
+proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRenderFlags = {}) =
   var r: TSrcGen
   var literal = ""
   initTokRender(r, n, renderFlags)
   var kind = tkEof
-  var tokenPos = 0
-  var procTokenPos = 0
   while true:
     getNextTok(r, kind, literal)
-    inc tokenPos
     case kind
     of tkEof:
       break
@@ -322,8 +293,6 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
       dispA(d.conf, result, "<span class=\"Comment\">$1</span>", "\\spanComment{$1}",
             [rope(esc(d.target, literal))])
     of tokKeywordLow..tokKeywordHigh:
-      if kind in {tkProc, tkMethod, tkIterator, tkMacro, tkTemplate, tkFunc, tkConverter}:
-        procTokenPos = tokenPos
       dispA(d.conf, result, "<span class=\"Keyword\">$1</span>", "\\spanKeyword{$1}",
             [rope(literal)])
     of tkOpr:
@@ -343,13 +312,9 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
             "\\spanFloatNumber{$1}", [rope(esc(d.target, literal))])
     of tkSymbol:
       let s = getTokSym(r)
-      # -2 because of the whitespace in between:
-      if procTokenPos == tokenPos-2 and procLink != nil:
-        dispA(d.conf, result, "<a href=\"#$2\"><span class=\"Identifier\">$1</span></a>",
-              "\\spanIdentifier{$1}", [rope(esc(d.target, literal)), procLink])
-      elif s != nil and s.kind in {skType, skVar, skLet, skConst} and
-           sfExported in s.flags and s.owner != nil and
-           belongsToPackage(d.conf, s.owner) and d.target == outHtml:
+      if s != nil and s.kind == skType and sfExported in s.flags and
+          s.owner != nil and belongsToPackage(d.conf, s.owner) and
+          d.target == outHtml:
         let external = externalDep(d, s.owner)
         result.addf "<a href=\"$1#$2\"><span class=\"Identifier\">$3</span></a>",
           [rope changeFileExt(external, "html"), rope literal,
@@ -427,7 +392,6 @@ proc prepareExamples(d: PDoc; n: PNode) =
   let loc = d.conf.toFileLineCol(n.info)
   docComment.comment = "autogenerated by docgen from " & loc
   var runnableExamples = newTree(nkStmtList,
-      docComment,
       newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
   runnableExamples.info = n.info
   let imports = newTree(nkStmtList)
@@ -437,11 +401,16 @@ proc prepareExamples(d: PDoc; n: PNode) =
   runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon)
   testExample(d, runnableExamples)
 
+proc isRunnableExample(n: PNode): bool =
+  # Templates and generics don't perform symbol lookups.
+  result = n.kind == nkSym and n.sym.magic == mRunnableExamples or
+    n.kind == nkIdent and n.ident.s == "runnableExamples"
+
 proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope) =
   if n.info.fileIndex != orig.info.fileIndex: return
   case n.kind
   of nkCallKinds:
-    if isRunnableExamples(n[0]) and
+    if isRunnableExample(n[0]) and
         n.len >= 2 and n.lastSon.kind == nkStmtList:
       prepareExamples(d, n)
       dispA(d.conf, dest, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
@@ -459,7 +428,7 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope) =
       for b in body:
         if i > 0: dest.add "\n"
         inc i
-        nodeToHighlightedHtml(d, b, dest, {}, nil)
+        nodeToHighlightedHtml(d, b, dest, {})
       dest.add(d.config.getOrDefault"doc.listing_end" % id)
   else: discard
   for i in 0 ..< n.safeLen:
@@ -478,10 +447,7 @@ proc isVisible(d: PDoc; n: PNode): bool =
     # we cannot generate code for forwarded symbols here as we have no
     # exception tracking information here. Instead we copy over the comment
     # from the proc header.
-    if optDocInternal in d.conf.globalOptions:
-      result = {sfFromGeneric, sfForward}*n.sym.flags == {}
-    else:
-      result = {sfExported, sfFromGeneric, sfForward}*n.sym.flags == {sfExported}
+    result = {sfExported, sfFromGeneric, sfForward}*n.sym.flags == {sfExported}
     if result and containsOrIncl(d.emitted, n.sym.id):
       result = false
   elif n.kind == nkPragmaExpr:
@@ -562,19 +528,16 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string =
   ## If you modify the output of this proc, please update the anchor generation
   ## section of ``doc/docgen.txt``.
   result = baseName
-  case k
-  of skProc, skFunc: discard
-  of skMacro: result.add(".m")
-  of skMethod: result.add(".e")
-  of skIterator: result.add(".i")
-  of skTemplate: result.add(".t")
-  of skConverter: result.add(".c")
+  case k:
+  of skProc, skFunc: result.add(defaultParamSeparator)
+  of skMacro: result.add(".m" & defaultParamSeparator)
+  of skMethod: result.add(".e" & defaultParamSeparator)
+  of skIterator: result.add(".i" & defaultParamSeparator)
+  of skTemplate: result.add(".t" & defaultParamSeparator)
+  of skConverter: result.add(".c" & defaultParamSeparator)
   else: discard
   if len(n) > paramsPos and n[paramsPos].kind == nkFormalParams:
-    let params = renderParamTypes(n[paramsPos])
-    if params.len > 0:
-      result.add(defaultParamSeparator)
-      result.add(params)
+    result.add(renderParamTypes(n[paramsPos]))
 
 proc isCallable(n: PNode): bool =
   ## Returns true if `n` contains a callable node.
@@ -611,23 +574,6 @@ proc docstringSummary(rstText: string): string =
     result.delete(pos, last)
     result.add("…")
 
-proc genDeprecationMsg(d: PDoc, n: PNode): Rope =
-  ## Given a nkPragma wDeprecated node output a well-formatted section
-  if n == nil: return
-
-  case n.safeLen:
-  of 0: # Deprecated w/o any message
-    result = ropeFormatNamedVars(d.conf,
-      getConfigVar(d.conf, "doc.deprecationmsg"), ["label", "message"],
-      [~"Deprecated", nil])
-  of 2: # Deprecated w/ a message
-    if n[1].kind in {nkStrLit..nkTripleStrLit}:
-      result = ropeFormatNamedVars(d.conf,
-        getConfigVar(d.conf, "doc.deprecationmsg"), ["label", "message"],
-        [~"Deprecated:", rope(xmltree.escape(n[1].strVal))])
-  else:
-    doAssert false
-
 proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
   if not isVisible(d, nameNode): return
   let
@@ -649,9 +595,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
       break
     plainName.add(literal)
 
-  var pragmaNode: PNode = nil
-  if n.isCallable and n.sons[pragmasPos].kind != nkEmpty:
-    pragmaNode = findPragma(n.sons[pragmasPos], wDeprecated)
+  nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,
+    renderDocComments, renderSyms})
 
   inc(d.id)
   let
@@ -664,10 +609,6 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
     symbolOrId = d.newUniquePlainSymbol(complexSymbol)
     symbolOrIdRope = symbolOrId.rope
     symbolOrIdEncRope = encodeUrl(symbolOrId).rope
-    deprecationMsgRope = genDeprecationMsg(d, pragmaNode)
-
-  nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,
-    renderDocComments, renderSyms}, symbolOrIdEncRope)
 
   var seeSrcRope: Rope = nil
   let docItemSeeSrc = getConfigVar(d.conf, "doc.item.seesrc")
@@ -680,10 +621,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
         path = path[cwd.len+1 .. ^1].replace('\\', '/')
     let gitUrl = getConfigVar(d.conf, "git.url")
     if gitUrl.len > 0:
-      let defaultBranch =
-        if NimPatch mod 2 == 1: "devel"
-        else: "version-$1-$2" % [$NimMajor, $NimMinor]
-      let commit = getConfigVar(d.conf, "git.commit", defaultBranch)
+      let commit = getConfigVar(d.conf, "git.commit", "master")
       let develBranch = getConfigVar(d.conf, "git.devel", "devel")
       dispA(d.conf, seeSrcRope, "$1", "", [ropeFormatNamedVars(d.conf, docItemSeeSrc,
           ["path", "line", "url", "commit", "devel"], [rope path.string,
@@ -691,12 +629,11 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
 
   add(d.section[k], ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.item"),
     ["name", "header", "desc", "itemID", "header_plain", "itemSym",
-      "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "seeSrc", "deprecationMsg"],
+      "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "seeSrc"],
     [nameRope, result, comm, itemIDRope, plainNameRope, plainSymbolRope,
-      symbolOrIdRope, plainSymbolEncRope, symbolOrIdEncRope, seeSrcRope,
-      deprecationMsgRope]))
+      symbolOrIdRope, plainSymbolEncRope, symbolOrIdEncRope, seeSrcRope]))
 
-  let external = d.destFile.relativeTo(d.conf.outDir, '/').changeFileExt(HtmlExt).string
+  let external = AbsoluteFile(d.filename).relativeTo(d.conf.projectPath, '/').changeFileExt(HtmlExt).string
 
   var attype: Rope
   if k in routineKinds and nameNode.kind == nkSym:
@@ -788,68 +725,8 @@ proc exportSym(d: PDoc; s: PSym) =
             "$1", [rope esc(d.target, s.name.s),
             rope changeFileExt(external, "html")])
 
-proc documentNewEffect(cache: IdentCache; n: PNode): PNode =
-  let s = n.sons[namePos].sym
-  if tfReturnsNew in s.typ.flags:
-    result = newIdentNode(getIdent(cache, "new"), n.info)
-
-proc documentEffect(cache: IdentCache; n, x: PNode, effectType: TSpecialWord, idx: int): PNode =
-  let spec = effectSpec(x, effectType)
-  if isNil(spec):
-    let s = n.sons[namePos].sym
-
-    let actual = s.typ.n.sons[0]
-    if actual.len != effectListLen: return
-    let real = actual.sons[idx]
-
-    # warning: hack ahead:
-    var effects = newNodeI(nkBracket, n.info, real.len)
-    for i in 0 ..< real.len:
-      var t = typeToString(real[i].typ)
-      if t.startsWith("ref "): t = substr(t, 4)
-      effects.sons[i] = newIdentNode(getIdent(cache, t), n.info)
-      # set the type so that the following analysis doesn't screw up:
-      effects.sons[i].typ = real[i].typ
-
-    result = newNode(nkExprColonExpr, n.info, @[
-      newIdentNode(getIdent(cache, specialWords[effectType]), n.info), effects])
-
-proc documentWriteEffect(cache: IdentCache; n: PNode; flag: TSymFlag; pragmaName: string): PNode =
-  let s = n.sons[namePos].sym
-  let params = s.typ.n
-
-  var effects = newNodeI(nkBracket, n.info)
-  for i in 1 ..< params.len:
-    if params[i].kind == nkSym and flag in params[i].sym.flags:
-      effects.add params[i]
-
-  if effects.len > 0:
-    result = newNode(nkExprColonExpr, n.info, @[
-      newIdentNode(getIdent(cache, pragmaName), n.info), effects])
-
-proc documentRaises*(cache: IdentCache; n: PNode) =
-  if n.sons[namePos].kind != nkSym: return
-  let pragmas = n.sons[pragmasPos]
-  let p1 = documentEffect(cache, n, pragmas, wRaises, exceptionEffects)
-  let p2 = documentEffect(cache, n, pragmas, wTags, tagEffects)
-  let p3 = documentWriteEffect(cache, n, sfWrittenTo, "writes")
-  let p4 = documentNewEffect(cache, n)
-  let p5 = documentWriteEffect(cache, n, sfEscapes, "escapes")
-
-  if p1 != nil or p2 != nil or p3 != nil or p4 != nil or p5 != nil:
-    if pragmas.kind == nkEmpty:
-      n.sons[pragmasPos] = newNodeI(nkPragma, n.info)
-    if p1 != nil: n.sons[pragmasPos].add p1
-    if p2 != nil: n.sons[pragmasPos].add p2
-    if p3 != nil: n.sons[pragmasPos].add p3
-    if p4 != nil: n.sons[pragmasPos].add p4
-    if p5 != nil: n.sons[pragmasPos].add p5
-
 proc generateDoc*(d: PDoc, n, orig: PNode) =
   case n.kind
-  of nkPragma:
-    let pragmaNode = findPragma(n, wDeprecated)
-    add(d.modDeprecationMsg, genDeprecationMsg(d, pragmaNode))
   of nkCommentStmt: add(d.modDesc, genComment(d, n))
   of nkProcDef:
     when useEffectSystem: documentRaises(d.cache, n)
@@ -869,13 +746,13 @@ proc generateDoc*(d: PDoc, n, orig: PNode) =
     when useEffectSystem: documentRaises(d.cache, n)
     genItem(d, n, n.sons[namePos], skConverter)
   of nkTypeSection, nkVarSection, nkLetSection, nkConstSection:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       if n.sons[i].kind != nkCommentStmt:
         # order is always 'type var let const':
         genItem(d, n.sons[i], n.sons[i].sons[0],
                 succ(skType, ord(n.kind)-ord(nkTypeSection)))
   of nkStmtList:
-    for i in 0 ..< sonsLen(n): generateDoc(d, n.sons[i], orig)
+    for i in countup(0, sonsLen(n) - 1): generateDoc(d, n.sons[i], orig)
   of nkWhenStmt:
     # generate documentation for the first branch only:
     if not checkForFalse(n.sons[0].sons[0]):
@@ -923,13 +800,13 @@ proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
     when useEffectSystem: documentRaises(d.cache, n)
     d.add genJsonItem(d, n, n.sons[namePos], skConverter)
   of nkTypeSection, nkVarSection, nkLetSection, nkConstSection:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       if n.sons[i].kind != nkCommentStmt:
         # order is always 'type var let const':
         d.add genJsonItem(d, n.sons[i], n.sons[i].sons[0],
                 succ(skType, ord(n.kind)-ord(nkTypeSection)))
   of nkStmtList:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       generateJson(d, n.sons[i], includeComments)
   of nkWhenStmt:
     # generate documentation for the first branch only:
@@ -966,13 +843,13 @@ proc generateTags*(d: PDoc, n: PNode, r: var Rope) =
     when useEffectSystem: documentRaises(d.cache, n)
     r.add genTagsItem(d, n, n.sons[namePos], skConverter)
   of nkTypeSection, nkVarSection, nkLetSection, nkConstSection:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       if n.sons[i].kind != nkCommentStmt:
         # order is always 'type var let const':
         r.add genTagsItem(d, n.sons[i], n.sons[i].sons[0],
                 succ(skType, ord(n.kind)-ord(nkTypeSection)))
   of nkStmtList:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       generateTags(d, n.sons[i], r)
   of nkWhenStmt:
     # generate documentation for the first branch only:
@@ -981,9 +858,9 @@ proc generateTags*(d: PDoc, n: PNode, r: var Rope) =
   else: discard
 
 proc genSection(d: PDoc, kind: TSymKind) =
-  const sectionNames: array[skModule..skField, string] = [
-    "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Funcs",
-    "Methods", "Iterators", "Converters", "Macros", "Templates", "Exports"
+  const sectionNames: array[skTemp..skTemplate, string] = [
+    "Exports", "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Funcs",
+    "Methods", "Iterators", "Converters", "Macros", "Templates"
   ]
   if d.section[kind] == nil: return
   var title = sectionNames[kind].rope
@@ -1002,12 +879,12 @@ proc genOutFile(d: PDoc): Rope =
   var tmp = ""
   renderTocEntries(d[], j, 1, tmp)
   var toc = tmp.rope
-  for i in low(TSymKind) .. high(TSymKind):
+  for i in countup(low(TSymKind), high(TSymKind)):
     genSection(d, i)
     add(toc, d.toc[i])
   if toc != nil:
     toc = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.toc"), ["content"], [toc])
-  for i in low(TSymKind) .. high(TSymKind): add(code, d.section[i])
+  for i in countup(low(TSymKind), high(TSymKind)): add(code, d.section[i])
 
   # Extract the title. Non API modules generate an entry in the index table.
   if d.meta[metaTitle].len != 0:
@@ -1022,25 +899,26 @@ proc genOutFile(d: PDoc): Rope =
                  elif d.hasToc: "doc.body_toc"
                  else: "doc.body_no_toc"
   content = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, bodyname), ["title",
-      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg"],
+      "tableofcontents", "moduledesc", "date", "time", "content"],
       [title.rope, toc, d.modDesc, rope(getDateStr()),
-      rope(getClockStr()), code, d.modDeprecationMsg])
+      rope(getClockStr()), code])
   if optCompileOnly notin d.conf.globalOptions:
     # XXX what is this hack doing here? 'optCompileOnly' means raw output!?
     code = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.file"), ["title",
         "tableofcontents", "moduledesc", "date", "time",
-        "content", "author", "version", "analytics", "deprecationMsg"],
+        "content", "author", "version", "analytics"],
         [title.rope, toc, d.modDesc, rope(getDateStr()),
                      rope(getClockStr()), content, d.meta[metaAuthor].rope,
-                     d.meta[metaVersion].rope, d.analytics.rope, d.modDeprecationMsg])
+                     d.meta[metaVersion].rope, d.analytics.rope])
   else:
     code = content
   result = code
 
 proc generateIndex*(d: PDoc) =
   if optGenIndex in d.conf.globalOptions:
-    let dir = if not d.conf.outDir.isEmpty: d.conf.outDir
-              else: d.conf.projectPath / RelativeDir"htmldocs"
+    let dir = if d.conf.outFile.isEmpty: d.conf.projectPath / RelativeDir"htmldocs"
+              elif optWholeProject in d.conf.globalOptions: AbsoluteDir(d.conf.outFile)
+              else: AbsoluteDir(d.conf.outFile.string.splitFile.dir)
     createDir(dir)
     let dest = dir / changeFileExt(relativeTo(AbsoluteFile d.filename,
                                               d.conf.projectPath), IndexExt)
@@ -1080,13 +958,7 @@ proc writeOutputJson*(d: PDoc, useWarning = false) =
                  warnUser, "unable to open file \"" & d.destFile.string &
                  "\" for writing")
 
-proc handleDocOutputOptions*(conf: ConfigRef) =
-  if optWholeProject in conf.globalOptions:
-    # Backward compatibility with previous versions
-    conf.outDir = AbsoluteDir(conf.outDir / conf.outFile)
-
 proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
-  handleDocOutputOptions conf
   var ast = parseFile(conf.projectMainIdx, cache, conf)
   if ast == nil: return
   var d = newDocumentor(conf.projectFull, cache, conf)
@@ -1099,10 +971,31 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef;
                    filename: AbsoluteFile, outExt: string) =
   var filen = addFileExt(filename, "txt")
   var d = newDocumentor(filen, cache, conf, outExt)
+  d.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string;
+                          status: int; content: string) =
+    var outp: AbsoluteFile
+    if filename.len == 0:
+      inc(d.id)
+      let nameOnly = splitFile(d.filename).name
+      let subdir = getNimcacheDir(conf) / RelativeDir(nameOnly)
+      createDir(subdir)
+      outp = subdir / RelativeFile(nameOnly & "_snippet_" & $d.id & ".nim")
+    elif isAbsolute(filename):
+      outp = AbsoluteFile filename
+    else:
+      # Nim's convention: every path is relative to the file it was written in:
+      outp = splitFile(d.filename).dir.AbsoluteDir / RelativeFile(filename)
+    writeFile(outp, content)
+    let c = if cmd.startsWith("nim "): os.getAppFilename() & cmd.substr(3)
+            else: cmd
+    let c2 = c % quoteShell(outp)
+    rawMessage(conf, hintExecuting, c2)
+    if execShellCmd(c2) != status:
+      rawMessage(conf, errGenerated, "executing of external program failed: " & c2)
 
   d.isPureRst = true
   var rst = parseRst(readFile(filen.string), filen.string, 0, 1, d.hasToc,
-                     {roSupportRawDirective, roSupportMarkdown}, conf)
+                     {roSupportRawDirective}, conf)
   var modDesc = newStringOfCap(30_000)
   renderRstToOut(d[], rst, modDesc)
   d.modDesc = rope(modDesc)
@@ -1119,10 +1012,6 @@ proc commandJson*(cache: IdentCache, conf: ConfigRef) =
   var ast = parseFile(conf.projectMainIdx, cache, conf)
   if ast == nil: return
   var d = newDocumentor(conf.projectFull, cache, conf)
-  d.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string;
-                          status: int; content: string) =
-    localError(conf, newLineInfo(conf, AbsoluteFile d.filename, -1, -1),
-               warnUser, "the ':test:' attribute is not supported by this backend")
   d.hasToc = true
   generateJson(d, ast)
   let json = d.jArray
@@ -1140,10 +1029,6 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) =
   var ast = parseFile(conf.projectMainIdx, cache, conf)
   if ast == nil: return
   var d = newDocumentor(conf.projectFull, cache, conf)
-  d.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string;
-                          status: int; content: string) =
-    localError(conf, newLineInfo(conf, AbsoluteFile d.filename, -1, -1),
-               warnUser, "the ':test:' attribute is not supported by this backend")
   d.hasToc = true
   var
     content: Rope

compiler/docgen2.nim

@@ -11,7 +11,8 @@
 # semantic checking.
 
 import
-  options, ast, msgs, idents, passes, docgen, lineinfos, pathutils
+  os, options, ast, astalgo, msgs, ropes, idents, passes, docgen, lineinfos,
+  pathutils
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -19,16 +20,16 @@ type
   TGen = object of PPassContext
     doc: PDoc
     module: PSym
-    config: ConfigRef
   PGen = ref TGen
 
 template shouldProcess(g): bool =
   (g.module.owner.id == g.doc.conf.mainPackageId and optWholeProject in g.doc.conf.globalOptions) or
-      sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
+      sfMainModule in g.module.flags
 
 template closeImpl(body: untyped) {.dirty.} =
   var g = PGen(p)
   let useWarning = sfMainModule notin g.module.flags
+  #echo g.module.name.s, " ", g.module.owner.id, " ", gMainPackageId
   if shouldProcess(g):
     body
     try:
@@ -60,7 +61,6 @@ template myOpenImpl(ext: untyped) {.dirty.} =
   var g: PGen
   new(g)
   g.module = module
-  g.config = graph.config
   var d = newDocumentor(AbsoluteFile toFullPath(graph.config, FileIndex module.position),
       graph.cache, graph.config, ext)
   d.hasToc = true

compiler/enumtostr.nim

@@ -1,98 +0,0 @@
-
-import ast, idents, lineinfos, modulegraphs, magicsys
-
-proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
-  result = newSym(skProc, getIdent(g.cache, "$"), t.owner, info)
-
-  let dest = newSym(skParam, getIdent(g.cache, "e"), result, info)
-  dest.typ = t
-
-  let res = newSym(skResult, getIdent(g.cache, "result"), result, info)
-  res.typ = getSysType(g, info, tyString)
-
-  result.typ = newType(tyProc, t.owner)
-  result.typ.n = newNodeI(nkFormalParams, info)
-  rawAddSon(result.typ, res.typ)
-  addSon(result.typ.n, newNodeI(nkEffectList, info))
-
-  result.typ.addParam dest
-
-  var body = newNodeI(nkStmtList, info)
-  var caseStmt = newNodeI(nkCaseStmt, info)
-  caseStmt.add(newSymNode dest)
-
-  # copy the branches over, but replace the fields with the for loop body:
-  for i in 0 ..< t.n.len:
-    assert(t.n[i].kind == nkSym)
-    var field = t.n[i].sym
-    let val = if field.ast == nil: field.name.s else: field.ast.strVal
-    caseStmt.add newTree(nkOfBranch, newSymNode(field),
-      newTree(nkStmtList, newTree(nkFastAsgn, newSymNode(res), newStrNode(val, info))))
-    #newIntTypeNode(nkIntLit, field.position, t)
-
-  body.add(caseStmt)
-
-  var n = newNodeI(nkProcDef, info, bodyPos+2)
-  for i in 0 ..< n.len: n.sons[i] = newNodeI(nkEmpty, info)
-  n.sons[namePos] = newSymNode(result)
-  n.sons[paramsPos] = result.typ.n
-  n.sons[bodyPos] = body
-  n.sons[resultPos] = newSymNode(res)
-  result.ast = n
-  incl result.flags, sfFromGeneric
-
-proc searchObjCase(obj: PNode; field: PSym): PNode =
-  case obj.kind
-  of nkSym:
-    result = nil
-  of nkElse, nkOfBranch:
-    result = searchObjCase(obj.lastSon, field)
-  else:
-    if obj.kind == nkRecCase and obj[0].kind == nkSym and obj[0].sym == field:
-      result = obj
-    else:
-      for x in obj:
-        result = searchObjCase(x, field)
-        if result != nil: break
-
-proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph): PSym =
-  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), t.owner, info)
-
-  let dest = newSym(skParam, getIdent(g.cache, "e"), result, info)
-  dest.typ = field.typ
-
-  let res = newSym(skResult, getIdent(g.cache, "result"), result, info)
-  res.typ = getSysType(g, info, tyUInt8)
-
-  result.typ = newType(tyProc, t.owner)
-  result.typ.n = newNodeI(nkFormalParams, info)
-  rawAddSon(result.typ, res.typ)
-  addSon(result.typ.n, newNodeI(nkEffectList, info))
-
-  result.typ.addParam dest
-
-  var body = newNodeI(nkStmtList, info)
-  var caseStmt = newNodeI(nkCaseStmt, info)
-  caseStmt.add(newSymNode dest)
-
-  let subObj = searchObjCase(t.n, field)
-  doAssert subObj != nil
-  for i in 1 ..< subObj.len:
-    let ofBranch = subObj[i]
-    var newBranch = newNodeI(ofBranch.kind, ofBranch.info)
-    for j in 0..ofBranch.len-2:
-      newBranch.add ofBranch[j]
-
-    newBranch.add newTree(nkStmtList, newTree(nkFastAsgn, newSymNode(res), newIntNode(nkInt8Lit, i)))
-    caseStmt.add newBranch
-
-  body.add(caseStmt)
-
-  var n = newNodeI(nkProcDef, info, bodyPos+2)
-  for i in 0 ..< n.len: n.sons[i] = newNodeI(nkEmpty, info)
-  n.sons[namePos] = newSymNode(result)
-  n.sons[paramsPos] = result.typ.n
-  n.sons[bodyPos] = body
-  n.sons[resultPos] = newSymNode(res)
-  result.ast = n
-  incl result.flags, sfFromGeneric

compiler/evalffi.nim

@@ -9,47 +9,43 @@
 
 ## This file implements the FFI part of the evaluator for Nim code.
 
-import ast, astalgo, ropes, types, options, tables, dynlib, msgs, os, lineinfos
-import pkg/libffi
+import ast, astalgo, ropes, types, options, tables, dynlib, libffi, msgs, os
 
 when defined(windows):
   const libcDll = "msvcrt.dll"
-elif defined(linux):
-  const libcDll = "libc.so(.6|.5|)"
-elif defined(osx):
-  const libcDll = "/usr/lib/libSystem.dylib"
 else:
-  {.error: "`libcDll` not implemented on this platform".}
+  const libcDll = "libc.so(.6|.5|)"
 
 type
-  TDllCache = tables.Table[string, LibHandle]
+  TDllCache = tables.TTable[string, TLibHandle]
 var
-  gDllCache = initTable[string, LibHandle]()
+  gDllCache = initTable[string, TLibHandle]()
 
 when defined(windows):
   var gExeHandle = loadLib(os.getAppFilename())
 else:
   var gExeHandle = loadLib()
 
-proc getDll(conf: ConfigRef, cache: var TDllCache; dll: string; info: TLineInfo): pointer =
-  if dll in cache:
-    return cache[dll]
-  var libs: seq[string]
-  libCandidates(dll, libs)
-  for c in libs:
-    result = loadLib(c)
-    if not result.isNil: break
+proc getDll(cache: var TDllCache; dll: string; info: TLineInfo): pointer =
+  result = cache[dll]
   if result.isNil:
-    globalError(conf, info, "cannot load: " & dll)
-  cache[dll] = result
+    var libs: seq[string] = @[]
+    libCandidates(dll, libs)
+    for c in libs:
+      result = loadLib(c)
+      if not result.isNil: break
+    if result.isNil:
+      globalError(info, "cannot load: " & dll)
+    cache[dll] = result
 
 const
   nkPtrLit = nkIntLit # hopefully we can get rid of this hack soon
 
 var myerrno {.importc: "errno", header: "<errno.h>".}: cint ## error variable
 
-proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode =
-  let name = $sym.loc.r
+proc importcSymbol*(sym: PSym): PNode =
+  let name = ropeToStr(sym.loc.r)
+
   # the AST does not support untyped pointers directly, so we use an nkIntLit
   # that contains the address instead:
   result = newNodeIT(nkPtrLit, sym.info, sym.typ)
@@ -61,28 +57,28 @@ proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode =
   else:
     let lib = sym.annex
     if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}:
-      globalError(conf, sym.info, "dynlib needs to be a string lit")
+      globalError(sym.info, "dynlib needs to be a string lit for the REPL")
     var theAddr: pointer
-    if (lib.isNil or lib.kind == libHeader) and not gExehandle.isNil:
+    if lib.isNil and not gExehandle.isNil:
       # first try this exe itself:
       theAddr = gExehandle.symAddr(name)
       # then try libc:
       if theAddr.isNil:
-        let dllhandle = getDll(conf, gDllCache, libcDll, sym.info)
+        let dllhandle = gDllCache.getDll(libcDll, sym.info)
         theAddr = dllhandle.symAddr(name)
     elif not lib.isNil:
-      let dll = if lib.kind == libHeader: libcDll else: lib.path.strVal
-      let dllhandle = getDll(conf, gDllCache, dll, sym.info)
+      let dllhandle = gDllCache.getDll(if lib.kind == libHeader: libcDll
+                                       else: lib.path.strVal, sym.info)
       theAddr = dllhandle.symAddr(name)
-    if theAddr.isNil: globalError(conf, sym.info, "cannot import: " & sym.name.s)
+    if theAddr.isNil: globalError(sym.info, "cannot import: " & sym.name.s)
     result.intVal = cast[ByteAddress](theAddr)
 
-proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType =
+proc mapType(t: ast.PType): ptr libffi.TType =
   if t == nil: return addr libffi.type_void
 
   case t.kind
   of tyBool, tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64, tySet:
-    case getSize(conf, t)
+    case t.getSize
     of 1: result = addr libffi.type_uint8
     of 2: result = addr libffi.type_sint16
     of 4: result = addr libffi.type_sint32
@@ -90,91 +86,91 @@ proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType =
     else: result = nil
   of tyFloat, tyFloat64: result = addr libffi.type_double
   of tyFloat32: result = addr libffi.type_float
-  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyUntyped,
-     tyTyped, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
+  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr,
+     tyStmt, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
     result = addr libffi.type_pointer
   of tyDistinct, tyAlias, tySink:
-    result = mapType(conf, t.sons[0])
+    result = mapType(t.sons[0])
   else:
     result = nil
   # too risky:
   #of tyFloat128: result = addr libffi.type_longdouble
 
-proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI =
+proc mapCallConv(cc: TCallingConvention, info: TLineInfo): TABI =
   case cc
   of ccDefault: result = DEFAULT_ABI
-  of ccStdCall: result = when defined(windows) and defined(x86): STDCALL else: DEFAULT_ABI
+  of ccStdCall: result = when defined(windows): STDCALL else: DEFAULT_ABI
   of ccCDecl: result = DEFAULT_ABI
   else:
-    globalError(conf, info, "cannot map calling convention to FFI")
+    globalError(info, "cannot map calling convention to FFI")
 
 template rd(T, p: untyped): untyped = (cast[ptr T](p))[]
 template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v
 template `+!`(x, y: untyped): untyped =
   cast[pointer](cast[ByteAddress](x) + y)
 
-proc packSize(conf: ConfigRef, v: PNode, typ: PType): int =
+proc packSize(v: PNode, typ: PType): int =
   ## computes the size of the blob
   case typ.kind
   of tyPtr, tyRef, tyVar, tyLent:
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     else:
-      result = sizeof(pointer) + packSize(conf, v.sons[0], typ.lastSon)
+      result = sizeof(pointer) + packSize(v.sons[0], typ.lastSon)
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    result = packSize(conf, v, typ.sons[0])
+    result = packSize(v, typ.sons[0])
   of tyArray:
     # consider: ptr array[0..1000_000, int] which is common for interfacing;
     # we use the real length here instead
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     elif v.len != 0:
-      result = v.len * packSize(conf, v.sons[0], typ.sons[1])
+      result = v.len * packSize(v.sons[0], typ.sons[1])
   else:
-    result = getSize(conf, typ).int
+    result = typ.getSize.int
 
-proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer)
+proc pack(v: PNode, typ: PType, res: pointer)
 
-proc getField(conf: ConfigRef, n: PNode; position: int): PSym =
+proc getField(n: PNode; position: int): PSym =
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      result = getField(conf, n.sons[i], position)
+    for i in countup(0, sonsLen(n) - 1):
+      result = getField(n.sons[i], position)
       if result != nil: return
   of nkRecCase:
-    result = getField(conf, n.sons[0], position)
+    result = getField(n.sons[0], position)
     if result != nil: return
-    for i in 1 ..< sonsLen(n):
+    for i in countup(1, sonsLen(n) - 1):
       case n.sons[i].kind
       of nkOfBranch, nkElse:
-        result = getField(conf, lastSon(n.sons[i]), position)
+        result = getField(lastSon(n.sons[i]), position)
         if result != nil: return
-      else: internalError(conf, n.info, "getField(record case branch)")
+      else: internalError(n.info, "getField(record case branch)")
   of nkSym:
     if n.sym.position == position: result = n.sym
   else: discard
 
-proc packObject(conf: ConfigRef, x: PNode, typ: PType, res: pointer) =
-  internalAssert conf, x.kind in {nkObjConstr, nkPar, nkTupleConstr}
+proc packObject(x: PNode, typ: PType, res: pointer) =
+  internalAssert x.kind in {nkObjConstr, nkPar, nkTupleConstr}
   # compute the field's offsets:
-  discard getSize(conf, typ)
-  for i in ord(x.kind == nkObjConstr) ..< sonsLen(x):
+  discard typ.getSize
+  for i in countup(ord(x.kind == nkObjConstr), sonsLen(x) - 1):
     var it = x.sons[i]
     if it.kind == nkExprColonExpr:
-      internalAssert conf, it.sons[0].kind == nkSym
+      internalAssert it.sons[0].kind == nkSym
       let field = it.sons[0].sym
-      pack(conf, it.sons[1], field.typ, res +! field.offset)
+      pack(it.sons[1], field.typ, res +! field.offset)
     elif typ.n != nil:
-      let field = getField(conf, typ.n, i)
-      pack(conf, it, field.typ, res +! field.offset)
+      let field = getField(typ.n, i)
+      pack(it, field.typ, res +! field.offset)
     else:
       # XXX: todo
-      globalError(conf, x.info, "cannot pack unnamed tuple")
+      globalError(x.info, "cannot pack unnamed tuple")
 
 const maxPackDepth = 20
 var packRecCheck = 0
 
-proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
+proc pack(v: PNode, typ: PType, res: pointer) =
   template awr(T, v: untyped): untyped =
     wr(T, res, v)
 
@@ -192,13 +188,13 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
   of tyUInt32: awr(uint32, v.intVal.uint32)
   of tyUInt64: awr(uint64, v.intVal.uint64)
   of tyEnum, tySet:
-    case getSize(conf, v.typ)
+    case v.typ.getSize
     of 1: awr(uint8, v.intVal.uint8)
     of 2: awr(uint16, v.intVal.uint16)
     of 4: awr(int32, v.intVal.int32)
     of 8: awr(int64, v.intVal.int64)
     else:
-      globalError(conf, v.info, "cannot map value to FFI (tyEnum, tySet)")
+      globalError(v.info, "cannot map value to FFI (tyEnum, tySet)")
   of tyFloat: awr(float, v.floatVal)
   of tyFloat32: awr(float32, v.floatVal)
   of tyFloat64: awr(float64, v.floatVal)
@@ -212,7 +208,7 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
     elif v.kind in {nkStrLit..nkTripleStrLit}:
       awr(cstring, cstring(v.strVal))
     else:
-      globalError(conf, v.info, "cannot map pointer/proc value to FFI")
+      globalError(v.info, "cannot map pointer/proc value to FFI")
   of tyPtr, tyRef, tyVar, tyLent:
     if v.kind == nkNilLit:
       # nothing to do since the memory is 0 initialized anyway
@@ -222,44 +218,44 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
     else:
       if packRecCheck > maxPackDepth:
         packRecCheck = 0
-        globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ))
+        globalError(v.info, "cannot map value to FFI " & typeToString(v.typ))
       inc packRecCheck
-      pack(conf, v.sons[0], typ.lastSon, res +! sizeof(pointer))
+      pack(v.sons[0], typ.lastSon, res +! sizeof(pointer))
       dec packRecCheck
       awr(pointer, res +! sizeof(pointer))
   of tyArray:
-    let baseSize = getSize(conf, typ.sons[1])
+    let baseSize = typ.sons[1].getSize
     for i in 0 ..< v.len:
-      pack(conf, v.sons[i], typ.sons[1], res +! i * baseSize)
+      pack(v.sons[i], typ.sons[1], res +! i * baseSize)
   of tyObject, tyTuple:
-    packObject(conf, v, typ, res)
+    packObject(v, typ, res)
   of tyNil:
     discard
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    pack(conf, v, typ.sons[0], res)
+    pack(v, typ.sons[0], res)
   else:
-    globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ))
+    globalError(v.info, "cannot map value to FFI " & typeToString(v.typ))
 
-proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode
+proc unpack(x: pointer, typ: PType, n: PNode): PNode
 
-proc unpackObjectAdd(conf: ConfigRef, x: pointer, n, result: PNode) =
+proc unpackObjectAdd(x: pointer, n, result: PNode) =
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      unpackObjectAdd(conf, x, n.sons[i], result)
+    for i in countup(0, sonsLen(n) - 1):
+      unpackObjectAdd(x, n.sons[i], result)
   of nkRecCase:
-    globalError(conf, result.info, "case objects cannot be unpacked")
+    globalError(result.info, "case objects cannot be unpacked")
   of nkSym:
     var pair = newNodeI(nkExprColonExpr, result.info, 2)
     pair.sons[0] = n
-    pair.sons[1] = unpack(conf, x +! n.sym.offset, n.sym.typ, nil)
+    pair.sons[1] = unpack(x +! n.sym.offset, n.sym.typ, nil)
     #echo "offset: ", n.sym.name.s, " ", n.sym.offset
     result.add pair
   else: discard
 
-proc unpackObject(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
+proc unpackObject(x: pointer, typ: PType, n: PNode): PNode =
   # compute the field's offsets:
-  discard getSize(conf, typ)
+  discard typ.getSize
 
   # iterate over any actual field of 'n' ... if n is nil we need to create
   # the nkPar node:
@@ -267,36 +263,36 @@ proc unpackObject(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
     result = newNode(nkTupleConstr)
     result.typ = typ
     if typ.n.isNil:
-      internalError(conf, "cannot unpack unnamed tuple")
-    unpackObjectAdd(conf, x, typ.n, result)
+      internalError("cannot unpack unnamed tuple")
+    unpackObjectAdd(x, typ.n, result)
   else:
     result = n
     if result.kind notin {nkObjConstr, nkPar, nkTupleConstr}:
-      globalError(conf, n.info, "cannot map value from FFI")
+      globalError(n.info, "cannot map value from FFI")
     if typ.n.isNil:
-      globalError(conf, n.info, "cannot unpack unnamed tuple")
-    for i in ord(n.kind == nkObjConstr) ..< sonsLen(n):
+      globalError(n.info, "cannot unpack unnamed tuple")
+    for i in countup(ord(n.kind == nkObjConstr), sonsLen(n) - 1):
       var it = n.sons[i]
       if it.kind == nkExprColonExpr:
-        internalAssert conf, it.sons[0].kind == nkSym
+        internalAssert it.sons[0].kind == nkSym
         let field = it.sons[0].sym
-        it.sons[1] = unpack(conf, x +! field.offset, field.typ, it.sons[1])
+        it.sons[1] = unpack(x +! field.offset, field.typ, it.sons[1])
       else:
-        let field = getField(conf, typ.n, i)
-        n.sons[i] = unpack(conf, x +! field.offset, field.typ, it)
+        let field = getField(typ.n, i)
+        n.sons[i] = unpack(x +! field.offset, field.typ, it)
 
-proc unpackArray(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
+proc unpackArray(x: pointer, typ: PType, n: PNode): PNode =
   if n.isNil:
     result = newNode(nkBracket)
     result.typ = typ
-    newSeq(result.sons, lengthOrd(conf, typ).int)
+    newSeq(result.sons, lengthOrd(typ).int)
   else:
     result = n
     if result.kind != nkBracket:
-      globalError(conf, n.info, "cannot map value from FFI")
-  let baseSize = getSize(conf, typ.sons[1])
+      globalError(n.info, "cannot map value from FFI")
+  let baseSize = typ.sons[1].getSize
   for i in 0 ..< result.len:
-    result.sons[i] = unpack(conf, x +! i * baseSize, typ.sons[1], result.sons[i])
+    result.sons[i] = unpack(x +! i * baseSize, typ.sons[1], result.sons[i])
 
 proc canonNodeKind(k: TNodeKind): TNodeKind =
   case k
@@ -305,7 +301,7 @@ proc canonNodeKind(k: TNodeKind): TNodeKind =
   of nkStrLit..nkTripleStrLit: result = nkStrLit
   else: result = k
 
-proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
+proc unpack(x: pointer, typ: PType, n: PNode): PNode =
   template aw(k, v, field: untyped): untyped =
     if n.isNil:
       result = newNode(k)
@@ -317,7 +313,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
         #echo "expected ", k, " but got ", result.kind
         #debug result
         return newNodeI(nkExceptBranch, n.info)
-        #globalError(conf, n.info, "cannot map value from FFI")
+        #globalError(n.info, "cannot map value from FFI")
     result.field = v
 
   template setNil() =
@@ -348,13 +344,13 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   of tyUInt32: awi(nkUInt32Lit, rd(uint32, x).BiggestInt)
   of tyUInt64: awi(nkUInt64Lit, rd(uint64, x).BiggestInt)
   of tyEnum:
-    case getSize(conf, typ)
+    case typ.getSize
     of 1: awi(nkIntLit, rd(uint8, x).BiggestInt)
     of 2: awi(nkIntLit, rd(uint16, x).BiggestInt)
     of 4: awi(nkIntLit, rd(int32, x).BiggestInt)
     of 8: awi(nkIntLit, rd(int64, x).BiggestInt)
     else:
-      globalError(conf, n.info, "cannot map value from FFI (tyEnum, tySet)")
+      globalError(n.info, "cannot map value from FFI (tyEnum, tySet)")
   of tyFloat: awf(nkFloatLit, rd(float, x))
   of tyFloat32: awf(nkFloat32Lit, rd(float32, x))
   of tyFloat64: awf(nkFloat64Lit, rd(float64, x))
@@ -375,15 +371,15 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
     elif n == nil or n.kind == nkPtrLit:
       awi(nkPtrLit, cast[ByteAddress](p))
     elif n != nil and n.len == 1:
-      internalAssert(conf, n.kind == nkRefTy)
-      n.sons[0] = unpack(conf, p, typ.lastSon, n.sons[0])
+      internalAssert n.kind == nkRefTy
+      n.sons[0] = unpack(p, typ.lastSon, n.sons[0])
       result = n
     else:
-      globalError(conf, n.info, "cannot map value from FFI " & typeToString(typ))
+      globalError(n.info, "cannot map value from FFI " & typeToString(typ))
   of tyObject, tyTuple:
-    result = unpackObject(conf, x, typ, n)
+    result = unpackObject(x, typ, n)
   of tyArray:
-    result = unpackArray(conf, x, typ, n)
+    result = unpackArray(x, typ, n)
   of tyCString, tyString:
     let p = rd(cstring, x)
     if p.isNil:
@@ -393,12 +389,12 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   of tyNil:
     setNil()
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    result = unpack(conf, x, typ.lastSon, n)
+    result = unpack(x, typ.lastSon, n)
   else:
     # XXX what to do with 'array' here?
-    globalError(conf, n.info, "cannot map value from FFI " & typeToString(typ))
+    globalError(n.info, "cannot map value from FFI " & typeToString(typ))
 
-proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode =
+proc fficast*(x: PNode, destTyp: PType): PNode =
   if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyLent, tyPointer,
                                            tyProc, tyCString, tyString,
                                            tySequence}:
@@ -408,93 +404,93 @@ proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode =
     result = newNodeIT(x.kind, x.info, destTyp)
   else:
     # we play safe here and allocate the max possible size:
-    let size = max(packSize(conf, x, x.typ), packSize(conf, x, destTyp))
+    let size = max(packSize(x, x.typ), packSize(x, destTyp))
     var a = alloc0(size)
-    pack(conf, x, x.typ, a)
+    pack(x, x.typ, a)
     # cast through a pointer needs a new inner object:
     let y = if x.kind == nkRefTy: newNodeI(nkRefTy, x.info, 1)
             else: x.copyTree
     y.typ = x.typ
-    result = unpack(conf, a, destTyp, y)
+    result = unpack(a, destTyp, y)
     dealloc a
 
-proc callForeignFunction*(conf: ConfigRef, call: PNode): PNode =
-  internalAssert conf, call.sons[0].kind == nkPtrLit
+proc callForeignFunction*(call: PNode): PNode =
+  internalAssert call.sons[0].kind == nkPtrLit
 
   var cif: TCif
   var sig: TParamList
   # use the arguments' types for varargs support:
   for i in 1..call.len-1:
-    sig[i-1] = mapType(conf, call.sons[i].typ)
+    sig[i-1] = mapType(call.sons[i].typ)
     if sig[i-1].isNil:
-      globalError(conf, call.info, "cannot map FFI type")
+      globalError(call.info, "cannot map FFI type")
 
   let typ = call.sons[0].typ
-  if prep_cif(cif, mapCallConv(conf, typ.callConv, call.info), cuint(call.len-1),
-              mapType(conf, typ.sons[0]), sig) != OK:
-    globalError(conf, call.info, "error in FFI call")
+  if prep_cif(cif, mapCallConv(typ.callConv, call.info), cuint(call.len-1),
+              mapType(typ.sons[0]), sig) != OK:
+    globalError(call.info, "error in FFI call")
 
   var args: TArgList
   let fn = cast[pointer](call.sons[0].intVal)
   for i in 1 .. call.len-1:
     var t = call.sons[i].typ
-    args[i-1] = alloc0(packSize(conf, call.sons[i], t))
-    pack(conf, call.sons[i], t, args[i-1])
+    args[i-1] = alloc0(packSize(call.sons[i], t))
+    pack(call.sons[i], t, args[i-1])
   let retVal = if isEmptyType(typ.sons[0]): pointer(nil)
-               else: alloc(getSize(conf, typ.sons[0]).int)
+               else: alloc(typ.sons[0].getSize.int)
 
   libffi.call(cif, fn, retVal, args)
 
   if retVal.isNil:
     result = newNode(nkEmpty)
   else:
-    result = unpack(conf, retVal, typ.sons[0], nil)
+    result = unpack(retVal, typ.sons[0], nil)
     result.info = call.info
 
   if retVal != nil: dealloc retVal
   for i in 1 .. call.len-1:
-    call.sons[i] = unpack(conf, args[i-1], typ.sons[i], call[i])
+    call.sons[i] = unpack(args[i-1], typ.sons[i], call[i])
     dealloc args[i-1]
 
-proc callForeignFunction*(conf: ConfigRef, fn: PNode, fntyp: PType,
+proc callForeignFunction*(fn: PNode, fntyp: PType,
                           args: var TNodeSeq, start, len: int,
                           info: TLineInfo): PNode =
-  internalAssert conf, fn.kind == nkPtrLit
+  internalAssert fn.kind == nkPtrLit
 
   var cif: TCif
   var sig: TParamList
   for i in 0..len-1:
     var aTyp = args[i+start].typ
     if aTyp.isNil:
-      internalAssert conf, i+1 < fntyp.len
+      internalAssert i+1 < fntyp.len
       aTyp = fntyp.sons[i+1]
       args[i+start].typ = aTyp
-    sig[i] = mapType(conf, aTyp)
-    if sig[i].isNil: globalError(conf, info, "cannot map FFI type")
+    sig[i] = mapType(aTyp)
+    if sig[i].isNil: globalError(info, "cannot map FFI type")
 
-  if prep_cif(cif, mapCallConv(conf, fntyp.callConv, info), cuint(len),
-              mapType(conf, fntyp.sons[0]), sig) != OK:
-    globalError(conf, info, "error in FFI call")
+  if prep_cif(cif, mapCallConv(fntyp.callConv, info), cuint(len),
+              mapType(fntyp.sons[0]), sig) != OK:
+    globalError(info, "error in FFI call")
 
   var cargs: TArgList
   let fn = cast[pointer](fn.intVal)
   for i in 0 .. len-1:
     let t = args[i+start].typ
-    cargs[i] = alloc0(packSize(conf, args[i+start], t))
-    pack(conf, args[i+start], t, cargs[i])
+    cargs[i] = alloc0(packSize(args[i+start], t))
+    pack(args[i+start], t, cargs[i])
   let retVal = if isEmptyType(fntyp.sons[0]): pointer(nil)
-               else: alloc(getSize(conf, fntyp.sons[0]).int)
+               else: alloc(fntyp.sons[0].getSize.int)
 
   libffi.call(cif, fn, retVal, cargs)
 
   if retVal.isNil:
     result = newNode(nkEmpty)
   else:
-    result = unpack(conf, retVal, fntyp.sons[0], nil)
+    result = unpack(retVal, fntyp.sons[0], nil)
     result.info = info
 
   if retVal != nil: dealloc retVal
   for i in 0 .. len-1:
     let t = args[i+start].typ
-    args[i+start] = unpack(conf, cargs[i], t, args[i+start])
+    args[i+start] = unpack(cargs[i], t, args[i+start])
     dealloc cargs[i]

compiler/evaltempl.nim

@@ -10,7 +10,8 @@
 ## Template evaluation engine. Now hygienic.
 
 import
-  strutils, options, ast, astalgo, msgs, renderer, lineinfos
+  strutils, options, ast, astalgo, msgs, os, idents, wordrecg, renderer,
+  lineinfos
 
 type
   TemplCtx = object
@@ -36,21 +37,18 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
   case templ.kind
   of nkSym:
     var s = templ.sym
-    if (s.owner == nil and s.kind == skParam) or s.owner == c.owner:
+    if s.owner.id == c.owner.id:
       if s.kind == skParam and sfGenSym notin s.flags:
         handleParam actual.sons[s.position]
-      elif (s.owner != nil) and (s.kind == skGenericParam or
-           s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam):
+      elif s.kind == skGenericParam or
+           s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam:
         handleParam actual.sons[s.owner.typ.len + s.position - 1]
       else:
         internalAssert c.config, sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
         if x == nil:
-          x = copySym(s)
-          # sem'check needs to set the owner properly later, see bug #9476
-          x.owner = nil # c.genSymOwner
-          #if x.kind == skParam and x.owner.kind == skModule:
-          #  internalAssert c.config, false
+          x = copySym(s, false)
+          x.owner = c.genSymOwner
           idTablePut(c.mapping, s, x)
         result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
     else:
@@ -63,7 +61,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
     # "declarative" context (bug #9235).
     if c.isDeclarative:
       var res = copyNode(c, templ, actual)
-      for i in 0 ..< sonsLen(templ):
+      for i in countup(0, sonsLen(templ) - 1):
         evalTemplateAux(templ.sons[i], actual, c, res)
       result.add res
     else:
@@ -77,7 +75,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
       c.isDeclarative = true
       isDeclarative = true
     var res = copyNode(c, templ, actual)
-    for i in 0 ..< sonsLen(templ):
+    for i in countup(0, sonsLen(templ) - 1):
       evalTemplateAux(templ.sons[i], actual, c, res)
     result.add res
     if isDeclarative: c.isDeclarative = false
@@ -103,7 +101,7 @@ proc evalTemplateArgs(n: PNode, s: PSym; conf: ConfigRef; fromHlo: bool): PNode
     # their bodies. We could try to fix this, but it may be
     # wiser to just deprecate immediate templates and macros
     # now that we have working untyped parameters.
-    genericParams = if fromHlo: 0
+    genericParams = if sfImmediate in s.flags or fromHlo: 0
                     else: s.ast[genericParamsPos].len
     expectedRegularParams = s.typ.len-1
     givenRegularParams = totalParams - genericParams
@@ -175,7 +173,6 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   initIdTable(ctx.mapping)
 
   let body = tmpl.getBody
-  #echo "instantion of ", renderTree(body, {renderIds})
   if isAtom(body):
     result = newNodeI(nkPar, body.info)
     evalTemplateAux(body, args, ctx, result)
@@ -185,13 +182,12 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
                   renderTree(result, {renderNoComments}))
   else:
     result = copyNode(body)
-    ctx.instLines = sfCallsite in tmpl.flags
-    if ctx.instLines:
-      result.info = n.info
-    for i in 0 ..< safeLen(body):
+    #ctx.instLines = body.kind notin {nkStmtList, nkStmtListExpr,
+    #                                 nkBlockStmt, nkBlockExpr}
+    #if ctx.instLines: result.info = n.info
+    for i in countup(0, safeLen(body) - 1):
       evalTemplateAux(body.sons[i], args, ctx, result)
   result.flags.incl nfFromTemplate
   result = wrapInComesFrom(n.info, tmpl, result)
-  #if ctx.debugActive:
-  #  echo "instantion of ", renderTree(result, {renderIds})
   dec(conf.evalTemplateCounter)
+

compiler/extccomp.nim

@@ -14,7 +14,7 @@
 
 import
   ropes, os, strutils, osproc, platform, condsyms, options, msgs,
-  lineinfos, std / sha1, streams, pathutils, sequtils, times
+  lineinfos, std / sha1, streams, pathutils
 
 type
   TInfoCCProp* = enum         # properties of the C compiler:
@@ -63,8 +63,8 @@ compiler gcc:
   result = (
     name: "gcc",
     objExt: "o",
-    optSpeed: " -O3 ",
-    optSize: " -Os ",
+    optSpeed: " -O3 -ffast-math ",
+    optSize: " -Os -ffast-math ",
     compilerExe: "gcc",
     cppCompiler: "g++",
     compileTmpl: "-c $options $include -o $objfile $file",
@@ -88,8 +88,8 @@ compiler nintendoSwitchGCC:
   result = (
     name: "switch_gcc",
     objExt: "o",
-    optSpeed: " -O3 ",
-    optSize: " -Os ",
+    optSpeed: " -O3 -ffast-math ",
+    optSize: " -Os -ffast-math ",
     compilerExe: "aarch64-none-elf-gcc",
     cppCompiler: "aarch64-none-elf-g++",
     compileTmpl: "-w -MMD -MP -MF $dfile -c $options $include -o $objfile $file",
@@ -134,16 +134,16 @@ compiler vcc:
   result = (
     name: "vcc",
     objExt: "obj",
-    optSpeed: " /Ogityb2 /G7 ",
+    optSpeed: " /Ogityb2 /G7 /arch:SSE2 ",
     optSize: " /O1 /G7 ",
     compilerExe: "cl",
     cppCompiler: "cl",
-    compileTmpl: "/c$vccplatform$options $include /Fo$objfile $file",
+    compileTmpl: "/c $options $include /Fo$objfile $file",
     buildGui: " /link /SUBSYSTEM:WINDOWS ",
     buildDll: " /LD",
     buildLib: "lib /OUT:$libfile $objfiles",
     linkerExe: "cl",
-    linkTmpl: "$options $builddll$vccplatform /Fe$exefile $objfiles $buildgui",
+    linkTmpl: "$options $builddll /Fe$exefile $objfiles $buildgui",
     includeCmd: " /I",
     linkDirCmd: " /LIBPATH:",
     linkLibCmd: " $1.lib",
@@ -153,13 +153,6 @@ compiler vcc:
     structStmtFmt: "$3$n$1 $2",
     props: {hasCpp, hasAssume, hasDeclspec})
 
-compiler clangcl:
-  result = vcc()
-  result.name = "clang_cl"
-  result.compilerExe = "clang-cl"
-  result.cppCompiler = "clang-cl"
-  result.linkerExe = "clang-cl"
-
 # Intel C/C++ Compiler
 compiler icl:
   result = vcc()
@@ -223,6 +216,7 @@ compiler bcc:
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard,
             hasAttribute})
 
+
 # Digital Mars C Compiler
 compiler dmc:
   result = (
@@ -359,8 +353,7 @@ const
     pcc(),
     ucc(),
     icl(),
-    icc(),
-    clangcl()]
+    icc()]
 
   hExt* = ".h"
 
@@ -370,20 +363,11 @@ proc libNameTmpl(conf: ConfigRef): string {.inline.} =
 proc nameToCC*(name: string): TSystemCC =
   ## Returns the kind of compiler referred to by `name`, or ccNone
   ## if the name doesn't refer to any known compiler.
-  for i in succ(ccNone) .. high(TSystemCC):
+  for i in countup(succ(ccNone), high(TSystemCC)):
     if cmpIgnoreStyle(name, CC[i].name) == 0:
       return i
   result = ccNone
 
-proc listCCnames(): seq[string] =
-  for i in succ(ccNone) .. high(TSystemCC):
-    result.add CC[i].name
-
-proc isVSCompatible*(conf: ConfigRef): bool =
-  return conf.cCompiler == ccVcc or
-          conf.cCompiler == ccClangCl or
-          (conf.cCompiler == ccIcl and conf.target.hostOS in osDos..osWindows)
-
 proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
   # use ``cpu.os.cc`` for cross compilation, unless ``--compileOnly`` is given
   # for niminst support
@@ -412,12 +396,11 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
 proc setCC*(conf: ConfigRef; ccname: string; info: TLineInfo) =
   conf.cCompiler = nameToCC(ccname)
   if conf.cCompiler == ccNone:
-    let ccList = listCCnames().join(", ")
-    localError(conf, info, "unknown C compiler: '$1'. Available options are: $2" % [ccname, ccList])
+    localError(conf, info, "unknown C compiler: '$1'" % ccname)
   conf.compileOptions = getConfigVar(conf, conf.cCompiler, ".options.always")
   conf.linkOptions = ""
-  conf.cCompilerPath = getConfigVar(conf, conf.cCompiler, ".path")
-  for i in low(CC) .. high(CC): undefSymbol(conf.symbols, CC[i].name)
+  conf.ccompilerpath = getConfigVar(conf, conf.cCompiler, ".path")
+  for i in countup(low(CC), high(CC)): undefSymbol(conf.symbols, CC[i].name)
   defineSymbol(conf.symbols, CC[conf.cCompiler].name)
 
 proc addOpt(dest: var string, src: string) =
@@ -439,14 +422,14 @@ proc addCompileOptionCmd*(conf: ConfigRef; option: string) =
 
 proc initVars*(conf: ConfigRef) =
   # we need to define the symbol here, because ``CC`` may have never been set!
-  for i in low(CC) .. high(CC): undefSymbol(conf.symbols, CC[i].name)
+  for i in countup(low(CC), high(CC)): undefSymbol(conf.symbols, CC[i].name)
   defineSymbol(conf.symbols, CC[conf.cCompiler].name)
   addCompileOption(conf, getConfigVar(conf, conf.cCompiler, ".options.always"))
   #addLinkOption(getConfigVar(cCompiler, ".options.linker"))
-  if len(conf.cCompilerPath) == 0:
-    conf.cCompilerPath = getConfigVar(conf, conf.cCompiler, ".path")
+  if len(conf.ccompilerpath) == 0:
+    conf.ccompilerpath = getConfigVar(conf, conf.cCompiler, ".path")
 
-proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile,
+proc completeCFilePath*(conf: ConfigRef; cfile: AbsoluteFile,
                         createSubDir: bool = true): AbsoluteFile =
   result = completeGeneratedFilePath(conf, cfile, createSubDir)
 
@@ -477,8 +460,8 @@ proc execExternalProgram*(conf: ConfigRef; cmd: string, msg = hintExecuting) =
     rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" %
       cmd)
 
-proc generateScript(conf: ConfigRef; script: Rope) =
-  let (_, name, _) = splitFile(conf.outFile.string)
+proc generateScript(conf: ConfigRef; projectFile: AbsoluteFile, script: Rope) =
+  let (_, name, _) = splitFile(projectFile)
   let filename = getNimcacheDir(conf) / RelativeFile(addFileExt("compile_" & name,
                                      platform.OS[conf.target.targetOS].scriptExt))
   if writeRope(script, filename):
@@ -509,42 +492,30 @@ proc noAbsolutePaths(conf: ConfigRef): bool {.inline.} =
   # `optGenMapping` is included here for niminst.
   result = conf.globalOptions * {optGenScript, optGenMapping} != {}
 
-proc cFileSpecificOptions(conf: ConfigRef; nimname: string): string =
+proc cFileSpecificOptions(conf: ConfigRef; cfilename: AbsoluteFile): string =
   result = conf.compileOptions
   for option in conf.compileOptionsCmd:
     if strutils.find(result, option, 0) < 0:
       addOpt(result, option)
 
+  let trunk = splitFile(cfilename).name
   if optCDebug in conf.globalOptions:
-    let key = nimname & ".debug"
+    let key = trunk & ".debug"
     if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key))
     else: addOpt(result, getDebug(conf, conf.cCompiler))
   if optOptimizeSpeed in conf.options:
-    let key = nimname & ".speed"
+    let key = trunk & ".speed"
     if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key))
     else: addOpt(result, getOptSpeed(conf, conf.cCompiler))
   elif optOptimizeSize in conf.options:
-    let key = nimname & ".size"
+    let key = trunk & ".size"
     if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key))
     else: addOpt(result, getOptSize(conf, conf.cCompiler))
-  let key = nimname & ".always"
+  let key = trunk & ".always"
   if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key))
 
 proc getCompileOptions(conf: ConfigRef): string =
-  result = cFileSpecificOptions(conf, "__dummy__")
-
-proc vccplatform(conf: ConfigRef): string =
-  # VCC specific but preferable over the config hacks people
-  # had to do before, see #11306
-  case conf.target.targetCPU
-  of cpuI386:
-    result = " --platform:x86"
-  of cpuArm:
-    result = " --platform:arm"
-  of cpuAmd64:
-    result = " --platform:amd64"
-  else:
-    result = ""
+  result = cFileSpecificOptions(conf, AbsoluteFile"__dummy__")
 
 proc getLinkOptions(conf: ConfigRef): string =
   result = conf.linkOptions & " " & conf.linkOptionsCmd & " "
@@ -572,14 +543,14 @@ proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
            elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler
            else: getCompilerExe(conf, compiler, AbsoluteFile"")
 
-proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): string =
+proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile): string =
   var c = conf.cCompiler
-  var options = cFileSpecificOptions(conf, cfile.nimname)
+  var options = cFileSpecificOptions(conf, cfile.cname)
   var exe = getConfigVar(conf, c, ".exe")
   if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname)
 
   if needsExeExt(conf): exe = addFileExt(exe, "exe")
-  if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and
+  if optGenDynLib in conf.globalOptions and
       ospNeedsPIC in platform.OS[conf.target.targetOS].props:
     add(options, ' ' & CC[c].pic)
 
@@ -591,13 +562,11 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): str
     for includeDir in items(conf.cIncludes):
       includeCmd.add(join([CC[c].includeCmd, includeDir.quoteShell]))
 
-    compilePattern = joinPath(conf.cCompilerPath, exe)
+    compilePattern = joinPath(conf.ccompilerpath, exe)
   else:
     includeCmd = ""
     compilePattern = getCompilerExe(conf, c, cfile.cname)
 
-  includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)]))
-
   var cf = if noAbsolutePaths(conf): AbsoluteFile extractFilename(cfile.cname.string)
            else: cfile.cname
 
@@ -606,7 +575,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): str
       if not cfile.flags.contains(CfileFlag.External) or noAbsolutePaths(conf):
         toObjFile(conf, cf).string
       else:
-        completeCfilePath(conf, toObjFile(conf, cf)).string
+        completeCFilePath(conf, toObjFile(conf, cf)).string
     elif noAbsolutePaths(conf):
       extractFilename(cfile.obj.string)
     else:
@@ -629,8 +598,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): str
     "file", cfsh, "objfile", objfile,
     "options", options, "include", includeCmd,
     "nim", quoteShell(getPrefixDir(conf)),
-    "lib", quoteShell(conf.libpath),
-    "vccplatform", vccplatform(conf)])
+    "lib", quoteShell(conf.libpath)])
 
 proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
   result = secureHash(
@@ -659,35 +627,31 @@ proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
       close(f)
 
 proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) =
-  if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and
-      not externalFileChanged(conf, c):
+  if optForceFullMake notin conf.globalOptions and not externalFileChanged(conf, c):
     c.flags.incl CfileFlag.Cached
   conf.toCompile.add(c)
 
 proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) =
-  var c = Cfile(nimname: splitFile(filename).name, cname: filename,
-    obj: toObjFile(conf, completeCfilePath(conf, filename, false)),
+  var c = Cfile(cname: filename,
+    obj: toObjFile(conf, completeCFilePath(conf, filename, false)),
     flags: {CfileFlag.External})
   addExternalFileToCompile(conf, c)
 
-proc compileCFiles(conf: ConfigRef; list: CfileList, script: var Rope, cmds: var TStringSeq,
+proc compileCFile(conf: ConfigRef; list: CFileList, script: var Rope, cmds: var TStringSeq,
                   prettyCmds: var TStringSeq) =
-  var currIdx = 0
   for it in list:
     # call the C compiler for the .c file:
     if it.flags.contains(CfileFlag.Cached): continue
-    var compileCmd = getCompileCFileCmd(conf, it, currIdx == list.len - 1)
-    inc currIdx
+    var compileCmd = getCompileCFileCmd(conf, it)
     if optCompileOnly notin conf.globalOptions:
       add(cmds, compileCmd)
       let (_, name, _) = splitFile(it.cname)
-      add(prettyCmds, if hintCC in conf.notes: "CC: " & demanglePackageName(name) else: "")
+      add(prettyCmds, if hintCC in conf.notes: "CC: " & name else: "")
     if optGenScript in conf.globalOptions:
       add(script, compileCmd)
       add(script, "\n")
 
-proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
-                objfiles: string, isDllBuild: bool): string =
+proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): string =
   if optGenStaticLib in conf.globalOptions:
     var libname: string
     if not conf.outFile.isEmpty:
@@ -697,28 +661,40 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
     else:
       libname = (libNameTmpl(conf) % splitFile(conf.projectName).name)
     result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(libname),
-                                            "objfiles", objfiles]
+                                       "objfiles", objfiles]
   else:
     var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe")
     if len(linkerExe) == 0: linkerExe = getLinkerExe(conf, conf.cCompiler)
     # bug #6452: We must not use ``quoteShell`` here for ``linkerExe``
     if needsExeExt(conf): linkerExe = addFileExt(linkerExe, "exe")
     if noAbsolutePaths(conf): result = linkerExe
-    else: result = joinPath(conf.cCompilerPath, linkerExe)
+    else: result = joinPath(conf.cCompilerpath, linkerExe)
     let buildgui = if optGenGuiApp in conf.globalOptions and conf.target.targetOS == osWindows:
                      CC[conf.cCompiler].buildGui
                    else:
                      ""
-    let builddll = if isDllBuild: CC[conf.cCompiler].buildDll else: ""
-    let exefile = quoteShell(output)
-
+    var exefile, builddll: string
+    if optGenDynLib in conf.globalOptions:
+      exefile = platform.OS[conf.target.targetOS].dllFrmt % splitFile(projectfile).name
+      builddll = CC[conf.cCompiler].buildDll
+    else:
+      exefile = splitFile(projectfile).name & platform.OS[conf.target.targetOS].exeExt
+      builddll = ""
+    if not conf.outFile.isEmpty:
+      exefile = conf.outFile.string.expandTilde
+      if not exefile.isAbsolute():
+        exefile = getCurrentDir() / exefile
+    if not noAbsolutePaths(conf):
+      if not exefile.isAbsolute():
+        exefile = string(splitFile(projectfile).dir / RelativeFile(exefile))
     when false:
       if optCDebug in conf.globalOptions:
         writeDebugInfo(exefile.changeFileExt("ndb"))
+    exefile = quoteShell(exefile)
 
     # Map files are required by Nintendo Switch compilation. They are a list
     # of all function calls in the library and where they come from.
-    let mapfile = quoteShell(getNimcacheDir(conf) / RelativeFile(splitFile(output).name & ".map"))
+    let mapfile = quoteShell(getNimcacheDir(conf) / RelativeFile(splitFile(projectFile).name & ".map"))
 
     let linkOptions = getLinkOptions(conf) & " " &
                       getConfigVar(conf, conf.cCompiler, ".options.linker")
@@ -735,53 +711,9 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
         "buildgui", buildgui, "options", linkOptions,
         "objfiles", objfiles, "exefile", exefile,
         "nim", quoteShell(getPrefixDir(conf)),
-        "lib", quoteShell(conf.libpath),
-        "vccplatform", vccplatform(conf)])
-    # On windows the debug information for binaries is emitted in a separate .pdb
-    # file and the binaries (.dll and .exe) contain a full path to that .pdb file.
-    # This is a problem for hot code reloading because even when we copy the .dll
-    # and load the copy so the build process may overwrite the original .dll on
-    # the disk (windows locks the files of running binaries) the copy still points
-    # to the original .pdb (and a simple copy of the .pdb won't help). This is a
-    # problem when a debugger is attached to the program we are hot-reloading.
-    # This problem is nonexistent on Unix since there by default debug symbols
-    # are embedded in the binaries so loading a copy of a .so will be fine. There
-    # is the '/Z7' flag for the MSVC compiler to embed the debug info of source
-    # files into their respective .obj files but the linker still produces a .pdb
-    # when a final .dll or .exe is linked so the debug info isn't embedded.
-    # There is also the issue that even when a .dll is unloaded the debugger
-    # still keeps the .pdb for that .dll locked. This is a major problem and
-    # because of this we cannot just alternate between 2 names for a .pdb file
-    # when rebuilding a .dll - instead we need to accumulate differently named
-    # .pdb files in the nimcache folder - this is the easiest and most reliable
-    # way of being able to debug and rebuild the program at the same time. This
-    # is accomplished using the /PDB:<filename> flag (there also exists the
-    # /PDBALTPATH:<filename> flag). The only downside is that the .pdb files are
-    # atleast 300kb big (when linking statically to the runtime - or else 5mb+)
-    # and will quickly accumulate. There is a hacky solution: we could try to
-    # delete all .pdb files with a pattern and swallow exceptions.
-    #
-    # links about .pdb files and hot code reloading:
-    # https://ourmachinery.com/post/dll-hot-reloading-in-theory-and-practice/
-    # https://ourmachinery.com/post/little-machines-working-together-part-2/
-    # https://github.com/fungos/cr
-    # https://fungos.github.io/blog/2017/11/20/cr.h-a-simple-c-hot-reload-header-only-library/
-    # on forcing the debugger to unlock a locked .pdb of an unloaded library:
-    # https://blog.molecular-matters.com/2017/05/09/deleting-pdb-files-locked-by-visual-studio/
-    # and a bit about the .pdb format in case that is ever needed:
-    # https://github.com/crosire/blink
-    # http://www.debuginfo.com/articles/debuginfomatch.html#pdbfiles
-    if conf.hcrOn and isVSCompatible(conf):
-      let t = now()
-      let pdb = output.string & "." & format(t, "MMMM-yyyy-HH-mm-") & $t.nanosecond & ".pdb"
-      result.add " /link /PDB:" & pdb
-  if optCDebug in conf.globalOptions and conf.cCompiler == ccVcc:
-    result.add " /Zi /FS /Od"
+        "lib", quoteShell(conf.libpath)])
 
-template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string): string =
-  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions)
-
-template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped) =
+template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped): typed =
   try:
     body
   except OSError:
@@ -808,7 +740,7 @@ proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx:
   if conf.numberOfProcessors == 0: conf.numberOfProcessors = countProcessors()
   var res = 0
   if conf.numberOfProcessors <= 1:
-    for i in 0 .. high(cmds):
+    for i in countup(0, high(cmds)):
       tryExceptOSErrorMessage(conf, "invocation of external compiler program failed."):
         res = execWithEcho(conf, cmds[i])
       if res != 0:
@@ -830,43 +762,7 @@ proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx:
       rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" %
         cmds.join())
 
-proc linkViaResponseFile(conf: ConfigRef; cmd: string) =
-  # Extracting the linker.exe here is a bit hacky but the best solution
-  # given ``buildLib``'s design.
-  var i = 0
-  var last = 0
-  if cmd.len > 0 and cmd[0] == '"':
-    inc i
-    while i < cmd.len and cmd[i] != '"': inc i
-    last = i
-    inc i
-  else:
-    while i < cmd.len and cmd[i] != ' ': inc i
-    last = i
-  while i < cmd.len and cmd[i] == ' ': inc i
-  let linkerArgs = conf.projectName & "_" & "linkerArgs.txt"
-  let args = cmd.substr(i)
-  # GCC's response files don't support backslashes. Junk.
-  if conf.cCompiler == ccGcc or conf.cCompiler == ccCLang:
-    writeFile(linkerArgs, args.replace('\\', '/'))
-  else:
-    writeFile(linkerArgs, args)
-  try:
-    execLinkCmd(conf, cmd.substr(0, last) & " @" & linkerArgs)
-  finally:
-    removeFile(linkerArgs)
-
-proc getObjFilePath(conf: ConfigRef, f: Cfile): string =
-  if noAbsolutePaths(conf): f.obj.extractFilename
-  else: f.obj.string
-
-proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): AbsoluteFile =
-  let basename = splitFile(objFile).name
-  let targetName = if isMain: basename & ".exe"
-                   else: platform.OS[conf.target.targetOS].dllFrmt % basename
-  result = conf.getNimcacheDir / RelativeFile(targetName)
-
-proc callCCompiler*(conf: ConfigRef) =
+proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) =
   var
     linkCmd: string
   if conf.globalOptions * {optCompileOnly, optGenScript} == {optCompileOnly}:
@@ -880,7 +776,7 @@ proc callCCompiler*(conf: ConfigRef) =
     when declared(echo):
       let cmd = prettyCmds[idx]
       if cmd != "": echo cmd
-  compileCFiles(conf, conf.toCompile, script, cmds, prettyCmds)
+  compileCFile(conf, conf.toCompile, script, cmds, prettyCmds)
   if optCompileOnly notin conf.globalOptions:
     execCmdsInParallel(conf, cmds, prettyCb)
   if optNoLinking notin conf.globalOptions:
@@ -891,65 +787,25 @@ proc callCCompiler*(conf: ConfigRef) =
       add(objfiles, ' ')
       add(objfiles, quoteShell(
           addFileExt(objFile, CC[conf.cCompiler].objExt)))
+    for x in conf.toCompile:
+      let objFile = if noAbsolutePaths(conf): x.obj.extractFilename else: x.obj.string
+      add(objfiles, ' ')
+      add(objfiles, quoteShell(objFile))
 
-    if conf.hcrOn: # lets assume that optCompileOnly isn't on
-      cmds = @[]
-      let mainFileIdx = conf.toCompile.len - 1
-      for idx, x in conf.toCompile:
-        # don't relink each of the many binaries (one for each source file) if the nim code is
-        # cached because that would take too much time for small changes - the only downside to
-        # this is that if an external-to-link file changes the final target wouldn't be relinked
-        if x.flags.contains(CfileFlag.Cached): continue
-        # we pass each object file as if it is the project file - a .dll will be created for each such
-        # object file in the nimcache directory, and only in the case of the main project file will
-        # there be probably an executable (if the project is such) which will be copied out of the nimcache
-        let objFile = conf.getObjFilePath(x)
-        let buildDll = idx != mainFileIdx
-        let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll)
-        add(cmds, getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll))
-        # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache
-        # for more info check the comment inside of getLinkCmd() where the /PDB:<filename> MSVC flag is used
-        if isVSCompatible(conf):
-          for pdb in walkFiles(objFile & ".*.pdb"):
-            discard tryRemoveFile(pdb)
-      # execute link commands in parallel - output will be a bit different
-      # if it fails than that from execLinkCmd() but that doesn't matter
-      prettyCmds = map(prettyCmds, proc (curr: string): string = return curr.replace("CC", "Link"))
-      execCmdsInParallel(conf, cmds, prettyCb)
-      # only if not cached - copy the resulting main file from the nimcache folder to its originally intended destination
-      if not conf.toCompile[mainFileIdx].flags.contains(CfileFlag.Cached):
-        let mainObjFile = getObjFilePath(conf, conf.toCompile[mainFileIdx])
-        var src = conf.hcrLinkTargetName(mainObjFile, true)
-        var dst = conf.prepareToWriteOutput
-        copyFileWithPermissions(src.string, dst.string)
-    else:
-      for x in conf.toCompile:
-        let objFile = if noAbsolutePaths(conf): x.obj.extractFilename else: x.obj.string
-        add(objfiles, ' ')
-        add(objfiles, quoteShell(objFile))
-      let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput
-                       else: AbsoluteFile(conf.projectName)
-      linkCmd = getLinkCmd(conf, mainOutput, objfiles)
-      if optCompileOnly notin conf.globalOptions:
-        const MaxCmdLen = when defined(windows): 8_000 else: 32_000
-        if linkCmd.len > MaxCmdLen:
-          # Windows's command line limit is about 8K (don't laugh...) so C compilers on
-          # Windows support a feature where the command line can be passed via ``@linkcmd``
-          # to them.
-          linkViaResponseFile(conf, linkCmd)
-        else:
-          execLinkCmd(conf, linkCmd)
+    linkCmd = getLinkCmd(conf, projectfile, objfiles)
+    if optCompileOnly notin conf.globalOptions:
+      execLinkCmd(conf, linkCmd)
   else:
     linkCmd = ""
   if optGenScript in conf.globalOptions:
     add(script, linkCmd)
     add(script, "\n")
-    generateScript(conf, script)
+    generateScript(conf, projectfile, script)
 
 #from json import escapeJson
 import json
 
-proc writeJsonBuildInstructions*(conf: ConfigRef) =
+proc writeJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) =
   template lit(x: untyped) = f.write x
   template str(x: untyped) =
     when compiles(escapeJson(x, buf)):
@@ -996,7 +852,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
 
   var buf = newStringOfCap(50)
 
-  let jsonFile = conf.getNimcacheDir / RelativeFile(conf.projectName & ".json")
+  let jsonFile = toGeneratedFile(conf, projectfile, "json")
 
   var f: File
   if open(f, jsonFile.string, fmWrite):
@@ -1008,7 +864,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
     linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink)
 
     lit "],\L\"linkcmd\": "
-    str getLinkCmd(conf, conf.absOutFile, objfiles)
+    str getLinkCmd(conf, projectfile, objfiles)
     lit "\L}\L"
     close(f)
 
@@ -1041,7 +897,7 @@ proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) =
       echo getCurrentException().getStackTrace()
     quit "error evaluating JSON file: " & jsonFile.string
 
-proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope =
+proc genMappingFiles(conf: ConfigRef; list: CFileList): Rope =
   for it in list:
     addf(result, "--file:r\"$1\"$N", [rope(it.cname.string)])
 

compiler/filter_tmpl.nim

@@ -10,8 +10,8 @@
 # This module implements Nim's standard template filter.
 
 import
-  llstream, strutils, ast, msgs, options,
-  filters, lineinfos, pathutils
+  llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
+  renderer, filters, lineinfos, pathutils
 
 type
   TParseState = enum

compiler/filters.nim

@@ -10,7 +10,7 @@
 # This module implements Nim's simple filters and helpers for filters.
 
 import
-  llstream, idents, strutils, ast, msgs, options,
+  llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
   renderer, pathutils
 
 proc invalidPragma(conf: ConfigRef; n: PNode) =
@@ -20,7 +20,7 @@ proc invalidPragma(conf: ConfigRef; n: PNode) =
 proc getArg(conf: ConfigRef; n: PNode, name: string, pos: int): PNode =
   result = nil
   if n.kind in {nkEmpty..nkNilLit}: return
-  for i in 1 ..< sonsLen(n):
+  for i in countup(1, sonsLen(n) - 1):
     if n.sons[i].kind == nkExprEqExpr:
       if n.sons[i].sons[0].kind != nkIdent: invalidPragma(conf, n)
       if cmpIgnoreStyle(n.sons[i].sons[0].ident.s, name) == 0:

compiler/gorgeimpl.nim

@@ -9,7 +9,7 @@
 
 ## Module that implements ``gorge`` for the compiler.
 
-import msgs, std / sha1, os, osproc, streams, options,
+import msgs, std / sha1, os, osproc, streams, strutils, options,
   lineinfos, pathutils
 
 proc readOutput(p: Process): (string, int) =
@@ -35,7 +35,7 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
     var readSuccessful = false
     try:
       var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStdErrToStdOut})
+                           options={poEvalCommand, poStderrToStdout})
       if input.len != 0:
         p.inputStream.write(input)
         p.inputStream.close()
@@ -49,7 +49,7 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
   else:
     try:
       var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStdErrToStdOut})
+                           options={poEvalCommand, poStderrToStdout})
       if input.len != 0:
         p.inputStream.write(input)
         p.inputStream.close()

compiler/guards.nim

@@ -70,7 +70,7 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
       n = n.sons[0]
       inc derefs
     of nkBracketExpr:
-      if isConstExpr(n.sons[1]) or isLet(n.sons[1]) or isConstExpr(n.sons[1].skipConv):
+      if isConstExpr(n.sons[1]) or isLet(n.sons[1]):
         n = n.sons[0]
       else: return
     of nkHiddenStdConv, nkHiddenSubConv, nkConv:
@@ -257,9 +257,9 @@ proc canon*(n: PNode; o: Operators): PNode =
     for i in 0 ..< n.len:
       result.sons[i] = canon(n.sons[i], o)
   elif n.kind == nkSym and n.sym.kind == skLet and
-      n.sym.astdef.getMagic in (someEq + someAdd + someMul + someMin +
+      n.sym.ast.getMagic in (someEq + someAdd + someMul + someMin +
       someMax + someHigh + {mUnaryLt} + someSub + someLen + someDiv):
-    result = n.sym.astdef.copyTree
+    result = n.sym.ast.copyTree
   else:
     result = n
   case result.getMagic
@@ -395,8 +395,8 @@ proc usefulFact(n: PNode; o: Operators): PNode =
     #   if a:
     #     ...
     # We make can easily replace 'a' by '2 < x' here:
-    if n.sym.astdef != nil:
-      result = usefulFact(n.sym.astdef, o)
+    if n.sym.ast != nil:
+      result = usefulFact(n.sym.ast, o)
   elif n.kind == nkStmtListExpr:
     result = usefulFact(n.lastSon, o)
 
@@ -443,7 +443,7 @@ proc sameTree*(a, b: PNode): bool =
     of nkEmpty, nkNilLit: result = true
     else:
       if sonsLen(a) == sonsLen(b):
-        for i in 0 ..< sonsLen(a):
+        for i in countup(0, sonsLen(a) - 1):
           if not sameTree(a.sons[i], b.sons[i]): return
         result = true
 
@@ -981,10 +981,9 @@ proc buildElse(n: PNode; o: Operators): PNode =
   var s = newNodeIT(nkCurly, n.info, settype(n.sons[0]))
   for i in 1..n.len-2:
     let branch = n.sons[i]
-    assert branch.kind != nkElse
-    if branch.kind == nkOfBranch:
-      for j in 0..branch.len-2:
-        s.add(branch.sons[j])
+    assert branch.kind == nkOfBranch
+    for j in 0..branch.len-2:
+      s.add(branch.sons[j])
   result = newNodeI(nkCall, n.info, 3)
   result.sons[0] = newSymNode(o.opContains)
   result.sons[1] = s

compiler/hlo.nim

@@ -92,12 +92,12 @@ proc hlo(c: PContext, n: PNode): PNode =
 
 proc hloBody(c: PContext, n: PNode): PNode =
   # fast exit:
-  if c.patterns.len == 0 or optTrMacros notin c.config.options: return n
+  if c.patterns.len == 0 or optPatterns notin c.config.options: return n
   c.hloLoopDetector = 0
   result = hlo(c, n)
 
 proc hloStmt(c: PContext, n: PNode): PNode =
   # fast exit:
-  if c.patterns.len == 0 or optTrMacros notin c.config.options: return n
+  if c.patterns.len == 0 or optPatterns notin c.config.options: return n
   c.hloLoopDetector = 0
   result = hlo(c, n)

compiler/idents.nim

@@ -12,7 +12,7 @@
 # id. This module is essential for the compiler's performance.
 
 import
-  hashes, wordrecg
+  hashes, strutils, wordrecg
 
 type
   TIdObj* = object of RootObj
@@ -87,7 +87,7 @@ proc getIdent*(ic: IdentCache; identifier: cstring, length: int, h: Hash): PIden
   new(result)
   result.h = h
   result.s = newString(length)
-  for i in 0 ..< length: result.s[i] = identifier[i]
+  for i in countup(0, length - 1): result.s[i] = identifier[i]
   result.next = ic.buckets[idx]
   ic.buckets[idx] = result
   if id == 0:
@@ -110,7 +110,7 @@ proc newIdentCache*(): IdentCache =
   result.idDelegator = result.getIdent":delegator"
   result.emptyIdent = result.getIdent("")
   # initialize the keywords:
-  for s in succ(low(specialWords)) .. high(specialWords):
+  for s in countup(succ(low(specialWords)), high(specialWords)):
     result.getIdent(specialWords[s], hashIgnoreStyle(specialWords[s])).id = ord(s)
 
 proc whichKeyword*(id: PIdent): TSpecialWord =

compiler/idgen.nim

@@ -9,7 +9,7 @@
 
 ## This module contains a simple persistent id generator.
 
-import idents, strutils, options, pathutils
+import idents, strutils, os, options, pathutils
 
 var gFrontEndId*: int
 

compiler/importer.nim

@@ -10,8 +10,8 @@
 ## This module implements the symbol importing mechanism.
 
 import
-  intsets, ast, astalgo, msgs, options, idents, lookups,
-  semdata, modulepaths, sigmatch, lineinfos
+  intsets, strutils, os, ast, astalgo, msgs, options, idents, lookups,
+  semdata, passes, renderer, modulepaths, sigmatch, lineinfos
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
@@ -47,7 +47,7 @@ proc rawImportSymbol(c: PContext, s: PSym) =
   if s.kind == skType:
     var etyp = s.typ
     if etyp.kind in {tyBool, tyEnum}:
-      for j in 0 ..< sonsLen(etyp.n):
+      for j in countup(0, sonsLen(etyp.n) - 1):
         var e = etyp.n.sons[j].sym
         if e.kind != skEnumField:
           internalError(c.config, s.info, "rawImportSymbol")
@@ -79,19 +79,19 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
   else:
     when false:
       if s.kind == skStub: loadStub(s)
-    let multiImport = s.kind notin ExportableSymKinds or s.kind in skProcKinds
+    if s.kind notin ExportableSymKinds:
+      internalError(c.config, n.info, "importSymbol: 2")
     # for an enumeration we have to add all identifiers
-    if multiImport:
+    case s.kind
+    of skProcKinds:
       # for a overloadable syms add all overloaded routines
       var it: TIdentIter
       var e = initIdentIter(it, fromMod.tab, s.name)
       while e != nil:
         if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3")
-        if s.kind in ExportableSymKinds:
-          rawImportSymbol(c, e)
+        rawImportSymbol(c, e)
         e = nextIdentIter(it, fromMod.tab)
-    else:
-      rawImportSymbol(c, s)
+    else: rawImportSymbol(c, s)
     suggestSym(c.config, n.info, s, c.graph.usageSym, false)
 
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
@@ -101,7 +101,7 @@ proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
     if s.kind != skModule:
       if s.kind != skEnumField:
         if s.kind notin ExportableSymKinds:
-          internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s)
+          internalError(c.config, s.info, "importAllSymbols: " & $s.kind)
         if exceptSet.isNil or s.name.id notin exceptSet:
           rawImportSymbol(c, s)
     s = nextIter(i, fromMod.tab)
@@ -139,14 +139,14 @@ proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
 
 proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
   let f = checkModuleName(c.config, n)
-  if f != InvalidFileIdx:
+  if f != InvalidFileIDX:
     let L = c.graph.importStack.len
     let recursion = c.graph.importStack.find(f)
     c.graph.importStack.add f
     #echo "adding ", toFullPath(f), " at ", L+1
     if recursion >= 0:
       var err = ""
-      for i in recursion ..< L:
+      for i in countup(recursion, L-1):
         if i > recursion: err.add "\n"
         err.add toFullPath(c.config, c.graph.importStack[i]) & " imports " &
                 toFullPath(c.config, c.graph.importStack[i+1])
@@ -162,9 +162,9 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
         localError(c.config, n.info, "A module cannot import itself")
     if sfDeprecated in result.flags:
       if result.constraint != nil:
-        message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated")
+        message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s)
       else:
-        message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated")
+        message(c.config, n.info, warnDeprecated, result.name.s)
     suggestSym(c.config, n.info, result, c.graph.usageSym, false)
     importStmtResult.add newSymNode(result, n.info)
     #newStrNode(toFullPath(c.config, f), n.info)
@@ -189,7 +189,7 @@ proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
 
 proc evalImport*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
-  for i in 0 ..< sonsLen(n):
+  for i in countup(0, sonsLen(n) - 1):
     let it = n.sons[i]
     if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
       let sep = it[0]
@@ -219,7 +219,7 @@ proc evalFrom*(c: PContext, n: PNode): PNode =
   if m != nil:
     n.sons[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
-    for i in 1 ..< sonsLen(n):
+    for i in countup(1, sonsLen(n) - 1):
       if n.sons[i].kind != nkNilLit:
         importSymbol(c, n.sons[i], m)
 

compiler/incremental.nim

@@ -10,14 +10,13 @@
 ## Basic type definitions the module graph needs in order to support
 ## incremental compilations.
 
-const nimIncremental* = true # defined(nimIncremental)
+const nimIncremental* = defined(nimIncremental)
 
-import options, lineinfos
+import options, lineinfos, pathutils
 
 when nimIncremental:
-  import ast, msgs, intsets, btrees, db_sqlite, std / sha1, pathutils
+  import ast, msgs, intsets, btrees, db_sqlite, std / sha1
   from strutils import parseInt
-  from os import isAbsolute
 
   type
     Writer* = object
@@ -48,7 +47,7 @@ when nimIncremental:
 
   proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: AbsoluteFile): string =
     result = msgs.getHash(conf, fileIdx)
-    if result.len == 0 and isAbsolute(string fullpath):
+    if result.len == 0:
       result = $secureHashFile(string fullpath)
       msgs.setHash(conf, fileIdx, result)
 
@@ -60,8 +59,8 @@ when nimIncremental:
     let id = row[0]
     let fullhash = hashFileCached(conf, fileIdx, AbsoluteFile fullpath)
     if id.len == 0:
-      result = int incr.db.insertID(sql"insert into filenames(nimid, fullpath, fullhash) values (?, ?, ?)",
-        int(fileIdx), fullpath, fullhash)
+      result = int incr.db.insertID(sql"insert into filenames(fullpath, fullhash) values (?, ?)",
+        fullpath, fullhash)
     else:
       if row[1] != fullhash:
         incr.db.exec(sql"update filenames set fullhash = ? where fullpath = ?", fullhash, fullpath)
@@ -103,7 +102,6 @@ when nimIncremental:
     db.exec(sql"""
       create table if not exists filenames(
         id integer primary key,
-        nimid integer not null,
         fullpath varchar(8000) not null,
         fullHash varchar(256) not null
       );

compiler/injectdestructors.nim

@@ -1,907 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2017 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Injects destructor calls into Nim code as well as
-## an optimizer that optimizes copies to moves. This is implemented as an
-## AST to AST transformation so that every backend benefits from it.
-
-## Rules for destructor injections:
-##
-## foo(bar(X(), Y()))
-## X and Y get destroyed after bar completes:
-##
-## foo( (tmpX = X(); tmpY = Y(); tmpBar = bar(tmpX, tmpY);
-##       destroy(tmpX); destroy(tmpY);
-##       tmpBar))
-## destroy(tmpBar)
-##
-## var x = f()
-## body
-##
-## is the same as:
-##
-##  var x;
-##  try:
-##    move(x, f())
-##  finally:
-##    destroy(x)
-##
-## But this really just an optimization that tries to avoid to
-## introduce too many temporaries, the 'destroy' is caused by
-## the 'f()' call. No! That is not true for 'result = f()'!
-##
-## x = y where y is read only once
-## is the same as:  move(x, y)
-##
-## Actually the more general rule is: The *last* read of ``y``
-## can become a move if ``y`` is the result of a construction.
-##
-## We also need to keep in mind here that the number of reads is
-## control flow dependent:
-## let x = foo()
-## while true:
-##   y = x  # only one read, but the 2nd iteration will fail!
-## This also affects recursions! Only usages that do not cross
-## a loop boundary (scope) and are not used in function calls
-## are safe.
-##
-##
-## x = f() is the same as:  move(x, f())
-##
-## x = y
-## is the same as:  copy(x, y)
-##
-## Reassignment works under this scheme:
-## var x = f()
-## x = y
-##
-## is the same as:
-##
-##  var x;
-##  try:
-##    move(x, f())
-##    copy(x, y)
-##  finally:
-##    destroy(x)
-##
-##  result = f()  must not destroy 'result'!
-##
-## The produced temporaries clutter up the code and might lead to
-## inefficiencies. A better strategy is to collect all the temporaries
-## in a single object that we put into a single try-finally that
-## surrounds the proc body. This means the code stays quite efficient
-## when compiled to C. In fact, we do the same for variables, so
-## destructors are called when the proc returns, not at scope exit!
-## This makes certains idioms easier to support. (Taking the slice
-## of a temporary object.)
-##
-## foo(bar(X(), Y()))
-## X and Y get destroyed after bar completes:
-##
-## var tmp: object
-## foo( (move tmp.x, X(); move tmp.y, Y(); tmp.bar = bar(tmpX, tmpY);
-##       tmp.bar))
-## destroy(tmp.bar)
-## destroy(tmp.x); destroy(tmp.y)
-##
-
-#[
-From https://github.com/nim-lang/Nim/wiki/Destructors
-
-Rule      Pattern                 Transformed into
-----      -------                 ----------------
-1.1	      var x: T; stmts	        var x: T; try stmts
-                                  finally: `=destroy`(x)
-2         x = f()                 `=sink`(x, f())
-3         x = lastReadOf z        `=sink`(x, z); wasMoved(z)
-3.2       x = path z; body        ``x = bitwiseCopy(path z);``
-                                  do not emit `=destroy(x)`. Note: body
-                                  must not mutate ``z`` nor ``x``. All
-                                  assignments to ``x`` must be of the form
-                                  ``path z`` but the ``z`` can differ.
-                                  Neither ``z`` nor ``x`` can have the
-                                  flag ``sfAddrTaken`` to ensure no other
-                                  aliasing is going on.
-4.1       y = sinkParam           `=sink`(y, sinkParam)
-4.2       x = y                   `=`(x, y) # a copy
-5.1       f_sink(g())             f_sink(g())
-5.2       f_sink(y)               f_sink(copy y); # copy unless we can see it's the last read
-5.3       f_sink(move y)          f_sink(y); wasMoved(y) # explicit moves empties 'y'
-5.4       f_noSink(g())           var tmp = bitwiseCopy(g()); f(tmp); `=destroy`(tmp)
-
-Rule 3.2 describes a "cursor" variable, a variable that is only used as a
-view into some data structure. See ``compiler/cursors.nim`` for details.
-
-Note: In order to avoid the very common combination ``reset(x); =sink(x, y)`` for
-variable definitions we must turn "the first sink/assignment" operation into a
-copyMem. This is harder than it looks:
-
-  while true:
-    try:
-      if cond: break # problem if we run destroy(x) here :-/
-      var x = f()
-    finally:
-      destroy(x)
-
-And the C++ optimizers don't sweat to optimize it for us, so we don't have
-to do it.
-]#
-
-import
-  intsets, ast, msgs, renderer, magicsys, types, idents,
-  strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
-  lineinfos, parampatterns, sighashes
-
-const
-  InterestingSyms = {skVar, skResult, skLet, skForVar, skTemp}
-
-type
-  Con = object
-    owner: PSym
-    g: ControlFlowGraph
-    jumpTargets: IntSet
-    destroys, topLevelVars: PNode
-    graph: ModuleGraph
-    emptyNode: PNode
-    otherRead: PNode
-    inLoop: int
-    uninit: IntSet # set of uninit'ed vars
-    uninitComputed: bool
-
-const toDebug = "" # "server" # "serverNimAsyncContinue"
-
-template dbg(body) =
-  when toDebug.len > 0:
-    if c.owner.name.s == toDebug or toDebug == "always":
-      body
-
-proc isLastRead(location: PNode; c: var Con; pc, comesFrom: int): int =
-  var pc = pc
-  while pc < c.g.len:
-    case c.g[pc].kind
-    of def:
-      if defInstrTargets(c.g[pc], location):
-        # the path lead to a redefinition of 's' --> abandon it.
-        return high(int)
-      inc pc
-    of use:
-      if useInstrTargets(c.g[pc], location):
-        c.otherRead = c.g[pc].n
-        return -1
-      inc pc
-    of goto:
-      pc = pc + c.g[pc].dest
-    of fork:
-      # every branch must lead to the last read of the location:
-      let variantA = isLastRead(location, c, pc+1, pc)
-      if variantA < 0: return -1
-      var variantB = isLastRead(location, c, pc + c.g[pc].dest, pc)
-      if variantB < 0: return -1
-      elif variantB == high(int):
-        variantB = variantA
-      pc = variantB
-    of InstrKind.join:
-      let dest = pc + c.g[pc].dest
-      if dest == comesFrom: return pc + 1
-      inc pc
-  return pc
-
-proc isLastRead(n: PNode; c: var Con): bool =
-  # first we need to search for the instruction that belongs to 'n':
-  c.otherRead = nil
-  var instr = -1
-  let m = dfa.skipConvDfa(n)
-
-  for i in 0..<c.g.len:
-    # This comparison is correct and MUST not be ``instrTargets``:
-    if c.g[i].kind == use and c.g[i].n == m:
-      if instr < 0:
-        instr = i
-        break
-
-  dbg:
-    echo "starting point for ", n, " is ", instr, " ", n.kind
-
-  if instr < 0: return false
-  # we go through all paths beginning from 'instr+1' and need to
-  # ensure that we don't find another 'use X' instruction.
-  if instr+1 >= c.g.len: return true
-
-  result = isLastRead(n, c, instr+1, -1) >= 0
-  dbg:
-    echo "ugh ", c.otherRead.isNil, " ", result
-
-  when false:
-    let s = n.sym
-    var pcs: seq[int] = @[instr+1]
-    var takenGotos: IntSet
-    var takenForks = initIntSet()
-    while pcs.len > 0:
-      var pc = pcs.pop
-
-      takenGotos = initIntSet()
-      while pc < c.g.len:
-        case c.g[pc].kind
-        of def:
-          if c.g[pc].sym == s:
-            # the path lead to a redefinition of 's' --> abandon it.
-            break
-          inc pc
-        of use:
-          if c.g[pc].sym == s:
-            c.otherRead = c.g[pc].n
-            return false
-          inc pc
-        of goto:
-          # we must leave endless loops eventually:
-          if not takenGotos.containsOrIncl(pc):
-            pc = pc + c.g[pc].dest
-          else:
-            inc pc
-        of fork:
-          # we follow the next instruction but push the dest onto our "work" stack:
-          if not takenForks.containsOrIncl(pc):
-            pcs.add pc + c.g[pc].dest
-          inc pc
-        of InstrKind.join:
-          inc pc
-    #echo c.graph.config $ n.info, " last read here!"
-    return true
-
-proc initialized(code: ControlFlowGraph; pc: int,
-                 init, uninit: var IntSet; comesFrom: int): int =
-  ## Computes the set of definitely initialized variables accross all code paths
-  ## as an IntSet of IDs.
-  var pc = pc
-  while pc < code.len:
-    case code[pc].kind
-    of goto:
-      pc = pc + code[pc].dest
-    of fork:
-      let target = pc + code[pc].dest
-      var initA = initIntSet()
-      var initB = initIntSet()
-      let pcA = initialized(code, pc+1, initA, uninit, pc)
-      discard initialized(code, target, initB, uninit, pc)
-      # we add vars if they are in both branches:
-      for v in initA:
-        if v in initB:
-          init.incl v
-      pc = pcA+1
-    of InstrKind.join:
-      let target = pc + code[pc].dest
-      if comesFrom == target: return pc
-      inc pc
-    of use:
-      let v = code[pc].sym
-      if v.kind != skParam and v.id notin init:
-        # attempt to read an uninit'ed variable
-        uninit.incl v.id
-      inc pc
-    of def:
-      let v = code[pc].sym
-      init.incl v.id
-      inc pc
-  return pc
-
-template interestingSym(s: PSym): bool =
-  s.owner == c.owner and s.kind in InterestingSyms and hasDestructor(s.typ)
-
-template isUnpackedTuple(s: PSym): bool =
-  ## we move out all elements of unpacked tuples,
-  ## hence unpacked tuples themselves don't need to be destroyed
-  s.kind == skTemp and s.typ.kind == tyTuple
-
-proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
-  var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">"
-  if opname == "=" and ri != nil:
-    m.add "; requires a copy because it's not the last read of '"
-    m.add renderTree(ri)
-    m.add '\''
-    if c.otherRead != nil:
-      m.add "; another read is done here: "
-      m.add c.graph.config $ c.otherRead.info
-    elif ri.kind == nkSym and ri.sym.kind == skParam and not isSinkType(ri.sym.typ):
-      m.add "; try to make "
-      m.add renderTree(ri)
-      m.add " a 'sink' parameter"
-  m.add "; routine: "
-  m.add c.owner.name.s
-  localError(c.graph.config, ri.info, errGenerated, m)
-
-proc makePtrType(c: Con, baseType: PType): PType =
-  result = newType(tyPtr, c.owner)
-  addSonSkipIntLit(result, baseType)
-
-proc genOp(c: Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode =
-  var op = t.attachedOps[kind]
-
-  if op == nil:
-    # give up and find the canonical type instead:
-    let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
-    let canon = c.graph.canonTypes.getOrDefault(h)
-    if canon != nil:
-      op = canon.attachedOps[kind]
-
-  if op == nil:
-    globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &
-      "' operator not found for type " & typeToString(t))
-  elif op.ast[genericParamsPos].kind != nkEmpty:
-    globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &
-      "' operator is generic")
-  if sfError in op.flags: checkForErrorPragma(c, t, ri, AttachedOpToStr[kind])
-  let addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
-  addrExp.add(dest)
-  result = newTree(nkCall, newSymNode(op), addrExp)
-
-when false:
-  proc preventMoveRef(dest, ri: PNode): bool =
-    let lhs = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink})
-    var ri = ri
-    if ri.kind in nkCallKinds and ri[0].kind == nkSym and ri[0].sym.magic == mUnown:
-      ri = ri[1]
-    let rhs = ri.typ.skipTypes({tyGenericInst, tyAlias, tySink})
-    result = lhs.kind == tyRef and rhs.kind == tyOwned
-
-proc canBeMoved(t: PType): bool {.inline.} =
-  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  result = t.kind != tyRef and t.attachedOps[attachedSink] != nil
-
-proc genSink(c: Con; t: PType; dest, ri: PNode): PNode =
-  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  let k = if t.attachedOps[attachedSink] != nil: attachedSink
-          else: attachedAsgn
-  if t.attachedOps[k] != nil:
-    result = genOp(c, t, k, dest, ri)
-  else:
-    # in rare cases only =destroy exists but no sink or assignment
-    # (see Pony object in tmove_objconstr.nim)
-    # we generate a fast assignment in this case:
-    result = newTree(nkFastAsgn, dest)
-
-proc genCopy(c: var Con; t: PType; dest, ri: PNode): PNode =
-  if tfHasOwned in t.flags:
-    # try to improve the error message here:
-    if c.otherRead == nil: discard isLastRead(ri, c)
-    checkForErrorPragma(c, t, ri, "=")
-  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  result = genOp(c, t, attachedAsgn, dest, ri)
-
-proc genCopyNoCheck(c: Con; t: PType; dest, ri: PNode): PNode =
-  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  result = genOp(c, t, attachedAsgn, dest, ri)
-
-proc genDestroy(c: Con; t: PType; dest: PNode): PNode =
-  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
-  result = genOp(c, t, attachedDestructor, dest, nil)
-
-proc addTopVar(c: var Con; v: PNode) =
-  c.topLevelVars.add newTree(nkIdentDefs, v, c.emptyNode, c.emptyNode)
-
-proc getTemp(c: var Con; typ: PType; info: TLineInfo): PNode =
-  let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), c.owner, info)
-  sym.typ = typ
-  result = newSymNode(sym)
-  c.addTopVar(result)
-
-proc p(n: PNode; c: var Con): PNode
-
-template recurse(n, dest) =
-  for i in 0..<n.len:
-    dest.add p(n[i], c)
-
-proc genMagicCall(n: PNode; c: var Con; magicname: string; m: TMagic): PNode =
-  result = newNodeI(nkCall, n.info)
-  result.add(newSymNode(createMagic(c.graph, magicname, m)))
-  result.add n
-
-proc genWasMoved(n: PNode; c: var Con): PNode =
-  # The mWasMoved builtin does not take the address.
-  result = genMagicCall(n, c, "wasMoved", mWasMoved)
-
-proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode =
-  result = newNodeI(nkCall, info)
-  result.add(newSymNode(createMagic(c.graph, "default", mDefault)))
-  result.typ = t
-
-proc destructiveMoveVar(n: PNode; c: var Con): PNode =
-  # generate: (let tmp = v; reset(v); tmp)
-  # XXX: Strictly speaking we can only move if there is a ``=sink`` defined
-  # or if no ``=sink`` is defined and also no assignment.
-  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
-
-  var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), c.owner, n.info)
-  temp.typ = n.typ
-  var v = newNodeI(nkLetSection, n.info)
-  let tempAsNode = newSymNode(temp)
-
-  var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
-  vpart.sons[0] = tempAsNode
-  vpart.sons[1] = c.emptyNode
-  vpart.sons[2] = n
-  add(v, vpart)
-
-  result.add v
-  result.add genWasMoved(skipConv(n), c)
-  result.add tempAsNode
-
-proc sinkParamIsLastReadCheck(c: var Con, s: PNode) =
-  assert s.kind == nkSym and s.sym.kind == skParam
-  if not isLastRead(s, c):
-     localError(c.graph.config, c.otherRead.info, "sink parameter `" & $s.sym.name.s &
-         "` is already consumed at " & toFileLineCol(c. graph.config, s.info))
-
-proc passCopyToSink(n: PNode; c: var Con): PNode =
-  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
-  let tmp = getTemp(c, n.typ, n.info)
-  # XXX This is only required if we are in a loop. Since we move temporaries
-  # out of loops we need to mark it as 'wasMoved'.
-  result.add genWasMoved(tmp, c)
-  if hasDestructor(n.typ):
-    var m = genCopy(c, n.typ, tmp, n)
-    m.add p(n, c)
-    result.add m
-    if isLValue(n):
-      message(c.graph.config, n.info, hintPerformance,
-        ("passing '$1' to a sink parameter introduces an implicit copy; " &
-        "use 'move($1)' to prevent it") % $n)
-  else:
-    result.add newTree(nkAsgn, tmp, p(n, c))
-  result.add tmp
-
-proc isDangerousSeq(t: PType): bool {.inline.} =
-  let t = t.skipTypes(abstractInst)
-  result = t.kind == tySequence and tfHasOwned notin t.sons[0].flags
-
-proc containsConstSeq(n: PNode): bool =
-  if n.kind == nkBracket and n.len > 0 and n.typ != nil and isDangerousSeq(n.typ):
-    return true
-  result = false
-  case n.kind
-  of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv:
-    result = containsConstSeq(n[1])
-  of nkObjConstr, nkClosure:
-    for i in 1 ..< n.len:
-      if containsConstSeq(n[i]): return true
-  of nkCurly, nkBracket, nkPar, nkTupleConstr:
-    for i in 0 ..< n.len:
-      if containsConstSeq(n[i]): return true
-  else: discard
-
-proc pArg(arg: PNode; c: var Con; isSink: bool): PNode =
-  template pArgIfTyped(argPart: PNode): PNode =
-    # typ is nil if we are in if/case expr branch with noreturn
-    if argPart.typ == nil: p(argPart, c)
-    else: pArg(argPart, c, isSink)
-
-  if isSink:
-    if arg.kind in nkCallKinds:
-      # recurse but skip the call expression in order to prevent
-      # destructor injections: Rule 5.1 is different from rule 5.4!
-      result = copyNode(arg)
-      let parameters = arg[0].typ
-      let L = if parameters != nil: parameters.len else: 0
-      result.add arg[0]
-      for i in 1..<arg.len:
-        result.add pArg(arg[i], c, i < L and isSinkTypeForParam(parameters[i]))
-    elif arg.containsConstSeq:
-      # const sequences are not mutable and so we need to pass a copy to the
-      # sink parameter (bug #11524). Note that the string implemenation is
-      # different and can deal with 'const string sunk into var'.
-      result = passCopyToSink(arg, c)
-    elif arg.kind in {nkBracket, nkObjConstr, nkTupleConstr, nkCharLit..nkTripleStrLit}:
-      discard "object construction to sink parameter: nothing to do"
-      result = arg
-    elif arg.kind == nkSym and isSinkParam(arg.sym):
-      # Sinked params can be consumed only once. We need to reset the memory
-      # to disable the destructor which we have not elided
-      sinkParamIsLastReadCheck(c, arg)
-      result = destructiveMoveVar(arg, c)
-    elif isAnalysableFieldAccess(arg, c.owner) and isLastRead(arg, c):
-      # it is the last read, can be sinked. We need to reset the memory
-      # to disable the destructor which we have not elided
-      result = destructiveMoveVar(arg, c)
-    elif arg.kind in {nkBlockExpr, nkBlockStmt}:
-      result = copyNode(arg)
-      result.add arg[0]
-      result.add pArg(arg[1], c, isSink)
-    elif arg.kind == nkStmtListExpr:
-      result = copyNode(arg)
-      for i in 0..arg.len-2:
-        result.add p(arg[i], c)
-      result.add pArg(arg[^1], c, isSink)
-    elif arg.kind in {nkIfExpr, nkIfStmt}:
-      result = copyNode(arg)
-      for i in 0..<arg.len:
-        var branch = copyNode(arg[i])
-        if arg[i].kind in {nkElifBranch, nkElifExpr}:
-          branch.add p(arg[i][0], c)
-          branch.add pArgIfTyped(arg[i][1])
-        else:
-          branch.add pArgIfTyped(arg[i][0])
-        result.add branch
-    elif arg.kind == nkCaseStmt:
-      result = copyNode(arg)
-      result.add p(arg[0], c)
-      for i in 1..<arg.len:
-        var branch: PNode
-        if arg[i].kind == nkOfBranch:
-          branch = arg[i] # of branch conditions are constants
-          branch[^1] = pArgIfTyped(arg[i][^1])
-        elif arg[i].kind in {nkElifBranch, nkElifExpr}:
-          branch = copyNode(arg[i])
-          branch.add p(arg[i][0], c)
-          branch.add pArgIfTyped(arg[i][1])
-        else:
-          branch = copyNode(arg[i])
-          branch.add pArgIfTyped(arg[i][0])
-        result.add branch
-    elif isAnalysableFieldAccess(arg, c.owner) and isLastRead(arg, c):
-      result = destructiveMoveVar(arg, c)
-    else:
-      # an object that is not temporary but passed to a 'sink' parameter
-      # results in a copy.
-      result = passCopyToSink(arg, c)
-  else:
-    result = p(arg, c)
-
-proc moveOrCopy(dest, ri: PNode; c: var Con): PNode =
-  # unfortunately, this needs to be kept consistent with the cases
-  # we handle in the 'case of' statement below:
-  const movableNodeKinds = (nkCallKinds + {nkSym, nkTupleConstr, nkObjConstr,
-                                           nkBracket, nkBracketExpr, nkNilLit})
-
-  template moveOrCopyIfTyped(riPart: PNode): PNode =
-    # typ is nil if we are in if/case expr branch with noreturn
-    if riPart.typ == nil: p(riPart, c)
-    else: moveOrCopy(dest, riPart, c)
-
-  case ri.kind
-  of nkCallKinds:
-    result = genSink(c, dest.typ, dest, ri)
-    # watch out and no not transform 'ri' twice if it's a call:
-    let ri2 = copyNode(ri)
-    let parameters = ri[0].typ
-    let L = if parameters != nil: parameters.len else: 0
-    ri2.add ri[0]
-    for i in 1..<ri.len:
-      ri2.add pArg(ri[i], c, i < L and isSinkTypeForParam(parameters[i]))
-    #recurse(ri, ri2)
-    result.add ri2
-  of nkBracketExpr:
-    if ri[0].kind == nkSym and isUnpackedTuple(ri[0].sym):
-      # unpacking of tuple: move out the elements
-      result = genSink(c, dest.typ, dest, ri)
-      result.add p(ri, c)
-    elif isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c):
-      # Rule 3: `=sink`(x, z); wasMoved(z)
-      var snk = genSink(c, dest.typ, dest, ri)
-      snk.add ri
-      result = newTree(nkStmtList, snk, genWasMoved(ri, c))
-    else:
-      result = genCopy(c, dest.typ, dest, ri)
-      result.add p(ri, c)
-  of nkStmtListExpr:
-    result = newNodeI(nkStmtList, ri.info)
-    for i in 0..ri.len-2:
-      result.add p(ri[i], c)
-    result.add moveOrCopy(dest, ri[^1], c)
-  of nkBlockExpr, nkBlockStmt:
-    result = newNodeI(nkBlockStmt, ri.info)
-    result.add ri[0] ## add label
-    result.add moveOrCopy(dest, ri[1], c)
-  of nkIfExpr, nkIfStmt:
-    result = newNodeI(nkIfStmt, ri.info)
-    for i in 0..<ri.len:
-      var branch = copyNode(ri[i])
-      if ri[i].kind in {nkElifBranch, nkElifExpr}:
-        branch.add p(ri[i][0], c)
-        branch.add moveOrCopyIfTyped(ri[i][1])
-      else:
-        branch.add moveOrCopyIfTyped(ri[i][0])
-      result.add branch
-  of nkCaseStmt:
-    result = newNodeI(nkCaseStmt, ri.info)
-    result.add p(ri[0], c)
-    for i in 1..<ri.len:
-      var branch: PNode
-      if ri[i].kind == nkOfBranch:
-        branch = ri[i] # of branch conditions are constants
-        branch[^1] = moveOrCopyIfTyped(ri[i][^1])
-      elif ri[i].kind in {nkElifBranch, nkElifExpr}:
-        branch = copyNode(ri[i])
-        branch.add p(ri[i][0], c)
-        branch.add moveOrCopyIfTyped(ri[i][1])
-      else:
-        branch = copyNode(ri[i])
-        branch.add moveOrCopyIfTyped(ri[i][0])
-      result.add branch
-  of nkBracket:
-    # array constructor
-    if ri.len > 0 and isDangerousSeq(ri.typ):
-      result = genCopy(c, dest.typ, dest, ri)
-    else:
-      result = genSink(c, dest.typ, dest, ri)
-    let ri2 = copyTree(ri)
-    for i in 0..<ri.len:
-      # everything that is passed to an array constructor is consumed,
-      # so these all act like 'sink' parameters:
-      ri2[i] = pArg(ri[i], c, isSink = true)
-    result.add ri2
-  of nkObjConstr:
-    result = genSink(c, dest.typ, dest, ri)
-    let ri2 = copyTree(ri)
-    for i in 1..<ri.len:
-      # everything that is passed to an object constructor is consumed,
-      # so these all act like 'sink' parameters:
-      ri2[i].sons[1] = pArg(ri[i][1], c, isSink = true)
-    result.add ri2
-  of nkTupleConstr, nkClosure:
-    result = genSink(c, dest.typ, dest, ri)
-    let ri2 = copyTree(ri)
-    for i in ord(ri.kind == nkClosure)..<ri.len:
-      # everything that is passed to an tuple constructor is consumed,
-      # so these all act like 'sink' parameters:
-      if ri[i].kind == nkExprColonExpr:
-        ri2[i].sons[1] = pArg(ri[i][1], c, isSink = true)
-      else:
-        ri2[i] = pArg(ri[i], c, isSink = true)
-    result.add ri2
-  of nkNilLit:
-    result = genSink(c, dest.typ, dest, ri)
-    result.add ri
-  of nkSym:
-    if isSinkParam(ri.sym):
-      # Rule 3: `=sink`(x, z); wasMoved(z)
-      sinkParamIsLastReadCheck(c, ri)
-      var snk = genSink(c, dest.typ, dest, ri)
-      snk.add ri
-      result = newTree(nkStmtList, snk, genWasMoved(ri, c))
-    elif ri.sym.kind != skParam and ri.sym.owner == c.owner and
-        isLastRead(ri, c) and canBeMoved(dest.typ):
-      # Rule 3: `=sink`(x, z); wasMoved(z)
-      var snk = genSink(c, dest.typ, dest, ri)
-      snk.add ri
-      result = newTree(nkStmtList, snk, genWasMoved(ri, c))
-    else:
-      result = genCopy(c, dest.typ, dest, ri)
-      result.add p(ri, c)
-  of nkHiddenSubConv, nkHiddenStdConv:
-    if sameType(ri.typ, ri[1].typ):
-      result = moveOrCopy(dest, ri[1], c)
-    elif ri[1].kind in movableNodeKinds:
-      result = moveOrCopy(dest, ri[1], c)
-      var b = newNodeIT(ri.kind, ri.info, ri.typ)
-      b.add ri[0] # add empty node
-      let L = result.len-1
-      b.add result[L]
-      result[L] = b
-    else:
-      result = genCopy(c, dest.typ, dest, ri)
-      result.add p(ri, c)
-  of nkObjDownConv, nkObjUpConv:
-    if ri[0].kind in movableNodeKinds:
-      result = moveOrCopy(dest, ri[0], c)
-      var b = newNodeIT(ri.kind, ri.info, ri.typ)
-      let L = result.len-1
-      b.add result[L]
-      result[L] = b
-    else:
-      result = genCopy(c, dest.typ, dest, ri)
-      result.add p(ri, c)
-  else:
-    if isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c) and
-        canBeMoved(dest.typ):
-      # Rule 3: `=sink`(x, z); wasMoved(z)
-      var snk = genSink(c, dest.typ, dest, ri)
-      snk.add ri
-      result = newTree(nkStmtList, snk, genWasMoved(ri, c))
-    else:
-      # XXX At least string literals can be moved?
-      result = genCopy(c, dest.typ, dest, ri)
-      result.add p(ri, c)
-
-proc computeUninit(c: var Con) =
-  if not c.uninitComputed:
-    c.uninitComputed = true
-    c.uninit = initIntSet()
-    var init = initIntSet()
-    discard initialized(c.g, pc = 0, init, c.uninit, comesFrom = -1)
-
-proc injectDefaultCalls(n: PNode, c: var Con) =
-  case n.kind
-  of nkVarSection, nkLetSection:
-    for i in 0..<n.len:
-      let it = n[i]
-      let L = it.len-1
-      let ri = it[L]
-      if it.kind == nkIdentDefs and ri.kind == nkEmpty:
-        computeUninit(c)
-        for j in 0..L-2:
-          let v = it[j]
-          doAssert v.kind == nkSym
-          if c.uninit.contains(v.sym.id):
-            it[L] = genDefaultCall(v.sym.typ, c, v.info)
-            break
-  of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef,
-      nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef:
-    discard
-  else:
-    for i in 0..<safeLen(n):
-      injectDefaultCalls(n[i], c)
-
-proc isCursor(n: PNode): bool {.inline.} =
-  result = n.kind == nkSym and sfCursor in n.sym.flags
-
-proc keepVar(n, it: PNode, c: var Con): PNode =
-  # keep the var but transform 'ri':
-  result = copyNode(n)
-  var itCopy = copyNode(it)
-  for j in 0..it.len-2:
-    itCopy.add it[j]
-  itCopy.add p(it[it.len-1], c)
-  result.add itCopy
-
-proc p(n: PNode; c: var Con): PNode =
-  case n.kind
-  of nkVarSection, nkLetSection:
-    discard "transform; var x = y to  var x; x op y  where op is a move or copy"
-    result = newNodeI(nkStmtList, n.info)
-
-    for i in 0..<n.len:
-      let it = n[i]
-      let L = it.len
-      var ri = it[L-1]
-      if it.kind == nkVarTuple and hasDestructor(ri.typ):
-        let x = lowerTupleUnpacking(c.graph, it, c.owner)
-        result.add p(x, c)
-      elif it.kind == nkIdentDefs and hasDestructor(it[0].typ) and not isCursor(it[0]):
-        for j in 0..L-3:
-          let v = it[j]
-          if v.kind == nkSym:
-            if sfCompileTime in v.sym.flags: continue
-            # move the variable declaration to the top of the frame:
-            c.addTopVar v
-            # make sure it's destroyed at the end of the proc:
-            if not isUnpackedTuple(it[0].sym):
-              c.destroys.add genDestroy(c, v.typ, v)
-          if ri.kind == nkEmpty and c.inLoop > 0:
-            ri = genDefaultCall(v.typ, c, v.info)
-          if ri.kind != nkEmpty:
-            let r = moveOrCopy(v, ri, c)
-            result.add r
-      else:
-        result.add keepVar(n, it, c)
-  of nkCallKinds:
-    let parameters = n[0].typ
-    let L = if parameters != nil: parameters.len else: 0
-    for i in 1 ..< n.len:
-      n.sons[i] = pArg(n[i], c, i < L and isSinkTypeForParam(parameters[i]))
-    if n.typ != nil and hasDestructor(n.typ):
-      discard "produce temp creation"
-      result = newNodeIT(nkStmtListExpr, n.info, n.typ)
-      let tmp = getTemp(c, n.typ, n.info)
-      var sinkExpr = genSink(c, n.typ, tmp, n)
-      sinkExpr.add n
-      result.add sinkExpr
-      result.add tmp
-      c.destroys.add genDestroy(c, n.typ, tmp)
-    else:
-      result = n
-  of nkAsgn, nkFastAsgn:
-    if hasDestructor(n[0].typ) and n[1].kind notin {nkProcDef, nkDo, nkLambda}:
-      # rule (self-assignment-removal):
-      if n[1].kind == nkSym and n[0].kind == nkSym and n[0].sym == n[1].sym:
-        result = newNodeI(nkEmpty, n.info)
-      else:
-        result = moveOrCopy(n[0], n[1], c)
-    else:
-      result = copyNode(n)
-      recurse(n, result)
-  of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef,
-      nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef:
-    result = n
-  of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv:
-    result = copyNode(n)
-    # Destination type
-    result.add n[0]
-    # Analyse the inner expression
-    result.add p(n[1], c)
-  of nkWhen:
-    # This should be a "when nimvm" node.
-    result = copyTree(n)
-    result[1][0] = p(result[1][0], c)
-  of nkRaiseStmt:
-    if optNimV2 in c.graph.config.globalOptions and n[0].kind != nkEmpty:
-      if n[0].kind in nkCallKinds:
-        let call = copyNode(n[0])
-        recurse(n[0], call)
-        result = copyNode(n)
-        result.add call
-      else:
-        let t = n[0].typ
-        let tmp = getTemp(c, t, n.info)
-        var m = genCopyNoCheck(c, t, tmp, n[0])
-
-        m.add p(n[0], c)
-        result = newTree(nkStmtList, genWasMoved(tmp, c), m)
-        var toDisarm = n[0]
-        if toDisarm.kind == nkStmtListExpr: toDisarm = toDisarm.lastSon
-        if toDisarm.kind == nkSym and toDisarm.sym.owner == c.owner:
-          result.add genWasMoved(toDisarm, c)
-        result.add newTree(nkRaiseStmt, tmp)
-    else:
-      result = copyNode(n)
-      recurse(n, result)
-  of nkForStmt, nkParForStmt, nkWhileStmt:
-    inc c.inLoop
-    result = copyNode(n)
-    recurse(n, result)
-    dec c.inLoop
-  else:
-    result = copyNode(n)
-    recurse(n, result)
-
-proc extractDestroysForTemporaries(c: Con, destroys: PNode): PNode =
-  result = newNodeI(nkStmtList, destroys.info)
-  for i in 0 ..< destroys.len:
-    if destroys[i][1][0].sym.kind == skTemp:
-      result.add destroys[i]
-      destroys[i] = c.emptyNode
-
-proc reverseDestroys(destroys: PNode) =
-  var reversed: seq[PNode]
-  for i in countdown(destroys.len - 1, 0):
-    reversed.add(destroys[i])
-  destroys.sons = reversed
-
-proc injectDestructorCalls*(g: ModuleGraph; owner: PSym; n: PNode): PNode =
-  if sfGeneratedOp in owner.flags or isInlineIterator(owner): return n
-  var c: Con
-  c.owner = owner
-  c.destroys = newNodeI(nkStmtList, n.info)
-  c.topLevelVars = newNodeI(nkVarSection, n.info)
-  c.graph = g
-  c.emptyNode = newNodeI(nkEmpty, n.info)
-  let cfg = constructCfg(owner, n)
-  shallowCopy(c.g, cfg)
-  c.jumpTargets = initIntSet()
-  for i in 0..<c.g.len:
-    if c.g[i].kind in {goto, fork}:
-      c.jumpTargets.incl(i+c.g[i].dest)
-  dbg:
-    echo "injecting into ", n
-    echoCfg(c.g)
-  if owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter}:
-    let params = owner.typ.n
-    for i in 1 ..< params.len:
-      let param = params[i].sym
-      if isSinkTypeForParam(param.typ) and hasDestructor(param.typ.skipTypes({tySink})):
-        c.destroys.add genDestroy(c, param.typ.skipTypes({tyGenericInst, tyAlias, tySink}), params[i])
-
-  #if optNimV2 in c.graph.config.globalOptions:
-  #  injectDefaultCalls(n, c)
-  let body = p(n, c)
-  result = newNodeI(nkStmtList, n.info)
-  if c.topLevelVars.len > 0:
-    result.add c.topLevelVars
-  if c.destroys.len > 0:
-    reverseDestroys(c.destroys)
-    if owner.kind == skModule:
-      result.add newTryFinally(body, extractDestroysForTemporaries(c, c.destroys))
-      g.globalDestructors.add c.destroys
-    else:
-      result.add newTryFinally(body, c.destroys)
-  else:
-    result.add body
-
-  dbg:
-    echo "------------------------------------"
-    echo owner.name.s, " transformed to: "
-    echo result

compiler/installer.ini

@@ -6,7 +6,7 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv64
+  linux: i386;amd64;powerpc64;arm;sparc;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv64
   macosx: i386;amd64;powerpc64
   solaris: i386;amd64;sparc;sparc64
   freebsd: i386;amd64
@@ -49,8 +49,11 @@ Start: "doc/html/overview.html"
 
 
 [Other]
-Files: "copying.txt"
+Files: "readme.txt;copying.txt;install.txt"
+Files: "makefile"
 Files: "koch.nim"
+Files: "install_nimble.nims"
+Files: "install_tools.nims"
 
 Files: "icons/nim.ico"
 Files: "icons/nim.rc"
@@ -69,6 +72,14 @@ Files: "nimpretty"
 Files: "testament"
 Files: "nimsuggest"
 Files: "nimsuggest/tests/*.nim"
+Files: "web/website.ini"
+Files: "web/ticker.html"
+Files: "web/*.nim"
+Files: "web/*.rst"
+Files: "web/*.csv"
+Files: "web/news/*.rst"
+Files: "bin/nimblepkg/*.nim"
+Files: "bin/nimblepkg/*.cfg"
 
 [Lib]
 Files: "lib"
@@ -76,17 +87,18 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/nimsuggest"
 
 Files: "tests"
 
 [Windows]
 Files: "bin/nim.exe"
+Files: "bin/c2nim.exe"
 Files: "bin/nimgrep.exe"
 Files: "bin/nimsuggest.exe"
 Files: "bin/nimble.exe"
 Files: "bin/vccexe.exe"
 Files: "bin/nimgrab.exe"
-Files: "bin/nimpretty.exe"
 
 Files: "koch.exe"
 Files: "finish.exe"

compiler/int128.nim

@@ -1,516 +0,0 @@
-
-
-type
-  Int128* = object
-    udata: array[4,uint32]
-
-template sdata(arg: Int128, idx: int): int32 =
-  # udata and sdata was supposed to be in a union, but unions are
-  # handled incorrectly in the VM.
-  cast[ptr int32](arg.udata[idx].unsafeAddr)[]
-
-# encoding least significant int first (like LittleEndian)
-
-type
-  InvalidArgument = object of Exception
-
-template require(cond: bool) =
-  if unlikely(not cond):
-    raise newException(InvalidArgument, "")
-
-const
-  Zero* = Int128(udata: [0'u32,0,0,0])
-  One* = Int128(udata: [1'u32,0,0,0])
-  Ten* = Int128(udata: [10'u32,0,0,0])
-  Min = Int128(udata: [0'u32,0,0,0x80000000'u32])
-  Max = Int128(udata: [high(uint32),high(uint32),high(uint32),uint32(high(int32))])
-
-template low*(t: typedesc[Int128]): Int128 = Min
-template high*(t: typedesc[Int128]): Int128 = Max
-
-proc `$`*(a: Int128): string
-
-proc toInt128*[T: SomeInteger](arg: T): Int128 =
-  when T is SomeUnsignedInt:
-    when sizeof(arg) <= 4:
-      result.udata[0] = uint32(arg)
-    else:
-      result.udata[0] = uint32(arg and T(0xffffffff))
-      result.udata[1] = uint32(arg shr 32)
-  else:
-    when sizeof(arg) <= 4:
-      result.sdata(0) = int32(arg)
-      if arg < 0: # sign extend
-        result.sdata(1) = -1
-        result.sdata(2) = -1
-        result.sdata(3) = -1
-    else:
-      let tmp = int64(arg)
-      result.udata[0] = uint32(tmp and 0xffffffff)
-      result.sdata(1) = int32(tmp shr 32)
-      if arg < 0: # sign extend
-        result.sdata(2) = -1
-        result.sdata(3) = -1
-
-template isNegative(arg: Int128): bool =
-  arg.sdata(3) < 0
-
-template isNegative(arg: int32): bool =
-  arg < 0
-
-proc bitconcat(a,b: uint32): uint64 =
-  (uint64(a) shl 32) or uint64(b)
-
-proc bitsplit(a: uint64): (uint32,uint32) =
-  (cast[uint32](a shr 32), cast[uint32](a))
-
-proc toInt64*(arg: Int128): int64 =
-  if isNegative(arg):
-    assert(arg.sdata(3) == -1, "out of range")
-    assert(arg.sdata(2) == -1, "out of range")
-  else:
-    assert(arg.sdata(3) == 0, "out of range")
-    assert(arg.sdata(2) == 0, "out of range")
-
-  cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
-
-proc toUInt64*(arg: Int128): uint64 =
-  assert(arg.udata[3] == 0)
-  assert(arg.udata[2] == 0)
-  bitconcat(arg.udata[1], arg.udata[0])
-
-proc addToHex(result: var string; arg: uint32) =
-  for i in 0 ..< 8:
-    let idx = (arg shr ((7-i) * 4)) and 0xf
-    result.add "0123456789abcdef"[idx]
-
-proc addToHex*(result: var string; arg: Int128) =
-  var i = 3
-  while i >= 0:
-    result.addToHex(arg.udata[i])
-    i -= 1
-
-proc toHex*(arg: Int128): string =
-  result.addToHex(arg)
-
-proc inc*(a: var Int128, y: uint32 = 1) =
-  let input = a
-  a.udata[0] += y
-  if unlikely(a.udata[0] < y):
-    a.udata[1].inc
-    if unlikely(a.udata[1] == 0):
-      a.udata[2].inc
-      if unlikely(a.udata[2] == 0):
-        a.udata[3].inc
-        doAssert(a.sdata(3) != low(int32), "overflow")
-
-proc cmp*(a,b: Int128): int =
-  let tmp1 = cmp(a.sdata(3), b.sdata(3))
-  if tmp1 != 0: return tmp1
-  let tmp2 = cmp(a.udata[2], b.udata[2])
-  if tmp2 != 0: return tmp2
-  let tmp3 = cmp(a.udata[1], b.udata[1])
-  if tmp3 != 0: return tmp3
-  let tmp4 = cmp(a.udata[0], b.udata[0])
-  return tmp4
-
-proc `<`*(a,b: Int128): bool =
-  cmp(a,b) < 0
-
-proc `<=`*(a,b: Int128): bool =
-  cmp(a,b) <= 0
-
-proc `==`*(a,b: Int128): bool =
-  if a.udata[0] != b.udata[0]: return false
-  if a.udata[1] != b.udata[1]: return false
-  if a.udata[2] != b.udata[2]: return false
-  if a.udata[3] != b.udata[3]: return false
-  return true
-
-proc inplaceBitnot(a: var Int128) =
-  a.udata[0] = not a.udata[0]
-  a.udata[1] = not a.udata[1]
-  a.udata[2] = not a.udata[2]
-  a.udata[3] = not a.udata[3]
-
-proc bitnot*(a: Int128): Int128 =
-  result.udata[0] = not a.udata[0]
-  result.udata[1] = not a.udata[1]
-  result.udata[2] = not a.udata[2]
-  result.udata[3] = not a.udata[3]
-
-proc bitand*(a,b: Int128): Int128 =
-  result.udata[0] = a.udata[0] and b.udata[0]
-  result.udata[1] = a.udata[1] and b.udata[1]
-  result.udata[2] = a.udata[2] and b.udata[2]
-  result.udata[3] = a.udata[3] and b.udata[3]
-
-proc bitor*(a,b: Int128): Int128 =
-  result.udata[0] = a.udata[0] or b.udata[0]
-  result.udata[1] = a.udata[1] or b.udata[1]
-  result.udata[2] = a.udata[2] or b.udata[2]
-  result.udata[3] = a.udata[3] or b.udata[3]
-
-proc bitxor*(a,b: Int128): Int128 =
-  result.udata[0] = a.udata[0] xor b.udata[0]
-  result.udata[1] = a.udata[1] xor b.udata[1]
-  result.udata[2] = a.udata[2] xor b.udata[2]
-  result.udata[3] = a.udata[3] xor b.udata[3]
-
-proc `shr`*(a: Int128, b: int): Int128 =
-  let b = b and 127
-  if b < 32:
-    result.sdata(3) = a.sdata(3) shr b
-    result.udata[2] = cast[uint32](bitconcat(a.udata[3], a.udata[2]) shr b)
-    result.udata[1] = cast[uint32](bitconcat(a.udata[2], a.udata[1]) shr b)
-    result.udata[0] = cast[uint32](bitconcat(a.udata[1], a.udata[0]) shr b)
-  elif b < 64:
-    if isNegative(a):
-      result.sdata(3) = -1
-    result.sdata(2) = a.sdata(3) shr (b and 31)
-    result.udata[1] = cast[uint32](bitconcat(a.udata[2], a.udata[1]) shr (b and 31))
-    result.udata[0] = cast[uint32](bitconcat(a.udata[1], a.udata[0]) shr (b and 31))
-  elif b < 96:
-    if isNegative(a):
-      result.sdata(3) = -1
-      result.sdata(2) = -1
-    result.sdata(1) = a.sdata(3) shr (b and 31)
-    result.udata[0] = cast[uint32](bitconcat(a.udata[1], a.udata[0]) shr (b and 31))
-  else: # b < 128
-    if isNegative(a):
-      result.sdata(3) = -1
-      result.sdata(2) = -1
-      result.sdata(1) = -1
-    result.sdata(0) = a.sdata(3) shr (b and 31)
-
-proc `shl`*(a: Int128, b: int): Int128 =
-  let b = b and 127
-  if b < 32:
-    result.udata[0] = a.udata[0] shl b
-    result.udata[1] = cast[uint32]((bitconcat(a.udata[1], a.udata[0]) shl b) shr 32)
-    result.udata[2] = cast[uint32]((bitconcat(a.udata[2], a.udata[1]) shl b) shr 32)
-    result.udata[3] = cast[uint32]((bitconcat(a.udata[3], a.udata[2]) shl b) shr 32)
-  elif b < 64:
-    result.udata[0] = 0
-    result.udata[1] = a.udata[0] shl (b and 31)
-    result.udata[2] = cast[uint32]((bitconcat(a.udata[1], a.udata[0]) shl (b and 31)) shr 32)
-    result.udata[3] = cast[uint32]((bitconcat(a.udata[2], a.udata[1]) shl (b and 31)) shr 32)
-  elif b < 96:
-    result.udata[0] = 0
-    result.udata[1] = 0
-    result.udata[2] = a.udata[0] shl (b and 31)
-    result.udata[3] = cast[uint32]((bitconcat(a.udata[1], a.udata[0]) shl (b and 31)) shr 32)
-  else:
-    result.udata[0] = 0
-    result.udata[1] = 0
-    result.udata[2] = 0
-    result.udata[3] = a.udata[0] shl (b and 31)
-
-
-proc `+`*(a,b: Int128): Int128 =
-  let tmp0 = uint64(a.udata[0]) + uint64(b.udata[0])
-  result.udata[0] = cast[uint32](tmp0)
-  let tmp1 = uint64(a.udata[1]) + uint64(b.udata[1]) + (tmp0 shr 32)
-  result.udata[1] = cast[uint32](tmp1)
-  let tmp2 = uint64(a.udata[2]) + uint64(b.udata[2]) + (tmp1 shr 32)
-  result.udata[2] = cast[uint32](tmp2)
-  let tmp3 = uint64(a.udata[3]) + uint64(b.udata[3]) + (tmp2 shr 32)
-  result.udata[3] = cast[uint32](tmp3)
-
-proc `+=`*(a: var Int128, b: Int128) =
-  a = a + b
-
-proc `-`*(a: Int128): Int128 =
-  result = bitnot(a)
-  result.inc
-
-proc `-`*(a,b: Int128): Int128 =
-  a + (-b)
-
-proc `-=`*(a: var Int128, b: Int128) =
-  a = a - b
-
-proc abs*(a: Int128): Int128 =
-  if isNegative(a):
-    -a
-  else:
-    a
-
-proc abs(a: int32): int =
-  if a < 0: -a else: a
-
-proc `*`(a: Int128, b: uint32): Int128 =
-  let tmp0 = uint64(a.udata[0]) * uint64(b)
-  let tmp1 = uint64(a.udata[1]) * uint64(b)
-  let tmp2 = uint64(a.udata[2]) * uint64(b)
-  let tmp3 = uint64(a.udata[3]) * uint64(b)
-
-  if unlikely(tmp3 > uint64(high(int32))):
-    assert(false, "overflow")
-
-  result.udata[0] = cast[uint32](tmp0)
-  result.udata[1] = cast[uint32](tmp1) + cast[uint32](tmp0 shr 32)
-  result.udata[2] = cast[uint32](tmp2) + cast[uint32](tmp1 shr 32)
-  result.udata[3] = cast[uint32](tmp3) + cast[uint32](tmp2 shr 32)
-
-proc `*`*(a: Int128, b: int32): Int128 =
-  let isNegative = isNegative(a) xor isNegative(b)
-  result = a * cast[uint32](abs(b))
-  if b < 0:
-    result = -result
-
-proc `*=`*(a: var Int128, b: int32): Int128 =
-  result = result * b
-
-proc makeInt128(high,low: uint64): Int128 =
-  result.udata[0] = cast[uint32](low)
-  result.udata[1] = cast[uint32](low shr 32)
-  result.udata[2] = cast[uint32](high)
-  result.udata[3] = cast[uint32](high shr 32)
-
-proc high64(a: Int128): uint64 =
-  bitconcat(a.udata[3], a.udata[2])
-
-proc low64(a: Int128): uint64 =
-  bitconcat(a.udata[1], a.udata[0])
-
-proc `*`*(lhs,rhs: Int128): Int128 =
-  let isNegative = isNegative(lhs) xor isNegative(rhs)
-
-  let
-    a = cast[uint64](lhs.udata[0])
-    b = cast[uint64](lhs.udata[1])
-    c = cast[uint64](lhs.udata[2])
-    d = cast[uint64](lhs.udata[3])
-
-    e = cast[uint64](rhs.udata[0])
-    f = cast[uint64](rhs.udata[1])
-    g = cast[uint64](rhs.udata[2])
-    h = cast[uint64](rhs.udata[3])
-
-
-  let a32 = cast[uint64](lhs.udata[1])
-  let a00 = cast[uint64](lhs.udata[0])
-  let b32 = cast[uint64](rhs.udata[1])
-  let b00 = cast[uint64](rhs.udata[0])
-
-  result = makeInt128(high64(lhs) * low64(rhs) + low64(lhs) * high64(rhs) + a32 * b32, a00 * b00)
-  result = result + toInt128(a32 * b00) shl 32
-  result = result + toInt128(a00 * b32) shl 32
-
-  if isNegative != isNegative(result):
-    echo result
-    assert(false, "overflow")
-
-proc `*=`*(a: var Int128, b: Int128) =
-  a = a * b
-
-import bitops
-
-proc fastLog2*(a: Int128): int =
-  if a.udata[3] != 0:
-    return 96 + fastLog2(a.udata[3])
-  if a.udata[2] != 0:
-    return 64 + fastLog2(a.udata[2])
-  if a.udata[1] != 0:
-    return 32 + fastLog2(a.udata[1])
-  if a.udata[0] != 0:
-    return      fastLog2(a.udata[0])
-
-proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
-  assert(divisor != Zero)
-  let isNegative = isNegative(dividend) xor isNegative(divisor)
-
-  var dividend = abs(dividend)
-  let divisor = abs(divisor)
-
-  if divisor > dividend:
-    result.quotient = Zero
-    result.remainder = dividend
-    return
-
-  if divisor == dividend:
-    result.quotient = One
-    result.remainder = Zero
-    return
-
-  var denominator = divisor
-  var quotient = Zero
-
-  # Left aligns the MSB of the denominator and the dividend.
-  let shift = fastLog2(dividend) - fastLog2(denominator)
-  denominator = denominator shl shift
-
-  # Uses shift-subtract algorithm to divide dividend by denominator. The
-  # remainder will be left in dividend.
-  for i in 0 .. shift:
-    quotient = quotient shl 1
-    if dividend >= denominator:
-      dividend = dividend - denominator
-      quotient = bitor(quotient, One)
-
-    denominator = denominator shr 1
-
-  result.quotient = quotient
-  result.remainder = dividend
-
-proc `div`*(a,b: Int128): Int128 =
-  let (a,b) = divMod(a,b)
-  return a
-
-proc `mod`*(a,b: Int128): Int128 =
-  let (a,b) = divMod(a,b)
-  return b
-
-proc `$`*(a: Int128): string =
-  if a == Zero:
-    result = "0"
-  elif a == low(Int128):
-    result = "-170141183460469231731687303715884105728"
-  else:
-    let isNegative = isNegative(a)
-    var a = abs(a)
-    while a > Zero:
-      let (quot, rem) = divMod(a, Ten)
-      result.add "0123456789"[rem.toInt64]
-      a = quot
-    if isNegative:
-      result.add '-'
-
-    var i = 0
-    var j = high(result)
-    while i < j:
-      swap(result[i], result[j])
-      i += 1
-      j -= 1
-
-proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =
-  assert(pos < arg.len)
-  assert(arg[pos] in {'-','0'..'9'})
-
-  var isNegative = false
-  var pos = pos
-  if arg[pos] == '-':
-    isNegative = true
-    pos += 1
-
-  result = Zero
-  while pos < arg.len and arg[pos] in '0' .. '9':
-    result = result * Ten
-    result.inc(uint32(arg[pos]) - uint32('0'))
-    pos += 1
-
-  if isNegative:
-    result = -result
-
-# fluff
-
-proc `<`*(a: Int128, b: BiggestInt): bool =
-  cmp(a,toInt128(b)) < 0
-
-proc `<`*(a: BiggestInt, b: Int128): bool =
-  cmp(toInt128(a), b) < 0
-
-proc `<=`*(a: Int128, b: BiggestInt): bool =
-  cmp(a,toInt128(b)) <= 0
-
-proc `<=`*(a: BiggestInt, b: Int128): bool =
-  cmp(toInt128(a), b) <= 0
-
-proc `==`*(a: Int128, b: BiggestInt): bool =
-  a == toInt128(b)
-
-proc `==`*(a: BiggestInt, b: Int128): bool =
-  toInt128(a) == b
-
-proc `-`*(a: BiggestInt, b: Int128): Int128 =
-  toInt128(a) - b
-
-proc `-`*(a: Int128, b: BiggestInt): Int128 =
-  a - toInt128(b)
-
-proc `+`*(a: BiggestInt, b: Int128): Int128 =
-  toInt128(a) + b
-
-proc `+`*(a: Int128, b: BiggestInt): Int128 =
-  a + toInt128(b)
-
-
-when isMainModule:
-  let (a,b) = divMod(Ten,Ten)
-
-  doAssert $One == "1"
-  doAssert $Ten == "10"
-  doAssert $Zero == "0"
-  let c = parseDecimalInt128("12345678989876543210123456789")
-  doAssert $c == "12345678989876543210123456789"
-
-  var d : array[39, Int128]
-  d[0] =  parseDecimalInt128("1")
-  d[1] =  parseDecimalInt128("10")
-  d[2] =  parseDecimalInt128("100")
-  d[3] =  parseDecimalInt128("1000")
-  d[4] =  parseDecimalInt128("10000")
-  d[5] =  parseDecimalInt128("100000")
-  d[6] =  parseDecimalInt128("1000000")
-  d[7] =  parseDecimalInt128("10000000")
-  d[8] =  parseDecimalInt128("100000000")
-  d[9] =  parseDecimalInt128("1000000000")
-  d[10] = parseDecimalInt128("10000000000")
-  d[11] = parseDecimalInt128("100000000000")
-  d[12] = parseDecimalInt128("1000000000000")
-  d[13] = parseDecimalInt128("10000000000000")
-  d[14] = parseDecimalInt128("100000000000000")
-  d[15] = parseDecimalInt128("1000000000000000")
-  d[16] = parseDecimalInt128("10000000000000000")
-  d[17] = parseDecimalInt128("100000000000000000")
-  d[18] = parseDecimalInt128("1000000000000000000")
-  d[19] = parseDecimalInt128("10000000000000000000")
-  d[20] = parseDecimalInt128("100000000000000000000")
-  d[21] = parseDecimalInt128("1000000000000000000000")
-  d[22] = parseDecimalInt128("10000000000000000000000")
-  d[23] = parseDecimalInt128("100000000000000000000000")
-  d[24] = parseDecimalInt128("1000000000000000000000000")
-  d[25] = parseDecimalInt128("10000000000000000000000000")
-  d[26] = parseDecimalInt128("100000000000000000000000000")
-  d[27] = parseDecimalInt128("1000000000000000000000000000")
-  d[28] = parseDecimalInt128("10000000000000000000000000000")
-  d[29] = parseDecimalInt128("100000000000000000000000000000")
-  d[30] = parseDecimalInt128("1000000000000000000000000000000")
-  d[31] = parseDecimalInt128("10000000000000000000000000000000")
-  d[32] = parseDecimalInt128("100000000000000000000000000000000")
-  d[33] = parseDecimalInt128("1000000000000000000000000000000000")
-  d[34] = parseDecimalInt128("10000000000000000000000000000000000")
-  d[35] = parseDecimalInt128("100000000000000000000000000000000000")
-  d[36] = parseDecimalInt128("1000000000000000000000000000000000000")
-  d[37] = parseDecimalInt128("10000000000000000000000000000000000000")
-  d[38] = parseDecimalInt128("100000000000000000000000000000000000000")
-
-  for i in 0 ..< d.len:
-    for j in 0 ..< d.len:
-      doAssert(cmp(d[i], d[j]) == cmp(i,j))
-      if i + j < d.len:
-        doAssert d[i] * d[j] == d[i+j]
-      if i - j >= 0:
-        doAssert d[i] div d[j] == d[i-j]
-
-  var sum: Int128
-
-  for it in d:
-    sum += it
-
-  doAssert $sum == "111111111111111111111111111111111111111"
-
-  for it in d.mitems:
-    it = -it
-
-  for i in 0 ..< d.len:
-    for j in 0 ..< d.len:
-      doAssert(cmp(d[i], d[j]) == -cmp(i,j))
-      if i + j < d.len:
-        doAssert d[i] * d[j] == -d[i+j]
-      if i - j >= 0:
-        doAssert d[i] div d[j] == -d[i-j]
-
-  doAssert $high(Int128) == "170141183460469231731687303715884105727"
-  doAssert $low(Int128) == "-170141183460469231731687303715884105728"

compiler/jstypes.nim

@@ -24,7 +24,7 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
       result = genObjectFields(p, typ, n.sons[0])
     else:
       s = nil
-      for i in 0 ..< length:
+      for i in countup(0, length - 1):
         if i > 0: add(s, ", \L")
         add(s, genObjectFields(p, typ, n.sons[i]))
       result = ("{kind: 2, len: $1, offset: 0, " &
@@ -41,14 +41,14 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
     if (n.sons[0].kind != nkSym): internalError(p.config, n.info, "genObjectFields")
     field = n.sons[0].sym
     s = genTypeInfo(p, field.typ)
-    for i in 1 ..< length:
+    for i in countup(1, length - 1):
       b = n.sons[i]           # branch
       u = nil
       case b.kind
       of nkOfBranch:
         if sonsLen(b) < 2:
           internalError(p.config, b.info, "genObjectFields; nkOfBranch broken")
-        for j in 0 .. sonsLen(b) - 2:
+        for j in countup(0, sonsLen(b) - 2):
           if u != nil: add(u, ", ")
           if b.sons[j].kind == nkRange:
             addf(u, "[$1, $2]", [rope(getOrdValue(b.sons[j].sons[0])),
@@ -103,7 +103,7 @@ proc genTupleInfo(p: PProc, typ: PType, name: Rope) =
 proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
   let length = sonsLen(typ.n)
   var s: Rope = nil
-  for i in 0 ..< length:
+  for i in countup(0, length - 1):
     if (typ.n.sons[i].kind != nkSym): internalError(p.config, typ.n.info, "genEnumInfo")
     let field = typ.n.sons[i].sym
     if i > 0: add(s, ", \L")
@@ -122,7 +122,7 @@ proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
          [name, genTypeInfo(p, typ.sons[0])])
 
 proc genTypeInfo(p: PProc, typ: PType): Rope =
-  let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink, tyOwned})
+  let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink})
   result = "NTI$1" % [rope(t.id)]
   if containsOrIncl(p.g.typeInfoGenerated, t.id): return
   case t.kind

compiler/lambdalifting.nim

@@ -10,9 +10,8 @@
 # This file implements lambda lifting for the transformator.
 
 import
-  intsets, strutils, options, ast, astalgo, msgs,
-  idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos,
-  transf, liftdestructors
+  intsets, strutils, options, ast, astalgo, trees, treetab, msgs,
+  idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos
 
 discard """
   The basic approach is that captured vars need to be put on the heap and
@@ -257,13 +256,11 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
     result.add(v)
   # add 'new' statement:
   result.add newCall(getSysSym(g, n.info, "internalNew"), env)
-  if optNimV2 in g.config.globalOptions:
-    createTypeBoundOps(g, nil, env.typ, n.info)
   result.add makeClosure(g, iter, env, n.info)
 
 proc freshVarForClosureIter*(g: ModuleGraph; s, owner: PSym): PNode =
   let envParam = getHiddenParam(g, owner)
-  let obj = envParam.typ.skipTypes({tyOwned, tyRef})
+  let obj = envParam.typ.lastSon
   addField(obj, s, g.cache)
 
   var access = newSymNode(envParam)
@@ -293,7 +290,7 @@ type
   DetectionPass = object
     processed, capturedVars: IntSet
     ownerToType: Table[int, PType]
-    somethingToDo, noDestructors: bool
+    somethingToDo: bool
     graph: ModuleGraph
 
 proc initDetectionPass(g: ModuleGraph; fn: PSym): DetectionPass =
@@ -322,39 +319,17 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
     rawAddSon(result, obj)
     c.ownerToType[owner.id] = result
 
-proc asOwnedRef(c: DetectionPass; t: PType): PType =
-  if optNimV2 in c.graph.config.globalOptions:
-    assert t.kind == tyRef
-    result = newType(tyOwned, t.owner)
-    result.flags.incl tfHasOwned
-    result.rawAddSon t
-  else:
-    result = t
-
-proc getEnvTypeForOwnerUp(c: var DetectionPass; owner: PSym;
-                          info: TLineInfo): PType =
-  var r = c.getEnvTypeForOwner(owner, info)
-  result = newType(tyPtr, owner)
-  rawAddSon(result, r.skipTypes({tyOwned, tyRef}))
-
 proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
   let refObj = c.getEnvTypeForOwner(dest, info) # getHiddenParam(dest).typ
-  let obj = refObj.skipTypes({tyOwned, tyRef})
-  # The assumption here is that gcDestructors means we cannot deal
-  # with cycles properly, so it's better to produce a weak ref (=ptr) here.
-  # This seems to be generally correct but since it's a bit risky it's only
-  # enabled for gcDestructors.
-  let fieldType = if false: # c.graph.config.selectedGC == gcDestructors:
-                    c.getEnvTypeForOwnerUp(dep, info) #getHiddenParam(dep).typ
-                  else:
-                    c.getEnvTypeForOwner(dep, info)
+  let obj = refObj.lastSon
+  let fieldType = c.getEnvTypeForOwner(dep, info) #getHiddenParam(dep).typ
   if refObj == fieldType:
     localError(c.graph.config, dep.info, "internal error: invalid up reference computed")
 
   let upIdent = getIdent(c.graph.cache, upName)
   let upField = lookupInRecord(obj.n, upIdent)
   if upField != nil:
-    if upField.typ.skipTypes({tyOwned, tyRef}) != fieldType.skipTypes({tyOwned, tyRef}):
+    if upField.typ != fieldType:
       localError(c.graph.config, dep.info, "internal error: up references do not agree")
   else:
     let result = newSym(skField, upIdent, obj.owner, obj.owner.info)
@@ -415,9 +390,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     if innerProc:
       if s.isIterator: c.somethingToDo = true
       if not c.processed.containsOrIncl(s.id):
-        let body = transformBody(c.graph, s, cache = true,
-                                 noDestructors = c.noDestructors)
-        detectCapturedVars(body, s, c)
+        detectCapturedVars(s.getBody, s, c)
     let ow = s.skipGenericOwner
     if ow == owner:
       if owner.isIterator:
@@ -425,15 +398,18 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
         addClosureParam(c, owner, n.info)
         if interestingIterVar(s):
           if not c.capturedVars.containsOrIncl(s.id):
-            let obj = getHiddenParam(c.graph, owner).typ.skipTypes({tyOwned, tyRef})
-            #let obj = c.getEnvTypeForOwner(s.owner).skipTypes({tyOwned, tyRef})
+            let obj = getHiddenParam(c.graph, owner).typ.lastSon
+            #let obj = c.getEnvTypeForOwner(s.owner).lastSon
 
             if s.name.id == getIdent(c.graph.cache, ":state").id:
               obj.n[0].sym.id = -s.id
             else:
               addField(obj, s, c.graph.cache)
+      # but always return because the rest of the proc is only relevant when
+      # ow != owner:
+      return
     # direct or indirect dependency:
-    elif (innerProc and s.typ.callConv == ccClosure) or interestingVar(s):
+    if (innerProc and s.typ.callConv == ccClosure) or interestingVar(s):
       discard """
         proc outer() =
           var x: int
@@ -451,8 +427,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       #echo "capturing ", n.info
       # variable 's' is actually captured:
       if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id):
-        let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef})
-        #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef})
+        let obj = c.getEnvTypeForOwner(ow, n.info).lastSon
+        #getHiddenParam(owner).typ.lastSon
         addField(obj, s, c.graph.cache)
       # create required upFields:
       var w = owner.skipGenericOwner
@@ -476,14 +452,17 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
           createUpField(c, w, up, n.info)
           w = up
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit,
-     nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef,
-     nkConverterDef, nkMacroDef, nkFuncDef, nkCommentStmt:
+     nkTemplateDef, nkTypeSection:
     discard
-  of nkLambdaKinds, nkIteratorDef:
+  of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef:
+    discard
+  of nkLambdaKinds, nkIteratorDef, nkFuncDef:
     if n.typ != nil:
       detectCapturedVars(n[namePos], owner, c)
   of nkReturnStmt:
-    detectCapturedVars(n[0], owner, c)
+    if n[0].kind in {nkAsgn, nkFastAsgn}:
+      detectCapturedVars(n[0].sons[1], owner, c)
+    else: assert n[0].kind == nkEmpty
   else:
     for i in 0..<n.len:
       detectCapturedVars(n[i], owner, c)
@@ -493,7 +472,6 @@ type
     processed: IntSet
     envVars: Table[int, PNode]
     inContainer: int
-    unownedEnvVars: Table[int, PNode] # only required for --newruntime
 
 proc initLiftingPass(fn: PSym): LiftingPass =
   result.processed = initIntSet()
@@ -518,9 +496,9 @@ proc accessViaEnvParam(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   localError(g.config, n.info, "internal error: environment misses: " & s.name.s)
   result = n
 
-proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo): PNode =
-  var v = newSym(skVar, getIdent(cache, envName), owner, info)
-  v.flags = {sfShadowed, sfGeneratedOp}
+proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType): PNode =
+  var v = newSym(skVar, getIdent(cache, envName), owner, owner.info)
+  incl(v.flags, sfShadowed)
   v.typ = typ
   result = newSymNode(v)
   when false:
@@ -532,61 +510,42 @@ proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo): PNo
       result = newSymNode(v)
 
 proc setupEnvVar(owner: PSym; d: DetectionPass;
-                 c: var LiftingPass; info: TLineInfo): PNode =
+                 c: var LiftingPass): PNode =
   if owner.isIterator:
     return getHiddenParam(d.graph, owner).newSymNode
-  result = c.envVars.getOrDefault(owner.id)
+  result = c.envvars.getOrDefault(owner.id)
   if result.isNil:
     let envVarType = d.ownerToType.getOrDefault(owner.id)
     if envVarType.isNil:
       localError d.graph.config, owner.info, "internal error: could not determine closure type"
-    result = newEnvVar(d.graph.cache, owner, asOwnedRef(d, envVarType), info)
+    result = newEnvVar(d.graph.cache, owner, envVarType)
     c.envVars[owner.id] = result
-    if optNimV2 in d.graph.config.globalOptions:
-      var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), owner, info)
-      v.flags = {sfShadowed, sfGeneratedOp}
-      v.typ = envVarType
-      c.unownedEnvVars[owner.id] = newSymNode(v)
 
 proc getUpViaParam(g: ModuleGraph; owner: PSym): PNode =
   let p = getHiddenParam(g, owner)
   result = p.newSymNode
   if owner.isIterator:
-    let upField = lookupInRecord(p.typ.skipTypes({tyOwned, tyRef}).n, getIdent(g.cache, upName))
+    let upField = lookupInRecord(p.typ.lastSon.n, getIdent(g.cache, upName))
     if upField == nil:
       localError(g.config, owner.info, "could not find up reference for closure iter")
     else:
       result = rawIndirectAccess(result, upField, p.info)
 
 proc rawClosureCreation(owner: PSym;
-                        d: DetectionPass; c: var LiftingPass;
-                        info: TLineInfo): PNode =
+                        d: DetectionPass; c: var LiftingPass): PNode =
   result = newNodeI(nkStmtList, owner.info)
 
   var env: PNode
   if owner.isIterator:
     env = getHiddenParam(d.graph, owner).newSymNode
   else:
-    env = setupEnvVar(owner, d, c, info)
+    env = setupEnvVar(owner, d, c)
     if env.kind == nkSym:
       var v = newNodeI(nkVarSection, env.info)
       addVar(v, env)
       result.add(v)
-      if optNimV2 in d.graph.config.globalOptions:
-        let unowned = c.unownedEnvVars[owner.id]
-        assert unowned != nil
-        addVar(v, unowned)
-
     # add 'new' statement:
     result.add(newCall(getSysSym(d.graph, env.info, "internalNew"), env))
-    if optNimV2 in d.graph.config.globalOptions:
-      let unowned = c.unownedEnvVars[owner.id]
-      assert unowned != nil
-      let env2 = copyTree(env)
-      env2.typ = unowned.typ
-      result.add newAsgnStmt(unowned, env2, env.info)
-      createTypeBoundOps(d.graph, nil, unowned.typ, env.info)
-
     # add assignment statements for captured parameters:
     for i in 1..<owner.typ.n.len:
       let local = owner.typ.n[i].sym
@@ -595,10 +554,10 @@ proc rawClosureCreation(owner: PSym;
         # add ``env.param = param``
         result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info))
 
-  let upField = lookupInRecord(env.typ.skipTypes({tyOwned, tyRef}).n, getIdent(d.graph.cache, upName))
+  let upField = lookupInRecord(env.typ.lastSon.n, getIdent(d.graph.cache, upName))
   if upField != nil:
     let up = getUpViaParam(d.graph, owner)
-    if up != nil and upField.typ.skipTypes({tyOwned, tyRef}) == up.typ.skipTypes({tyOwned, tyRef}):
+    if up != nil and upField.typ == up.typ:
       result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info),
                  up, env.info))
     #elif oldenv != nil and oldenv.typ == upField.typ:
@@ -606,18 +565,6 @@ proc rawClosureCreation(owner: PSym;
     #             oldenv, env.info))
     else:
       localError(d.graph.config, env.info, "internal error: cannot create up reference")
-  # we are not in the sem'check phase anymore! so pass 'nil' for the PContext
-  # and hope for the best:
-  if optNimV2 in d.graph.config.globalOptions:
-    createTypeBoundOps(d.graph, nil, env.typ, owner.info)
-
-proc finishClosureCreation(owner: PSym; d: DetectionPass; c: LiftingPass;
-                           info: TLineInfo; res: PNode) =
-  if optNimV2 in d.graph.config.globalOptions:
-    let unowned = c.unownedEnvVars[owner.id]
-    assert unowned != nil
-    let nilLit = newNodeIT(nkNilLit, info, unowned.typ)
-    res.add newAsgnStmt(unowned, nilLit, info)
 
 proc closureCreationForIter(iter: PNode;
                             d: DetectionPass; c: var LiftingPass): PNode =
@@ -625,11 +572,11 @@ proc closureCreationForIter(iter: PNode;
   let owner = iter.sym.skipGenericOwner
   var v = newSym(skVar, getIdent(d.graph.cache, envName), owner, iter.info)
   incl(v.flags, sfShadowed)
-  v.typ = asOwnedRef(d, getHiddenParam(d.graph, iter.sym).typ)
+  v.typ = getHiddenParam(d.graph, iter.sym).typ
   var vnode: PNode
   if owner.isIterator:
     let it = getHiddenParam(d.graph, owner)
-    addUniqueField(it.typ.skipTypes({tyOwned, tyRef}), v, d.graph.cache)
+    addUniqueField(it.typ.sons[0], v, d.graph.cache)
     vnode = indirectAccess(newSymNode(it), v, v.info)
   else:
     vnode = v.newSymNode
@@ -637,13 +584,11 @@ proc closureCreationForIter(iter: PNode;
     addVar(vs, vnode)
     result.add(vs)
   result.add(newCall(getSysSym(d.graph, iter.info, "internalNew"), vnode))
-  if optNimV2 in d.graph.config.globalOptions:
-    createTypeBoundOps(d.graph, nil, vnode.typ, iter.info)
 
-  let upField = lookupInRecord(v.typ.skipTypes({tyOwned, tyRef}).n, getIdent(d.graph.cache, upName))
+  let upField = lookupInRecord(v.typ.lastSon.n, getIdent(d.graph.cache, upName))
   if upField != nil:
-    let u = setupEnvVar(owner, d, c, iter.info)
-    if u.typ.skipTypes({tyOwned, tyRef}) == upField.typ.skipTypes({tyOwned, tyRef}):
+    let u = setupEnvVar(owner, d, c)
+    if u.typ == upField.typ:
       result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info),
                  u, iter.info))
     else:
@@ -652,10 +597,8 @@ proc closureCreationForIter(iter: PNode;
 
 proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
                      c: var LiftingPass): PNode =
-  var access = setupEnvVar(owner, d, c, n.info)
-  if optNimV2 in d.graph.config.globalOptions:
-    access = c.unownedEnvVars[owner.id]
-  let obj = access.typ.skipTypes({tyOwned, tyRef})
+  let access = setupEnvVar(owner, d, c)
+  let obj = access.typ.sons[0]
   let field = getFieldFromObj(obj, n.sym)
   if field != nil:
     result = rawIndirectAccess(access, field, n.info)
@@ -664,7 +607,7 @@ proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
     result = n
 
 proc getStateField*(g: ModuleGraph; owner: PSym): PSym =
-  getHiddenParam(g, owner).typ.skipTypes({tyOwned, tyRef}).n.sons[0].sym
+  getHiddenParam(g, owner).typ.sons[0].n.sons[0].sym
 
 proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
                       c: var LiftingPass): PNode
@@ -680,7 +623,7 @@ proc symToClosure(n: PNode; owner: PSym; d: DetectionPass;
     result = closureCreationForIter(n, d, c)
   elif s.skipGenericOwner == owner:
     # direct dependency, so use the outer's env variable:
-    result = makeClosure(d.graph, s, setupEnvVar(owner, d, c, n.info), n.info)
+    result = makeClosure(d.graph, s, setupEnvVar(owner, d, c), n.info)
   else:
     let available = getHiddenParam(d.graph, owner)
     let wanted = getHiddenParam(d.graph, s).typ
@@ -689,7 +632,7 @@ proc symToClosure(n: PNode; owner: PSym; d: DetectionPass;
     while true:
       if access.typ == wanted:
         return makeClosure(d.graph, s, access, n.info)
-      let obj = access.typ.skipTypes({tyOwned, tyRef})
+      let obj = access.typ.sons[0]
       let upField = lookupInRecord(obj.n, getIdent(d.graph.cache, upName))
       if upField == nil:
         localError(d.graph.config, n.info, "internal error: no environment found")
@@ -708,18 +651,14 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         #  echo renderTree(s.getBody, {renderIds})
         let oldInContainer = c.inContainer
         c.inContainer = 0
-        var body = transformBody(d.graph, s)
-        body = liftCapturedVars(body, s, d, c)
-        if c.envVars.getOrDefault(s.id).isNil:
-          s.transformedBody = body
+        var body = liftCapturedVars(s.getBody, s, d, c)
+        if c.envvars.getOrDefault(s.id).isNil:
+          s.ast.sons[bodyPos] = body
         else:
-          s.transformedBody = newTree(nkStmtList, rawClosureCreation(s, d, c, n.info), body)
-          finishClosureCreation(s, d, c, n.info, s.transformedBody)
+          s.ast.sons[bodyPos] = newTree(nkStmtList, rawClosureCreation(s, d, c), body)
         c.inContainer = oldInContainer
-
       if s.typ.callConv == ccClosure:
         result = symToClosure(n, owner, d, c)
-
     elif s.id in d.capturedVars:
       if s.owner != owner:
         result = accessViaEnvParam(d.graph, n, owner)
@@ -728,8 +667,9 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
       else:
         result = accessViaEnvVar(n, owner, d, c)
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom,
-     nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef, nkConverterDef,
-     nkMacroDef, nkFuncDef:
+     nkTemplateDef, nkTypeSection:
+    discard
+  of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef:
     discard
   of nkClosure:
     if n[1].kind == nkNilLit:
@@ -740,7 +680,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         # now we know better, so patch it:
         n.sons[0] = x.sons[0]
         n.sons[1] = x.sons[1]
-  of nkLambdaKinds, nkIteratorDef:
+  of nkLambdaKinds, nkIteratorDef, nkFuncDef:
     if n.typ != nil and n[namePos].kind == nkSym:
       let oldInContainer = c.inContainer
       c.inContainer = 0
@@ -758,8 +698,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
       # transform, let's not touch the LHS in order to make the lifting pass
       # correct when `result` is lifted
       n[0].sons[1] = liftCapturedVars(n[0].sons[1], owner, d, c)
-    else:
-      n.sons[0] = liftCapturedVars(n[0], owner, d, c)
+    else: assert n[0].kind == nkEmpty
   else:
     if owner.isIterator:
       if nfLL in n.flags:
@@ -776,37 +715,19 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
 # ------------------ old stuff -------------------------------------------
 
 proc semCaptureSym*(s, owner: PSym) =
-  discard """
-    proc outer() =
-      var x: int
-      proc inner() =
-        proc innerInner() =
-          echo x
-        innerInner()
-      inner()
-    # inner() takes a closure too!
-  """
-  proc propagateClosure(start, last: PSym) =
-    var o = start
-    while o != nil and o.kind != skModule:
-      if o == last: break
-      o.typ.callConv = ccClosure
-      o = o.skipGenericOwner
-
   if interestingVar(s) and s.kind != skResult:
     if owner.typ != nil and not isGenericRoutine(owner):
       # XXX: is this really safe?
       # if we capture a var from another generic routine,
       # it won't be consider captured.
       var o = owner.skipGenericOwner
-      while o != nil and o.kind != skModule:
+      while o.kind != skModule and o != nil:
         if s.owner == o:
           if owner.typ.callConv in {ccClosure, ccDefault} or owner.kind == skIterator:
             owner.typ.callConv = ccClosure
-            propagateClosure(owner.skipGenericOwner, s.owner)
           else:
             discard "do not produce an error here, but later"
-          #echo "computing .closure for ", owner.name.s, " because of ", s.name.s
+          #echo "computing .closure for ", owner.name.s, " ", owner.info, " because of ", s.name.s
         o = o.skipGenericOwner
     # since the analysis is not entirely correct, we don't set 'tfCapturesEnv'
     # here
@@ -825,8 +746,7 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType): PNo
   fn.kind = oldKind
   fn.typ.callConv = oldCC
 
-proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
-                  noDestructors: bool): PNode =
+proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PNode =
   # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
 
@@ -842,7 +762,6 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
     tooEarly = true
   else:
     var d = initDetectionPass(g, fn)
-    d.noDestructors = noDestructors
     detectCapturedVars(body, fn, d)
     if not d.somethingToDo and fn.isIterator:
       addClosureParam(d, fn, body.info)
@@ -851,9 +770,8 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
       var c = initLiftingPass(fn)
       result = liftCapturedVars(body, fn, d, c)
       # echo renderTree(result, {renderIds})
-      if c.envVars.getOrDefault(fn.id) != nil:
-        result = newTree(nkStmtList, rawClosureCreation(fn, d, c, body.info), result)
-        finishClosureCreation(fn, d, c, body.info, result)
+      if c.envvars.getOrDefault(fn.id) != nil:
+        result = newTree(nkStmtList, rawClosureCreation(fn, d, c), result)
     else:
       result = body
     #if fn.name.s == "get2":
@@ -921,9 +839,6 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
     result.add(v)
     # add 'new' statement:
     result.add(newCall(getSysSym(g, env.info, "internalNew"), env.newSymNode))
-    if optNimV2 in g.config.globalOptions:
-      createTypeBoundOps(g, nil, env.typ, body.info)
-
   elif op.kind == nkStmtListExpr:
     let closure = op.lastSon
     if closure.kind == nkClosure:

compiler/layouter.nim

@@ -12,10 +12,10 @@
 import idents, lexer, lineinfos, llstream, options, msgs, strutils,
   pathutils
 from os import changeFileExt
-from sequtils import delete
 
 const
-  MinLineLen = 15
+  MaxLineLen = 80
+  LineCommentColumn = 30
 
 type
   SplitKind = enum
@@ -24,36 +24,23 @@ type
   SemicolonKind = enum
     detectSemicolonKind, useSemicolon, dontTouch
 
-  LayoutToken = enum
-    ltSpaces,
-    ltCrucialNewline, ## a semantically crucial newline (indentation!)
-    ltSplittingNewline, ## newline used for splitting up long
-                        ## expressions (like after a comma or a binary operator)
-    ltTab,
-    ltOptionalNewline, ## optional newline introduced by nimpretty
-    ltComment, ltLit, ltKeyword, ltExportMarker, ltIdent,
-    ltOther, ltOpr, ltSomeParLe, ltSomeParRi,
-    ltBeginSection, ltEndSection
-
   Emitter* = object
     config: ConfigRef
     fid: FileIndex
     lastTok: TTokType
     inquote, lastTokWasTerse: bool
     semicolons: SemicolonKind
-    col, lastLineNumber, lineSpan, indentLevel, indWidth*, inSection: int
+    col, lastLineNumber, lineSpan, indentLevel, indWidth*: int
     keepIndents*: int
     doIndentMore*: int
-    kinds: seq[LayoutToken]
-    tokens: seq[string]
+    content: string
     indentStack: seq[int]
     fixedUntil: int # marks where we must not go in the content
     altSplitPos: array[SplitKind, int] # alternative split positions
-    maxLineLen*: int
 
 proc openEmitter*(em: var Emitter, cache: IdentCache;
                   config: ConfigRef, fileIdx: FileIndex) =
-  let fullPath = AbsoluteFile config.toFullPath(fileIdx)
+  let fullPath = Absolutefile config.toFullPath(fileIdx)
   if em.indWidth == 0:
     em.indWidth = getIndentWidth(fileIdx, llStreamOpen(fullPath, fmRead),
                                 cache, config)
@@ -63,47 +50,21 @@ proc openEmitter*(em: var Emitter, cache: IdentCache;
   em.lastTok = tkInvalid
   em.inquote = false
   em.col = 0
+  em.content = newStringOfCap(16_000)
   em.indentStack = newSeqOfCap[int](30)
   em.indentStack.add 0
   em.lastLineNumber = 1
 
-proc computeMax(em: Emitter; pos: int): int =
-  var p = pos
-  var extraSpace = 0
-  result = 0
-  while p < em.tokens.len and em.kinds[p] != ltEndSection:
-    var lhs = 0
-    var lineLen = 0
-    var foundTab = false
-    while p < em.tokens.len and em.kinds[p] != ltEndSection:
-      if em.kinds[p] in {ltCrucialNewline, ltSplittingNewline}:
-        if foundTab and lineLen <= em.maxLineLen:
-          result = max(result, lhs + extraSpace)
-        inc p
-        break
-      if em.kinds[p] == ltTab:
-        extraSpace = if em.kinds[p-1] == ltSpaces: 0 else: 1
-        foundTab = true
-      else:
-        if not foundTab:
-          inc lhs, em.tokens[p].len
-        inc lineLen, em.tokens[p].len
-      inc p
-
-proc computeRhs(em: Emitter; pos: int): int =
-  var p = pos
-  result = 0
-  while p < em.tokens.len and em.kinds[p] notin {ltCrucialNewline, ltSplittingNewline}:
-    inc result, em.tokens[p].len
-    inc p
-
-proc isLongEnough(lineLen, startPos, endPos: int): bool =
-  result = lineLen > MinLineLen and endPos > startPos + 4
-
-proc findNewline(em: Emitter; p, lineLen: var int) =
-  while p < em.tokens.len and em.kinds[p] notin {ltCrucialNewline, ltSplittingNewline}:
-    inc lineLen, em.tokens[p].len
-    inc p
+proc closeEmitter*(em: var Emitter) =
+  if fileExists(em.config.outFile) and readFile(em.config.outFile.string) == em.content:
+    discard "do nothing, see #9499"
+    return
+  var f = llStreamOpen(em.config.outFile, fmWrite)
+  if f == nil:
+    rawMessage(em.config, errGenerated, "cannot open file: " & em.config.outFile.string)
+    return
+  f.llStreamWrite em.content
+  llStreamClose(f)
 
 proc countNewlines(s: string): int =
   result = 0
@@ -117,337 +78,95 @@ proc calcCol(em: var Emitter; s: string) =
     dec i
     inc em.col
 
-proc optionalIsGood(em: var Emitter; pos, currentLen: int): bool =
-  let ourIndent = em.tokens[pos].len
-  var p = pos+1
-  var lineLen = 0
-  em.findNewline(p, lineLen)
-  if p == pos+1: # optionalNewline followed by another newline
-    result = false
-  elif em.kinds[p-1] == ltComment and currentLen+lineLen < em.maxLineLen+MinLineLen:
-    result = false
-  elif p+1 < em.tokens.len and em.kinds[p+1] == ltSpaces and
-      em.kinds[p-1] == ltOptionalNewline:
-    if em.tokens[p+1].len == ourIndent:
-      # concatenate lines with the same indententation
-      var nlPos = p
-      var lineLenTotal = lineLen
-      inc p
-      em.findNewline(p, lineLenTotal)
-      if isLongEnough(lineLenTotal, nlPos, p):
-        em.kinds[nlPos] = ltOptionalNewline
-        if em.kinds[nlPos+1] == ltSpaces:
-          # inhibit extra spaces when concatenating two lines
-          em.tokens[nlPos+1] = if em.tokens[nlPos-2] == ",": " " else: ""
-      result = true
-    elif em.tokens[p+1].len < ourIndent:
-      result = isLongEnough(lineLen, pos, p)
-  elif em.kinds[pos+1] in {ltOther, ltSomeParLe, ltSomeParRi}: # note: pos+1, not p+1
-    result = false
-  else:
-    result = isLongEnough(lineLen, pos, p)
-
-proc lenOfNextTokens(em: Emitter; pos: int): int =
-  result = 0
-  for i in 1 ..< em.tokens.len-pos:
-    if em.kinds[pos+i] in {ltCrucialNewline, ltSplittingNewline, ltOptionalNewline}: break
-    inc result, em.tokens[pos+i].len
-
-proc guidingInd(em: Emitter; pos: int): int =
-  var i = pos - 1
-  while i >= 0 and em.kinds[i] != ltSomeParLe:
-    dec i
-  while i+1 <= em.kinds.high and em.kinds[i] != ltSomeParRi:
-    if em.kinds[i] == ltSplittingNewline and em.kinds[i+1] == ltSpaces:
-      return em.tokens[i+1].len
-    inc i
-  result = -1
-
-proc closeEmitter*(em: var Emitter) =
-  template defaultCase() =
-    content.add em.tokens[i]
-    inc lineLen, em.tokens[i].len
-
-  let outFile = em.config.absOutFile
-
-  var content = newStringOfCap(16_000)
-  var maxLhs = 0
-  var lineLen = 0
-  var lineBegin = 0
-  var openPars = 0
-  var i = 0
-  while i <= em.tokens.high:
-    when defined(debug):
-      echo (token: em.tokens[i], kind: em.kinds[i])
-    case em.kinds[i]
-    of ltBeginSection:
-      maxLhs = computeMax(em, lineBegin)
-    of ltEndSection:
-      maxLhs = 0
-      lineBegin = i+1
-    of ltTab:
-      if i >= 2 and em.kinds[i-2] in {ltCrucialNewline, ltSplittingNewline} and
-          em.kinds[i-1] in {ltCrucialNewline, ltSplittingNewline, ltSpaces}:
-        # a previous section has ended
-        maxLhs = 0
-
-      if maxLhs == 0:
-        if em.kinds[i-1] != ltSpaces:
-          content.add em.tokens[i]
-          inc lineLen, em.tokens[i].len
-      else:
-        # pick the shorter indentation token:
-        var spaces = maxLhs - lineLen
-        if spaces < em.tokens[i].len or computeRhs(em, i+1)+maxLhs <= em.maxLineLen+MinLineLen:
-          if spaces <= 0 and content[^1] notin {' ', '\L'}: spaces = 1
-          for j in 1..spaces: content.add ' '
-          inc lineLen, spaces
-        else:
-          content.add em.tokens[i]
-          inc lineLen, em.tokens[i].len
-    of ltCrucialNewline, ltSplittingNewline:
-      content.add em.tokens[i]
-      lineLen = 0
-      lineBegin = i+1
-    of ltOptionalNewline:
-      let totalLineLen = lineLen + lenOfNextTokens(em, i)
-      if totalLineLen > em.maxLineLen and optionalIsGood(em, i, lineLen):
-        if i-1 >= 0 and em.kinds[i-1] == ltSpaces:
-          let spaces = em.tokens[i-1].len
-          content.setLen(content.len - spaces)
-        content.add "\L"
-        let guide = if openPars > 0: guidingInd(em, i) else: -1
-        if guide >= 0:
-          content.add repeat(' ', guide)
-          lineLen = guide
-        else:
-          content.add em.tokens[i]
-          lineLen = em.tokens[i].len
-        lineBegin = i+1
-        if i+1 < em.kinds.len and em.kinds[i+1] == ltSpaces:
-          # inhibit extra spaces at the start of a new line
-          inc i
-    of ltLit:
-      let lineSpan = countNewlines(em.tokens[i])
-      if lineSpan > 0:
-        em.calcCol(em.tokens[i])
-        lineLen = em.col
-      else:
-        inc lineLen, em.tokens[i].len
-      content.add em.tokens[i]
-    of ltSomeParLe:
-      inc openPars
-      defaultCase()
-    of ltSomeParRi:
-      doAssert openPars > 0
-      dec openPars
-      defaultCase()
-    else:
-      defaultCase()
-    inc i
-
-  if fileExists(outFile) and readFile(outFile.string) == content:
-    discard "do nothing, see #9499"
-    return
-  var f = llStreamOpen(outFile, fmWrite)
-  if f == nil:
-    rawMessage(em.config, errGenerated, "cannot open file: " & outFile.string)
-    return
-  f.llStreamWrite content
-  llStreamClose(f)
-
-proc wr(em: var Emitter; x: string; lt: LayoutToken) =
-  em.tokens.add x
-  em.kinds.add lt
+template wr(x) =
+  em.content.add x
   inc em.col, x.len
-  assert em.tokens.len == em.kinds.len
-
-proc wrNewline(em: var Emitter; kind = ltCrucialNewline) =
-  em.tokens.add "\L"
-  em.kinds.add kind
-  em.col = 0
-
-proc newlineWasSplitting*(em: var Emitter) =
-  if em.kinds.len >= 3 and em.kinds[^3] == ltCrucialNewline:
-    em.kinds[^3] = ltSplittingNewline
-
-#[
-Splitting newlines can occur:
-- after commas, semicolon, '[', '('.
-- after binary operators, '='.
-- after ':' type
-
-We only need parser support for the "after type" case.
-]#
-
-proc wrSpaces(em: var Emitter; spaces: int) =
-  if spaces > 0:
-    wr(em, strutils.repeat(' ', spaces), ltSpaces)
-
-proc wrSpace(em: var Emitter) =
-  wr(em, " ", ltSpaces)
-
-proc wrTab(em: var Emitter) =
-  wr(em, " ", ltTab)
-
-proc beginSection*(em: var Emitter) =
-  let pos = max(0, em.tokens.len-2)
-  em.tokens.insert "", pos
-  em.kinds.insert ltBeginSection, pos
-  inc em.inSection
-
-#wr(em, "", ltBeginSection)
-proc endSection*(em: var Emitter) =
-  em.tokens.insert "", em.tokens.len-2
-  em.kinds.insert ltEndSection, em.kinds.len-2
-  dec em.inSection
-
-#wr(em, "", ltEndSection)
-
-proc removeSpaces(em: var Emitter) =
-  while em.kinds.len > 0 and em.kinds[^1] == ltSpaces:
-    let tokenLen = em.tokens[^1].len
-    setLen(em.tokens, em.tokens.len-1)
-    setLen(em.kinds, em.kinds.len-1)
-    dec em.col, tokenLen
 
+template goodCol(col): bool = col in 40..MaxLineLen
 
 const
   openPars = {tkParLe, tkParDotLe,
               tkBracketLe, tkBracketLeColon, tkCurlyDotLe,
               tkCurlyLe}
-  closedPars = {tkParRi, tkParDotRi,
-              tkBracketRi, tkCurlyDotRi,
-              tkCurlyRi, tkBracketDotRi}
-
-  splitters = openPars + {tkComma, tkSemiColon} # do not add 'tkColon' here!
+  splitters = openPars + {tkComma, tkSemicolon}
   oprSet = {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
             tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor}
 
-template goodCol(col): bool = col >= em.maxLineLen div 2
+template rememberSplit(kind) =
+  if goodCol(em.col):
+    em.altSplitPos[kind] = em.content.len
 
 template moreIndent(em): int =
-  if em.doIndentMore > 0: em.indWidth*2 else: em.indWidth
+  (if em.doIndentMore > 0: em.indWidth*2 else: em.indWidth)
 
-template rememberSplit(kind) =
-  if goodCol(em.col) and not em.inquote:
-    let spaces = em.indentLevel+moreIndent(em)
-    if spaces < em.col and spaces > 0:
-      wr(em, strutils.repeat(' ', spaces), ltOptionalNewline)
-    #em.altSplitPos[kind] = em.tokens.len
-
-proc emitMultilineComment(em: var Emitter, lit: string, col: int; dontIndent: bool) =
-  # re-align every line in the multi-line comment:
-  var i = 0
-  var lastIndent = if em.keepIndents > 0: em.indentLevel else: em.indentStack[^1]
-  var b = 0
-  var dontIndent = dontIndent
-  var hasEmptyLine = false
-  for commentLine in splitLines(lit):
-    if i == 0 and (commentLine.endsWith("\\") or commentLine.endsWith("[")):
-      dontIndent = true
-      wr em, commentLine, ltComment
-    elif dontIndent:
-      if i > 0: wrNewline em
-      wr em, commentLine, ltComment
+proc softLinebreak(em: var Emitter, lit: string) =
+  # XXX Use an algorithm that is outlined here:
+  # https://llvm.org/devmtg/2013-04/jasper-slides.pdf
+  # +2 because we blindly assume a comma or ' &' might follow
+  if not em.inquote and em.col+lit.len+2 >= MaxLineLen:
+    if em.lastTok in splitters:
+      while em.content.len > 0 and em.content[em.content.high] == ' ':
+        setLen(em.content, em.content.len-1)
+      wr("\L")
+      em.col = 0
+      for i in 1..em.indentLevel+moreIndent(em): wr(" ")
     else:
-      let stripped = commentLine.strip()
-      if i == 0:
-        if em.kinds.len > 0 and em.kinds[^1] != ltTab:
-          wr(em, "", ltTab)
-      elif stripped.len == 0:
-        wrNewline em
-        hasEmptyLine = true
-      else:
-        var a = 0
-        while a < commentLine.len and commentLine[a] == ' ': inc a
-
-        if a > lastIndent:
-          b += em.indWidth
-          lastIndent = a
-        elif a < lastIndent:
-          b -= em.indWidth
-          lastIndent = a
-        wrNewline em
-        if not hasEmptyLine or col + b < 15:
-          if col + b > 0:
-            wr(em, repeat(' ', col+b), ltTab)
-          else:
-            wr(em, "", ltTab)
-        else:
-          wr(em, repeat(' ', a), ltSpaces)
-      wr em, stripped, ltComment
-    inc i
-
-proc lastChar(s: string): char =
-  result = if s.len > 0: s[s.high] else: '\0'
-
-proc endsInWhite(em: Emitter): bool =
-  var i = em.tokens.len-1
-  while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection}: dec(i)
-  result = if i >= 0: em.kinds[i] in {ltSpaces, ltCrucialNewline, ltSplittingNewline, ltTab} else: true
-
-proc endsInNewline(em: Emitter): bool =
-  var i = em.tokens.len-1
-  while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection, ltSpaces}: dec(i)
-  result = if i >= 0: em.kinds[i] in {ltCrucialNewline, ltSplittingNewline, ltTab} else: true
-
-proc endsInAlpha(em: Emitter): bool =
-  var i = em.tokens.len-1
-  while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection}: dec(i)
-  result = if i >= 0: em.tokens[i].lastChar in SymChars+{'_'} else: false
-
-proc emitComment(em: var Emitter; tok: TToken; dontIndent: bool) =
-  var col = em.col
-  let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA, tok.commentOffsetB)
-  em.lineSpan = countNewlines(lit)
-  if em.lineSpan > 0: calcCol(em, lit)
-  if em.lineSpan == 0:
-    if not endsInNewline(em):
-      wrTab em
-    wr em, lit, ltComment
-  else:
-    if not endsInWhite(em):
-      wrTab em
-      inc col
-    emitMultilineComment(em, lit, col, dontIndent)
+      # search backwards for a good split position:
+      for a in mitems(em.altSplitPos):
+        if a > em.fixedUntil:
+          var spaces = 0
+          while a+spaces < em.content.len and em.content[a+spaces] == ' ':
+            inc spaces
+          if spaces > 0: delete(em.content, a, a+spaces-1)
+          em.col = em.content.len - a
+          let ws = "\L" & repeat(' ', em.indentLevel+moreIndent(em))
+          em.content.insert(ws, a)
+          a = -1
+          break
 
 proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
-  template wasExportMarker(em): bool =
-    em.kinds.len > 0 and em.kinds[^1] == ltExportMarker
+
+  template endsInWhite(em): bool =
+    em.content.len == 0 or em.content[em.content.high] in {' ', '\L'}
+  template endsInAlpha(em): bool =
+    em.content.len > 0 and em.content[em.content.high] in SymChars+{'_'}
+
+  proc emitComment(em: var Emitter; tok: TToken) =
+    let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA, tok.commentOffsetB)
+    em.lineSpan = countNewlines(lit)
+    if em.lineSpan > 0: calcCol(em, lit)
+    if not endsInWhite(em):
+      wr(" ")
+      if em.lineSpan == 0 and max(em.col, LineCommentColumn) + lit.len <= MaxLineLen:
+        for i in 1 .. LineCommentColumn - em.col: wr(" ")
+    wr lit
 
   if tok.tokType == tkComment and tok.literal.startsWith("#!nimpretty"):
     case tok.literal
     of "#!nimpretty off":
       inc em.keepIndents
-      wrNewline em
+      wr("\L")
       em.lastLineNumber = tok.line + 1
     of "#!nimpretty on":
       dec em.keepIndents
       em.lastLineNumber = tok.line
-    wrNewline em
-    wr em, tok.literal, ltComment
+    wr("\L")
+    #for i in 1 .. tok.indent: wr " "
+    wr tok.literal
     em.col = 0
     em.lineSpan = 0
     return
 
   var preventComment = false
-  if tok.tokType == tkComment and tok.line == em.lastLineNumber:
+  if tok.tokType == tkComment and tok.line == em.lastLineNumber and tok.indent >= 0:
     # we have an inline comment so handle it before the indentation token:
-    emitComment(em, tok, dontIndent = (em.inSection == 0))
+    emitComment(em, tok)
     preventComment = true
-    em.fixedUntil = em.tokens.high
+    em.fixedUntil = em.content.high
 
   elif tok.indent >= 0:
-    var newlineKind = ltCrucialNewline
-    if em.keepIndents > 0:
+    if em.lastTok in (splitters + oprSet) or em.keepIndents > 0:
       em.indentLevel = tok.indent
-    elif (em.lastTok in (splitters + oprSet) and
-        tok.tokType notin (closedPars - {tkBracketDotRi})):
-      # aka: we are in an expression context:
-      let alignment = max(tok.indent - em.indentStack[^1], 0)
-      em.indentLevel = alignment + em.indentStack.high * em.indWidth
-      newlineKind = ltSplittingNewline
     else:
       if tok.indent > em.indentStack[^1]:
         em.indentStack.add tok.indent
@@ -466,122 +185,121 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
        is not touched.
     ]#
     # remove trailing whitespace:
-    removeSpaces em
-    wrNewline em, newlineKind
-    for i in 2..tok.line - em.lastLineNumber: wrNewline(em)
-    wrSpaces em, em.indentLevel
-    em.fixedUntil = em.tokens.high
+    while em.content.len > 0 and em.content[em.content.high] == ' ':
+      setLen(em.content, em.content.len-1)
+    wr("\L")
+    for i in 2..tok.line - em.lastLineNumber: wr("\L")
+    em.col = 0
+    for i in 1..em.indentLevel:
+      wr(" ")
+    em.fixedUntil = em.content.high
 
   var lastTokWasTerse = false
   case tok.tokType
   of tokKeywordLow..tokKeywordHigh:
     if endsInAlpha(em):
-      wrSpace em
+      wr(" ")
     elif not em.inquote and not endsInWhite(em) and
-        em.lastTok notin (openPars+{tkOpr, tkDotDot}) and not em.lastTokWasTerse:
+        em.lastTok notin openPars and not em.lastTokWasTerse:
       #and tok.tokType in oprSet
-      wrSpace em
+      wr(" ")
 
     if not em.inquote:
-      wr(em, TokTypeToStr[tok.tokType], ltKeyword)
-      if tok.tokType in {tkAnd, tkOr, tkIn, tkNotin}:
+      wr(TokTypeToStr[tok.tokType])
+
+      case tok.tokType
+      of tkAnd: rememberSplit(splitAnd)
+      of tkOr: rememberSplit(splitOr)
+      of tkIn, tkNotin:
         rememberSplit(splitIn)
-        wrSpace em
+        wr(" ")
+      else: discard
     else:
       # keywords in backticks are not normalized:
-      wr(em, tok.ident.s, ltIdent)
+      wr(tok.ident.s)
 
   of tkColon:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
-    wrSpace em
-  of tkSemiColon, tkComma:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(TokTypeToStr[tok.tokType])
+    wr(" ")
+  of tkSemicolon, tkComma:
+    wr(TokTypeToStr[tok.tokType])
     rememberSplit(splitComma)
-    wrSpace em
-  of openPars:
-    if tok.strongSpaceA > 0 and not em.endsInWhite and not em.wasExportMarker:
-      wrSpace em
-    wr(em, TokTypeToStr[tok.tokType], ltSomeParLe)
+    wr(" ")
+  of tkParDotLe, tkParLe, tkBracketDotLe, tkBracketLe,
+     tkCurlyLe, tkCurlyDotLe, tkBracketLeColon:
+    if tok.strongSpaceA > 0 and not em.endsInWhite:
+      wr(" ")
+    wr(TokTypeToStr[tok.tokType])
     rememberSplit(splitParLe)
-  of closedPars:
-    wr(em, TokTypeToStr[tok.tokType], ltSomeParRi)
-  of tkColonColon:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+  of tkParRi,
+     tkBracketRi, tkCurlyRi,
+     tkBracketDotRi,
+     tkCurlyDotRi,
+     tkParDotRi,
+     tkColonColon:
+    wr(TokTypeToStr[tok.tokType])
   of tkDot:
     lastTokWasTerse = true
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(TokTypeToStr[tok.tokType])
   of tkEquals:
-    if not em.inquote and not em.endsInWhite: wrSpace(em)
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
-    if not em.inquote: wrSpace(em)
+    if not em.inquote and not em.endsInWhite: wr(" ")
+    wr(TokTypeToStr[tok.tokType])
+    if not em.inquote: wr(" ")
   of tkOpr, tkDotDot:
-    if em.inquote or ((tok.strongSpaceA == 0 and tok.strongSpaceB == 0) and
-        tok.ident.s notin ["<", ">", "<=", ">=", "==", "!="]):
+    if (tok.strongSpaceA == 0 and tok.strongSpaceB == 0) or em.inquote:
       # bug #9504: remember to not spacify a keyword:
       lastTokWasTerse = true
       # if not surrounded by whitespace, don't produce any whitespace either:
-      wr(em, tok.ident.s, ltOpr)
+      wr(tok.ident.s)
     else:
-      if not em.endsInWhite: wrSpace(em)
-      wr(em, tok.ident.s, ltOpr)
+      if not em.endsInWhite: wr(" ")
+      wr(tok.ident.s)
       template isUnary(tok): bool =
         tok.strongSpaceB == 0 and tok.strongSpaceA > 0
 
       if not isUnary(tok):
         rememberSplit(splitBinary)
-        wrSpace(em)
+        wr(" ")
   of tkAccent:
-    if not em.inquote and endsInAlpha(em): wrSpace(em)
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    if not em.inquote and endsInAlpha(em): wr(" ")
+    wr(TokTypeToStr[tok.tokType])
     em.inquote = not em.inquote
   of tkComment:
     if not preventComment:
-      emitComment(em, tok, dontIndent = false)
+      emitComment(em, tok)
   of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, tkGTripleStrLit, tkCharLit:
     let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
-    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
+    softLinebreak(em, lit)
+    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wr(" ")
     em.lineSpan = countNewlines(lit)
     if em.lineSpan > 0: calcCol(em, lit)
-    wr em, lit, ltLit
+    wr lit
   of tkEof: discard
   else:
     let lit = if tok.ident != nil: tok.ident.s else: tok.literal
-    if endsInAlpha(em): wrSpace(em)
-    wr em, lit, ltIdent
+    softLinebreak(em, lit)
+    if endsInAlpha(em): wr(" ")
+    wr lit
 
   em.lastTok = tok.tokType
   em.lastTokWasTerse = lastTokWasTerse
   em.lastLineNumber = tok.line + em.lineSpan
   em.lineSpan = 0
 
-proc endsWith(em: Emitter; k: varargs[string]): bool =
-  if em.tokens.len < k.len: return false
-  for i in 0..high(k):
-    if em.tokens[em.tokens.len - k.len + i] != k[i]: return false
-  return true
-
-proc rfind(em: Emitter, t: string): int =
-  for i in 1 .. 5:
-    if em.tokens[^i] == t:
-      return i
-
 proc starWasExportMarker*(em: var Emitter) =
-  if em.endsWith(" ", "*", " "):
-    setLen(em.tokens, em.tokens.len-3)
-    setLen(em.kinds, em.kinds.len-3)
-    em.tokens.add("*")
-    em.kinds.add ltExportMarker
+  if em.content.endsWith(" * "):
+    setLen(em.content, em.content.len-3)
+    em.content.add("*")
     dec em.col, 2
 
 proc commaWasSemicolon*(em: var Emitter) =
   if em.semicolons == detectSemicolonKind:
-    em.semicolons = if em.rfind(";") > 0: useSemicolon else: dontTouch
-  if em.semicolons == useSemicolon:
-    let commaPos = em.rfind(",")
-    if commaPos > 0:
-      em.tokens[^commaPos] = ";"
+    em.semicolons = if em.content.endsWith(", "): dontTouch else: useSemicolon
+  if em.semicolons == useSemicolon and em.content.endsWith(", "):
+    setLen(em.content, em.content.len-2)
+    em.content.add("; ")
 
 proc curlyRiWasPragma*(em: var Emitter) =
-  if em.endsWith("}"):
-    em.tokens[^1] = ".}"
-    inc em.col
+  if em.content.endsWith("}"):
+    setLen(em.content, em.content.len-1)
+    em.content.add(".}")

compiler/lexer.nim

@@ -17,7 +17,7 @@
 
 import
   hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream,
-  wordrecg, lineinfos, pathutils, parseutils
+  wordrecg, lineinfos, pathutils
 
 const
   MaxLineLength* = 80         # lines longer than this lead to a warning
@@ -146,6 +146,10 @@ type
       previousToken: TLineInfo
     config*: ConfigRef
 
+when defined(nimpretty):
+  var
+    gIndentationWidth*: int
+
 proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
   result = newLineInfo(L.fileIdx, tok.line, tok.col)
   when defined(nimpretty):
@@ -169,7 +173,7 @@ proc isNimIdentifier*(s: string): bool =
       inc(i)
     result = true
 
-proc `$`*(tok: TToken): string =
+proc tokToStr*(tok: TToken): string =
   case tok.tokType
   of tkIntLit..tkInt64Lit: result = $tok.iNumber
   of tkFloatLit..tkFloat64Lit: result = $tok.fNumber
@@ -184,11 +188,11 @@ proc `$`*(tok: TToken): string =
 
 proc prettyTok*(tok: TToken): string =
   if isKeyword(tok.tokType): result = "keyword " & tok.ident.s
-  else: result = $tok
+  else: result = tokToStr(tok)
 
 proc printTok*(conf: ConfigRef; tok: TToken) =
   msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" &
-      TokTypeToStr[tok.tokType] & " " & $tok)
+      TokTypeToStr[tok.tokType] & " " & tokToStr(tok))
 
 proc initToken*(L: var TToken) =
   L.tokType = tkInvalid
@@ -219,7 +223,7 @@ proc fillToken(L: var TToken) =
 proc openLexer*(lex: var TLexer, fileIdx: FileIndex, inputstream: PLLStream;
                  cache: IdentCache; config: ConfigRef) =
   openBaseLexer(lex, inputstream)
-  lex.fileIdx = fileIdx
+  lex.fileIdx = fileidx
   lex.indentAhead = -1
   lex.currLineIndent = 0
   inc(lex.lineNumber, inputstream.lineOffset)
@@ -303,6 +307,20 @@ template tokenEndPrevious(tok, pos) =
   when defined(nimpretty):
     tok.offsetB = L.offsetBase + pos
 
+{.push overflowChecks: off.}
+# We need to parse the largest uint literal without overflow checks
+proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int =
+  var i = start
+  if i < s.len and s[i] in {'0'..'9'}:
+    b = 0
+    while i < s.len and s[i] in {'0'..'9'}:
+      b = b * 10 + (ord(s[i]) - ord('0'))
+      inc(i)
+      while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored
+    result = i - start
+{.pop.} # overflowChecks
+
+
 template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
   add(t.literal, replacementChar)
   inc(L.bufpos)
@@ -314,16 +332,17 @@ template eatChar(L: var TLexer, t: var TToken) =
 proc getNumber(L: var TLexer, result: var TToken) =
   proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]): Natural =
     var pos = L.bufpos              # use registers for pos, buf
+    var buf = L.buf
     result = 0
     while true:
-      if L.buf[pos] in chars:
-        add(tok.literal, L.buf[pos])
+      if buf[pos] in chars:
+        add(tok.literal, buf[pos])
         inc(pos)
         inc(result)
       else:
         break
-      if L.buf[pos] == '_':
-        if L.buf[pos+1] notin chars:
+      if buf[pos] == '_':
+        if buf[pos+1] notin chars:
           lexMessage(L, errGenerated,
             "only single underscores may occur in a token and token may not " &
             "end with an underscore: e.g. '1__1' and '1_' are invalid")
@@ -334,8 +353,9 @@ proc getNumber(L: var TLexer, result: var TToken) =
 
   proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
     var pos = L.bufpos              # use registers for pos, buf
-    while L.buf[pos] in chars:
-      add(tok.literal, L.buf[pos])
+    var buf = L.buf
+    while buf[pos] in chars:
+      add(tok.literal, buf[pos])
       inc(pos)
     L.bufpos = pos
 
@@ -375,7 +395,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
   result.literal = ""
   result.base = base10
   startpos = L.bufpos
-  tokenBegin(result, startpos)
+  tokenBegin(result, startPos)
 
   # First stage: find out base, make verifications, build token literal string
   # {'c', 'C'} is added for deprecation reasons to provide a clear error message
@@ -530,13 +550,13 @@ proc getNumber(L: var TLexer, result: var TToken) =
 
       case result.tokType
       of tkIntLit, tkInt64Lit: result.iNumber = xi
-      of tkInt8Lit: result.iNumber = ashr(xi shl 56, 56)
-      of tkInt16Lit: result.iNumber = ashr(xi shl 48, 48)
-      of tkInt32Lit: result.iNumber = ashr(xi shl 32, 32)
+      of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi))))
+      of tkInt16Lit: result.iNumber = BiggestInt(int16(toU16(int(xi))))
+      of tkInt32Lit: result.iNumber = BiggestInt(int32(toU32(int64(xi))))
       of tkUIntLit, tkUInt64Lit: result.iNumber = xi
-      of tkUInt8Lit: result.iNumber = xi and 0xff
-      of tkUInt16Lit: result.iNumber = xi and 0xffff
-      of tkUInt32Lit: result.iNumber = xi and 0xffffffff
+      of tkUInt8Lit: result.iNumber = BiggestInt(uint8(toU8(int(xi))))
+      of tkUInt16Lit: result.iNumber = BiggestInt(uint16(toU16(int(xi))))
+      of tkUInt32Lit: result.iNumber = BiggestInt(uint32(toU32(int64(xi))))
       of tkFloat32Lit:
         result.fNumber = (cast[PFloat32](addr(xi)))[]
         # note: this code is endian neutral!
@@ -565,44 +585,34 @@ proc getNumber(L: var TLexer, result: var TToken) =
       case result.tokType
       of floatTypes:
         result.fNumber = parseFloat(result.literal)
-      of tkUInt64Lit:
-        var iNumber: uint64
-        var len: int
-        try:
-          len = parseBiggestUInt(result.literal, iNumber)
-        except ValueError:
-          raise newException(OverflowError, "number out of range: " & $result.literal)
-        if len != result.literal.len:
-          raise newException(ValueError, "invalid integer: " & $result.literal)
-        result.iNumber = cast[int64](iNumber)
+      of tkUint64Lit:
+        xi = 0
+        let len = unsafeParseUInt(result.literal, xi)
+        if len != result.literal.len or len == 0:
+          raise newException(ValueError, "invalid integer: " & $xi)
+        result.iNumber = xi
       else:
-        var iNumber: int64
-        var len: int
-        try:
-          len = parseBiggestInt(result.literal, iNumber)
-        except ValueError:
-          raise newException(OverflowError, "number out of range: " & $result.literal)
-        if len != result.literal.len:
-          raise newException(ValueError, "invalid integer: " & $result.literal)
-        result.iNumber = iNumber
+        result.iNumber = parseBiggestInt(result.literal)
 
-      # Explicit bounds checks. Only T.high needs to be considered
-      # since result.iNumber can't be negative.
+      # Explicit bounds checks
       let outOfRange =
         case result.tokType
-        of tkInt8Lit: result.iNumber > int8.high
-        of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high)
-        of tkInt16Lit: result.iNumber > int16.high
-        of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high)
-        of tkInt32Lit: result.iNumber > int32.high
-        of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high)
+        of tkInt8Lit: (result.iNumber < int8.low or result.iNumber > int8.high)
+        of tkUInt8Lit: (result.iNumber < BiggestInt(uint8.low) or
+                        result.iNumber > BiggestInt(uint8.high))
+        of tkInt16Lit: (result.iNumber < int16.low or result.iNumber > int16.high)
+        of tkUInt16Lit: (result.iNumber < BiggestInt(uint16.low) or
+                        result.iNumber > BiggestInt(uint16.high))
+        of tkInt32Lit: (result.iNumber < int32.low or result.iNumber > int32.high)
+        of tkUInt32Lit: (result.iNumber < BiggestInt(uint32.low) or
+                        result.iNumber > BiggestInt(uint32.high))
         else: false
 
       if outOfRange: lexMessageLitNum(L, "number out of range: '$1'", startpos)
 
     # Promote int literal to int64? Not always necessary, but more consistent
     if result.tokType == tkIntLit:
-      if result.iNumber > high(int32):
+      if (result.iNumber < low(int32)) or (result.iNumber > high(int32)):
         result.tokType = tkInt64Lit
 
   except ValueError:
@@ -612,12 +622,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
   tokenEnd(result, postPos-1)
   L.bufpos = postPos
 
-proc handleHexChar(L: var TLexer, xi: var int; position: range[0..4]) =
-  template invalid() =
-    lexMessage(L, errGenerated,
-      "expected a hex digit, but found: " & L.buf[L.bufpos] &
-        "; maybe prepend with 0")
-
+proc handleHexChar(L: var TLexer, xi: var int) =
   case L.buf[L.bufpos]
   of '0'..'9':
     xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('0'))
@@ -628,58 +633,13 @@ proc handleHexChar(L: var TLexer, xi: var int; position: range[0..4]) =
   of 'A'..'F':
     xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('A') + 10)
     inc(L.bufpos)
-  of '"', '\'':
-    if position <= 1: invalid()
-    # do not progress the bufpos here.
-    if position == 0: inc(L.bufpos)
-  else:
-    invalid()
-    # Need to progress for `nim check`
-    inc(L.bufpos)
+  else: discard
 
 proc handleDecChars(L: var TLexer, xi: var int) =
   while L.buf[L.bufpos] in {'0'..'9'}:
     xi = (xi * 10) + (ord(L.buf[L.bufpos]) - ord('0'))
     inc(L.bufpos)
 
-proc addUnicodeCodePoint(s: var string, i: int) =
-  let i = cast[uint](i)
-  # inlined toUTF-8 to avoid unicode and strutils dependencies.
-  let pos = s.len
-  if i <= 127:
-    s.setLen(pos+1)
-    s[pos+0] = chr(i)
-  elif i <= 0x07FF:
-    s.setLen(pos+2)
-    s[pos+0] = chr((i shr 6) or 0b110_00000)
-    s[pos+1] = chr((i and ones(6)) or 0b10_0000_00)
-  elif i <= 0xFFFF:
-    s.setLen(pos+3)
-    s[pos+0] = chr(i shr 12 or 0b1110_0000)
-    s[pos+1] = chr(i shr 6 and ones(6) or 0b10_0000_00)
-    s[pos+2] = chr(i and ones(6) or 0b10_0000_00)
-  elif i <= 0x001FFFFF:
-    s.setLen(pos+4)
-    s[pos+0] = chr(i shr 18 or 0b1111_0000)
-    s[pos+1] = chr(i shr 12 and ones(6) or 0b10_0000_00)
-    s[pos+2] = chr(i shr 6 and ones(6) or 0b10_0000_00)
-    s[pos+3] = chr(i and ones(6) or 0b10_0000_00)
-  elif i <= 0x03FFFFFF:
-    s.setLen(pos+5)
-    s[pos+0] = chr(i shr 24 or 0b111110_00)
-    s[pos+1] = chr(i shr 18 and ones(6) or 0b10_0000_00)
-    s[pos+2] = chr(i shr 12 and ones(6) or 0b10_0000_00)
-    s[pos+3] = chr(i shr 6 and ones(6) or 0b10_0000_00)
-    s[pos+4] = chr(i and ones(6) or 0b10_0000_00)
-  elif i <= 0x7FFFFFFF:
-    s.setLen(pos+6)
-    s[pos+0] = chr(i shr 30 or 0b1111110_0)
-    s[pos+1] = chr(i shr 24 and ones(6) or 0b10_0000_00)
-    s[pos+2] = chr(i shr 18 and ones(6) or 0b10_0000_00)
-    s[pos+3] = chr(i shr 12 and ones(6) or 0b10_0000_00)
-    s[pos+4] = chr(i shr 6 and ones(6) or 0b10_0000_00)
-    s[pos+5] = chr(i and ones(6) or 0b10_0000_00)
-
 proc getEscapedChar(L: var TLexer, tok: var TToken) =
   inc(L.bufpos)               # skip '\'
   case L.buf[L.bufpos]
@@ -726,36 +686,29 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
   of '\\':
     add(tok.literal, '\\')
     inc(L.bufpos)
-  of 'x', 'X':
+  of 'x', 'X', 'u', 'U':
+    var tp = L.buf[L.bufpos]
     inc(L.bufpos)
     var xi = 0
-    handleHexChar(L, xi, 1)
-    handleHexChar(L, xi, 2)
-    add(tok.literal, chr(xi))
-  of 'u', 'U':
-    if tok.tokType == tkCharLit:
-      lexMessage(L, errGenerated, "\\u not allowed in character literal")
-    inc(L.bufpos)
-    var xi = 0
-    if L.buf[L.bufpos] == '{':
-      inc(L.bufpos)
-      var start = L.bufpos
-      while L.buf[L.bufpos] != '}':
-        handleHexChar(L, xi, 0)
-      if start == L.bufpos:
-        lexMessage(L, errGenerated,
-          "Unicode codepoint cannot be empty")
-      inc(L.bufpos)
-      if xi > 0x10FFFF:
-        let hex = ($L.buf)[start..L.bufpos-2]
-        lexMessage(L, errGenerated,
-          "Unicode codepoint must be lower than 0x10FFFF, but was: " & hex)
+    handleHexChar(L, xi)
+    handleHexChar(L, xi)
+    if tp in {'u', 'U'}:
+      handleHexChar(L, xi)
+      handleHexChar(L, xi)
+      # inlined toUTF-8 to avoid unicode and strutils dependencies.
+      if xi <=% 127:
+        add(tok.literal, xi.char )
+      elif xi <=% 0x07FF:
+        add(tok.literal, ((xi shr 6) or 0b110_00000).char )
+        add(tok.literal, ((xi and ones(6)) or 0b10_0000_00).char )
+      elif xi <=% 0xFFFF:
+        add(tok.literal, (xi shr 12 or 0b1110_0000).char )
+        add(tok.literal, (xi shr 6 and ones(6) or 0b10_0000_00).char )
+        add(tok.literal, (xi and ones(6) or 0b10_0000_00).char )
+      else: # value is 0xFFFF
+        add(tok.literal, "\xef\xbf\xbf" )
     else:
-      handleHexChar(L, xi, 1)
-      handleHexChar(L, xi, 2)
-      handleHexChar(L, xi, 3)
-      handleHexChar(L, xi, 4)
-    addUnicodeCodePoint(tok.literal, xi)
+      add(tok.literal, chr(xi))
   of '0'..'9':
     if matchTwoChars(L, '0', {'0'..'9'}):
       lexMessage(L, warnOctalEscape)
@@ -795,23 +748,25 @@ type
 
 proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
   var pos = L.bufpos
+  var buf = L.buf                 # put `buf` in a register
   var line = L.lineNumber         # save linenumber for better error message
   tokenBegin(tok, pos - ord(mode == raw))
   inc pos # skip "
-  if L.buf[pos] == '\"' and L.buf[pos+1] == '\"':
+  if buf[pos] == '\"' and buf[pos+1] == '\"':
     tok.tokType = tkTripleStrLit # long string literal:
     inc(pos, 2)               # skip ""
     # skip leading newline:
-    if L.buf[pos] in {' ', '\t'}:
+    if buf[pos] in {' ', '\t'}:
       var newpos = pos+1
-      while L.buf[newpos] in {' ', '\t'}: inc newpos
-      if L.buf[newpos] in {CR, LF}: pos = newpos
+      while buf[newpos] in {' ', '\t'}: inc newpos
+      if buf[newpos] in {CR, LF}: pos = newpos
     pos = handleCRLF(L, pos)
+    buf = L.buf
     while true:
-      case L.buf[pos]
+      case buf[pos]
       of '\"':
-        if L.buf[pos+1] == '\"' and L.buf[pos+2] == '\"' and
-            L.buf[pos+3] != '\"':
+        if buf[pos+1] == '\"' and buf[pos+2] == '\"' and
+            buf[pos+3] != '\"':
           tokenEndIgnore(tok, pos+2)
           L.bufpos = pos + 3 # skip the three """
           break
@@ -820,6 +775,7 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
       of CR, LF:
         tokenEndIgnore(tok, pos)
         pos = handleCRLF(L, pos)
+        buf = L.buf
         add(tok.literal, "\n")
       of nimlexbase.EndOfFile:
         tokenEndIgnore(tok, pos)
@@ -830,16 +786,16 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
         L.bufpos = pos
         break
       else:
-        add(tok.literal, L.buf[pos])
+        add(tok.literal, buf[pos])
         inc(pos)
   else:
     # ordinary string literal
     if mode != normal: tok.tokType = tkRStrLit
     else: tok.tokType = tkStrLit
     while true:
-      var c = L.buf[pos]
+      var c = buf[pos]
       if c == '\"':
-        if mode != normal and L.buf[pos+1] == '\"':
+        if mode != normal and buf[pos+1] == '\"':
           inc(pos, 2)
           add(tok.literal, '"')
         else:
@@ -877,10 +833,10 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
 proc getSymbol(L: var TLexer, tok: var TToken) =
   var h: Hash = 0
   var pos = L.bufpos
+  var buf = L.buf
   tokenBegin(tok, pos)
-  var suspicious = false
   while true:
-    var c = L.buf[pos]
+    var c = buf[pos]
     case c
     of 'a'..'z', '0'..'9', '\x80'..'\xFF':
       h = h !& ord(c)
@@ -889,26 +845,21 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
       c = chr(ord(c) + (ord('a') - ord('A'))) # toLower()
       h = h !& ord(c)
       inc(pos)
-      suspicious = true
     of '_':
-      if L.buf[pos+1] notin SymChars:
+      if buf[pos+1] notin SymChars:
         lexMessage(L, errGenerated, "invalid token: trailing underscore")
         break
       inc(pos)
-      suspicious = true
     else: break
   tokenEnd(tok, pos-1)
   h = !$h
   tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
+  L.bufpos = pos
   if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
       (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
     tok.tokType = tkSymbol
   else:
     tok.tokType = TTokType(tok.ident.id + ord(tkSymbol))
-    if suspicious and {optStyleHint, optStyleError} * L.config.globalOptions != {}:
-      lintReport(L.config, getLineInfo(L), tok.ident.s.normalize, tok.ident.s)
-  L.bufpos = pos
-
 
 proc endOperator(L: var TLexer, tok: var TToken, pos: int,
                  hash: Hash) {.inline.} =
@@ -920,10 +871,11 @@ proc endOperator(L: var TLexer, tok: var TToken, pos: int,
 
 proc getOperator(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
+  var buf = L.buf
   tokenBegin(tok, pos)
   var h: Hash = 0
   while true:
-    var c = L.buf[pos]
+    var c = buf[pos]
     if c notin OpChars: break
     h = h !& ord(c)
     inc(pos)
@@ -932,10 +884,10 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   # advance pos but don't store it in L.bufpos so the next token (which might
   # be an operator too) gets the preceding spaces:
   tok.strongSpaceB = 0
-  while L.buf[pos] == ' ':
+  while buf[pos] == ' ':
     inc pos
     inc tok.strongSpaceB
-  if L.buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
+  if buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
 proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
@@ -966,7 +918,7 @@ proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
     of '?': result = 2
     else: considerAsgn(2)
   of tkDiv, tkMod, tkShl, tkShr: result = 9
-  of tkIn, tkNotin, tkIs, tkIsnot, tkOf, tkAs: result = 5
+  of tkIn, tkNotin, tkIs, tkIsnot, tkNot, tkOf, tkAs: result = 5
   of tkDotDot: result = 6
   of tkAnd: result = 4
   of tkOr, tkXor, tkPtr, tkRef: result = 3
@@ -976,8 +928,9 @@ proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
 
 proc newlineFollows*(L: TLexer): bool =
   var pos = L.bufpos
+  var buf = L.buf
   while true:
-    case L.buf[pos]
+    case buf[pos]
     of ' ', '\t':
       inc(pos)
     of CR, LF:
@@ -985,47 +938,49 @@ proc newlineFollows*(L: TLexer): bool =
       break
     of '#':
       inc(pos)
-      if L.buf[pos] == '#': inc(pos)
-      if L.buf[pos] != '[': return true
+      if buf[pos] == '#': inc(pos)
+      if buf[pos] != '[': return true
     else:
       break
 
 proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
                           isDoc: bool) =
   var pos = start
+  var buf = L.buf
   var toStrip = 0
   tokenBegin(tok, pos)
   # detect the amount of indentation:
   if isDoc:
     toStrip = getColNumber(L, pos)
-    while L.buf[pos] == ' ': inc pos
-    if L.buf[pos] in {CR, LF}:
+    while buf[pos] == ' ': inc pos
+    if buf[pos] in {CR, LF}:
       pos = handleCRLF(L, pos)
+      buf = L.buf
       toStrip = 0
-      while L.buf[pos] == ' ':
+      while buf[pos] == ' ':
         inc pos
         inc toStrip
   var nesting = 0
   while true:
-    case L.buf[pos]
+    case buf[pos]
     of '#':
       if isDoc:
-        if L.buf[pos+1] == '#' and L.buf[pos+2] == '[':
+        if buf[pos+1] == '#' and buf[pos+2] == '[':
           inc nesting
         tok.literal.add '#'
-      elif L.buf[pos+1] == '[':
+      elif buf[pos+1] == '[':
         inc nesting
       inc pos
     of ']':
       if isDoc:
-        if L.buf[pos+1] == '#' and L.buf[pos+2] == '#':
+        if buf[pos+1] == '#' and buf[pos+2] == '#':
           if nesting == 0:
             tokenEndIgnore(tok, pos+2)
             inc(pos, 3)
             break
           dec nesting
         tok.literal.add ']'
-      elif L.buf[pos+1] == '#':
+      elif buf[pos+1] == '#':
         if nesting == 0:
           tokenEndIgnore(tok, pos+1)
           inc(pos, 2)
@@ -1035,13 +990,14 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
     of CR, LF:
       tokenEndIgnore(tok, pos)
       pos = handleCRLF(L, pos)
+      buf = L.buf
       # strip leading whitespace:
       when defined(nimpretty): tok.literal.add "\L"
       if isDoc:
         when not defined(nimpretty): tok.literal.add "\n"
         inc tok.iNumber
         var c = toStrip
-        while L.buf[pos] == ' ' and c > 0:
+        while buf[pos] == ' ' and c > 0:
           inc pos
           dec c
     of nimlexbase.EndOfFile:
@@ -1049,7 +1005,7 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
       lexMessagePos(L, errGenerated, pos, "end of multiline comment expected")
       break
     else:
-      if isDoc or defined(nimpretty): tok.literal.add L.buf[pos]
+      if isDoc or defined(nimpretty): tok.literal.add buf[pos]
       inc(pos)
   L.bufpos = pos
   when defined(nimpretty):
@@ -1057,47 +1013,49 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
 
 proc scanComment(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
+  var buf = L.buf
   tok.tokType = tkComment
   # iNumber contains the number of '\n' in the token
   tok.iNumber = 0
-  assert L.buf[pos+1] == '#'
+  assert buf[pos+1] == '#'
   when defined(nimpretty):
     tok.commentOffsetA = L.offsetBase + pos - 1
 
-  if L.buf[pos+2] == '[':
+  if buf[pos+2] == '[':
     skipMultiLineComment(L, tok, pos+3, true)
     return
   tokenBegin(tok, pos)
   inc(pos, 2)
 
   var toStrip = 0
-  while L.buf[pos] == ' ':
+  while buf[pos] == ' ':
     inc pos
     inc toStrip
 
   while true:
     var lastBackslash = -1
-    while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
-      if L.buf[pos] == '\\': lastBackslash = pos+1
-      add(tok.literal, L.buf[pos])
+    while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
+      if buf[pos] == '\\': lastBackslash = pos+1
+      add(tok.literal, buf[pos])
       inc(pos)
     tokenEndIgnore(tok, pos)
     pos = handleCRLF(L, pos)
+    buf = L.buf
     var indent = 0
-    while L.buf[pos] == ' ':
+    while buf[pos] == ' ':
       inc(pos)
       inc(indent)
 
-    if L.buf[pos] == '#' and L.buf[pos+1] == '#':
+    if buf[pos] == '#' and buf[pos+1] == '#':
       tok.literal.add "\n"
       inc(pos, 2)
       var c = toStrip
-      while L.buf[pos] == ' ' and c > 0:
+      while buf[pos] == ' ' and c > 0:
         inc pos
         dec c
       inc tok.iNumber
     else:
-      if L.buf[pos] > ' ':
+      if buf[pos] > ' ':
         L.indentAhead = indent
       tokenEndIgnore(tok, pos)
       break
@@ -1107,6 +1065,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
 
 proc skip(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
+  var buf = L.buf
   tokenBegin(tok, pos)
   tok.strongSpaceA = 0
   when defined(nimpretty):
@@ -1116,22 +1075,23 @@ proc skip(L: var TLexer, tok: var TToken) =
     tok.commentOffsetB = tok.commentOffsetA
     tok.line = -1
   while true:
-    case L.buf[pos]
+    case buf[pos]
     of ' ':
       inc(pos)
       inc(tok.strongSpaceA)
     of '\t':
-      if not L.allowTabs: lexMessagePos(L, errGenerated, pos, "tabs are not allowed, use spaces instead")
+      if not L.allowTabs: lexMessagePos(L, errGenerated, pos, "tabulators are not allowed")
       inc(pos)
     of CR, LF:
       tokenEndPrevious(tok, pos)
       pos = handleCRLF(L, pos)
+      buf = L.buf
       var indent = 0
       while true:
-        if L.buf[pos] == ' ':
+        if buf[pos] == ' ':
           inc(pos)
           inc(indent)
-        elif L.buf[pos] == '#' and L.buf[pos+1] == '[':
+        elif buf[pos] == '#' and buf[pos+1] == '[':
           when defined(nimpretty):
             hasComment = true
             if tok.line < 0:
@@ -1139,30 +1099,32 @@ proc skip(L: var TLexer, tok: var TToken) =
               commentIndent = indent
           skipMultiLineComment(L, tok, pos+2, false)
           pos = L.bufpos
+          buf = L.buf
         else:
           break
       tok.strongSpaceA = 0
       when defined(nimpretty):
-        if L.buf[pos] == '#' and tok.line < 0: commentIndent = indent
-      if L.buf[pos] > ' ' and (L.buf[pos] != '#' or L.buf[pos+1] == '#'):
+        if buf[pos] == '#' and tok.line < 0: commentIndent = indent
+      if buf[pos] > ' ' and (buf[pos] != '#' or buf[pos+1] == '#'):
         tok.indent = indent
         L.currLineIndent = indent
         break
     of '#':
       # do not skip documentation comment:
-      if L.buf[pos+1] == '#': break
+      if buf[pos+1] == '#': break
       when defined(nimpretty):
         hasComment = true
         if tok.line < 0:
           tok.line = L.lineNumber
 
-      if L.buf[pos+1] == '[':
+      if buf[pos+1] == '[':
         skipMultiLineComment(L, tok, pos+2, false)
         pos = L.bufpos
+        buf = L.buf
       else:
         tokenBegin(tok, pos)
-        while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
-          when defined(nimpretty): tok.literal.add L.buf[pos]
+        while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
+          when defined(nimpretty): tok.literal.add buf[pos]
           inc(pos)
         tokenEndIgnore(tok, pos+1)
         when defined(nimpretty):
@@ -1176,6 +1138,8 @@ proc skip(L: var TLexer, tok: var TToken) =
       tok.commentOffsetB = L.offsetBase + pos - 1
       tok.tokType = tkComment
       tok.indent = commentIndent
+    if gIndentationWidth <= 0:
+      gIndentationWidth = tok.indent
 
 proc rawGetTok*(L: var TLexer, tok: var TToken) =
   template atTokenEnd() {.dirty.} =
@@ -1334,13 +1298,10 @@ proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
   var tok: TToken
   initToken(tok)
   openLexer(lex, fileIdx, inputstream, cache, config)
-  var prevToken = tkEof
-  while tok.tokType != tkEof:
+  while true:
     rawGetTok(lex, tok)
-    if tok.indent > 0 and prevToken in {tkColon, tkEquals, tkType, tkConst, tkLet, tkVar, tkUsing}:
-      result = tok.indent
-      if result > 0: break
-    prevToken = tok.tokType
+    result = tok.indent
+    if result > 0 or tok.tokType == tkEof: break
   closeLexer(lex)
 
 proc getPrecedence*(ident: PIdent): int =

compiler/liftdestructors.nim

@@ -1,656 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements lifting for type-bound operations
-## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``).
-
-# Todo:
-# - use openArray instead of array to avoid over-specializations
-
-import modulegraphs, lineinfos, idents, ast, renderer, semdata,
-  sighashes, lowerings, options, types, msgs, magicsys, tables
-
-type
-  TLiftCtx = object
-    g: ModuleGraph
-    info: TLineInfo # for construction
-    kind: TTypeAttachedOp
-    fn: PSym
-    asgnForType: PType
-    recurse: bool
-    c: PContext # c can be nil, then we are called from lambdalifting!
-
-proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode)
-proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
-              info: TLineInfo): PSym
-
-proc createTypeBoundOps*(g: ModuleGraph; c: PContext; orig: PType; info: TLineInfo)
-
-proc at(a, i: PNode, elemType: PType): PNode =
-  result = newNodeI(nkBracketExpr, a.info, 2)
-  result.sons[0] = a
-  result.sons[1] = i
-  result.typ = elemType
-
-proc fillBodyTup(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  for i in 0 ..< t.len:
-    let lit = lowerings.newIntLit(c.g, x.info, i)
-    fillBody(c, t.sons[i], body, x.at(lit, t.sons[i]), y.at(lit, t.sons[i]))
-
-proc dotField(x: PNode, f: PSym): PNode =
-  result = newNodeI(nkDotExpr, x.info, 2)
-  result.sons[0] = x
-  result.sons[1] = newSymNode(f, x.info)
-  result.typ = f.typ
-
-proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode) =
-  case n.kind
-  of nkSym:
-    let f = n.sym
-    fillBody(c, f.typ, body, x.dotField(f), y.dotField(f))
-  of nkNilLit: discard
-  of nkRecCase:
-    if c.kind in {attachedSink, attachedAsgn, attachedDeepCopy}:
-      ## the value needs to be destroyed before we assign the selector
-      ## or the value is lost
-      let prevKind = c.kind
-      c.kind = attachedDestructor
-      fillBodyObj(c, n, body, x, y)
-      c.kind = prevKind
-
-    # copy the selector:
-    fillBodyObj(c, n[0], body, x, y)
-    # we need to generate a case statement:
-    var caseStmt = newNodeI(nkCaseStmt, c.info)
-    # XXX generate 'if' that checks same branches
-    # generate selector:
-    var access = dotField(x, n[0].sym)
-    caseStmt.add(access)
-    var emptyBranches = 0
-    # copy the branches over, but replace the fields with the for loop body:
-    for i in 1 ..< n.len:
-      var branch = copyTree(n[i])
-      let L = branch.len
-      branch.sons[L-1] = newNodeI(nkStmtList, c.info)
-
-      fillBodyObj(c, n[i].lastSon, branch.sons[L-1], x, y)
-      if branch.sons[L-1].len == 0: inc emptyBranches
-      caseStmt.add(branch)
-    if emptyBranches != n.len-1:
-      body.add(caseStmt)
-  of nkRecList:
-    for t in items(n): fillBodyObj(c, t, body, x, y)
-  else:
-    illFormedAstLocal(n, c.g.config)
-
-proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
-  if t.len > 0 and t.sons[0] != nil:
-    fillBodyObjT(c, skipTypes(t.sons[0], abstractPtrs), body, x, y)
-  fillBodyObj(c, t.n, body, x, y)
-
-proc genAddr(g: ModuleGraph; x: PNode): PNode =
-  if x.kind == nkHiddenDeref:
-    checkSonsLen(x, 1, g.config)
-    result = x.sons[0]
-  else:
-    result = newNodeIT(nkHiddenAddr, x.info, makeVarType(x.typ.owner, x.typ))
-    addSon(result, x)
-
-proc newAsgnCall(g: ModuleGraph; op: PSym; x, y: PNode): PNode =
-  #if sfError in op.flags:
-  #  localError(c.config, x.info, "usage of '$1' is a user-defined error" % op.name.s)
-  result = newNodeI(nkCall, x.info)
-  result.add newSymNode(op)
-  result.add genAddr(g, x)
-  result.add y
-
-proc newAsgnStmt(le, ri: PNode): PNode =
-  result = newNodeI(nkAsgn, le.info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
-
-proc newOpCall(op: PSym; x: PNode): PNode =
-  result = newNodeIT(nkCall, x.info, op.typ.sons[0])
-  result.add(newSymNode(op))
-  result.add x
-
-proc destructorCall(g: ModuleGraph; op: PSym; x: PNode): PNode =
-  result = newNodeIT(nkCall, x.info, op.typ.sons[0])
-  result.add(newSymNode(op))
-  result.add genAddr(g, x)
-
-proc newDeepCopyCall(op: PSym; x, y: PNode): PNode =
-  result = newAsgnStmt(x, newOpCall(op, y))
-
-proc useNoGc(c: TLiftCtx; t: PType): bool {.inline.} =
-  result = optNimV2 in c.g.config.globalOptions and
-    ({tfHasGCedMem, tfHasOwned} * t.flags != {} or t.isGCedMem)
-
-proc instantiateGeneric(c: var TLiftCtx; op: PSym; t, typeInst: PType): PSym =
-  if c.c != nil and typeInst != nil:
-    result = c.c.instTypeBoundOp(c.c, op, typeInst, c.info, attachedAsgn, 1)
-  else:
-    localError(c.g.config, c.info,
-      "cannot generate destructor for generic type: " & typeToString(t))
-    result = nil
-
-proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
-                        field: var PSym): bool =
-  if optNimV2 in c.g.config.globalOptions:
-    let op = field
-    if field != nil and sfOverriden in field.flags:
-      if sfError in op.flags:
-        incl c.fn.flags, sfError
-      #else:
-      #  markUsed(c.g.config, c.info, op, c.g.usageSym)
-      onUse(c.info, op)
-      body.add newAsgnCall(c.g, op, x, y)
-      result = true
-  elif tfHasAsgn in t.flags:
-    var op: PSym
-    if sameType(t, c.asgnForType):
-      # generate recursive call:
-      if c.recurse:
-        op = c.fn
-      else:
-        c.recurse = true
-        return false
-    else:
-      op = field
-      if op == nil:
-        op = produceSym(c.g, c.c, t, c.kind, c.info)
-    if sfError in op.flags:
-      incl c.fn.flags, sfError
-    #else:
-    #  markUsed(c.g.config, c.info, op, c.g.usageSym)
-    onUse(c.info, op)
-    # We also now do generic instantiations in the destructor lifting pass:
-    if op.ast[genericParamsPos].kind != nkEmpty:
-      op = instantiateGeneric(c, op, t, t.typeInst)
-      field = op
-      #echo "trying to use ", op.ast
-      #echo "for ", op.name.s, " "
-      #debug(t)
-      #return false
-    assert op.ast[genericParamsPos].kind == nkEmpty
-    body.add newAsgnCall(c.g, op, x, y)
-    result = true
-
-proc addDestructorCall(c: var TLiftCtx; t: PType; body, x: PNode) =
-  var op = t.destructor
-  if op == nil and useNoGc(c, t):
-    op = produceSym(c.g, c.c, t, attachedDestructor, c.info)
-    doAssert op != nil
-    doAssert op == t.destructor
-
-  if op != nil:
-    #markUsed(c.g.config, c.info, op, c.g.usageSym)
-    onUse(c.info, op)
-    body.add destructorCall(c.g, op, x)
-  elif useNoGc(c, t):
-    internalError(c.g.config, c.info,
-      "type-bound operator could not be resolved")
-
-proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
-  case c.kind
-  of attachedDestructor:
-    var op = t.destructor
-    if op != nil and sfOverriden in op.flags:
-
-      if op.ast[genericParamsPos].kind != nkEmpty:
-        # patch generic destructor:
-        op = instantiateGeneric(c, op, t, t.typeInst)
-        t.attachedOps[attachedDestructor] = op
-
-      #markUsed(c.g.config, c.info, op, c.g.usageSym)
-      onUse(c.info, op)
-      body.add destructorCall(c.g, op, x)
-      result = true
-    #result = addDestructorCall(c, t, body, x)
-  of attachedAsgn:
-    result = considerAsgnOrSink(c, t, body, x, y, t.assignment)
-  of attachedSink:
-    result = considerAsgnOrSink(c, t, body, x, y, t.asink)
-  of attachedDeepCopy:
-    let op = t.attachedOps[attachedDeepCopy]
-    if op != nil:
-      #markUsed(c.g.config, c.info, op, c.g.usageSym)
-      onUse(c.info, op)
-      body.add newDeepCopyCall(op, x, y)
-      result = true
-
-proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  if c.kind != attachedDestructor:
-    body.add newAsgnStmt(x, y)
-
-proc addVar(father, v, value: PNode) =
-  var vpart = newNodeI(nkIdentDefs, v.info, 3)
-  vpart.sons[0] = v
-  vpart.sons[1] = newNodeI(nkEmpty, v.info)
-  vpart.sons[2] = value
-  addSon(father, vpart)
-
-proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
-  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.fn, c.info)
-  temp.typ = getSysType(c.g, body.info, tyInt)
-  incl(temp.flags, sfFromGeneric)
-
-  var v = newNodeI(nkVarSection, c.info)
-  result = newSymNode(temp)
-  v.addVar(result, lowerings.newIntLit(c.g, body.info, first))
-  body.add v
-
-proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode =
-  result = newNodeI(nkCall, i.info)
-  result.add createMagic(g, name, magic).newSymNode
-  result.add i
-
-proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
-  result = newNodeI(nkWhileStmt, c.info, 2)
-  let cmp = genBuiltin(c.g, mLtI, "<", i)
-  cmp.add genLen(c.g, dest)
-  cmp.typ = getSysType(c.g, c.info, tyBool)
-  result.sons[0] = cmp
-  result.sons[1] = newNodeI(nkStmtList, c.info)
-
-proc genIf(c: var TLiftCtx; cond, action: PNode): PNode =
-  result = newTree(nkIfStmt, newTree(nkElifBranch, cond, action))
-
-proc addIncStmt(c: var TLiftCtx; body, i: PNode) =
-  let incCall = genBuiltin(c.g, mInc, "inc", i)
-  incCall.add lowerings.newIntLit(c.g, c.info, 1)
-  body.add incCall
-
-proc newSeqCall(g: ModuleGraph; x, y: PNode): PNode =
-  # don't call genAddr(c, x) here:
-  result = genBuiltin(g, mNewSeq, "newSeq", x)
-  let lenCall = genBuiltin(g, mLengthSeq, "len", y)
-  lenCall.typ = getSysType(g, x.info, tyInt)
-  result.add lenCall
-
-proc setLenStrCall(g: ModuleGraph; x, y: PNode): PNode =
-  let lenCall = genBuiltin(g, mLengthStr, "len", y)
-  lenCall.typ = getSysType(g, x.info, tyInt)
-  result = genBuiltin(g, mSetLengthStr, "setLen", x) # genAddr(g, x))
-  result.add lenCall
-
-proc setLenSeqCall(c: var TLiftCtx; t: PType; x, y: PNode): PNode =
-  let lenCall = genBuiltin(c.g, mLengthSeq, "len", y)
-  lenCall.typ = getSysType(c.g, x.info, tyInt)
-  var op = getSysMagic(c.g, x.info, "setLen", mSetLengthSeq)
-  op = instantiateGeneric(c, op, t, t)
-  result = newTree(nkCall, newSymNode(op, x.info), x, lenCall)
-
-proc forallElements(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  let i = declareCounter(c, body, firstOrd(c.g.config, t))
-  let whileLoop = genWhileLoop(c, i, x)
-  let elemType = t.lastSon
-  fillBody(c, elemType, whileLoop.sons[1], x.at(i, elemType),
-                                           y.at(i, elemType))
-  addIncStmt(c, whileLoop.sons[1], i)
-  body.add whileLoop
-
-proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  case c.kind
-  of attachedAsgn, attachedDeepCopy:
-    # we generate:
-    # setLen(dest, y.len)
-    # var i = 0
-    # while i < y.len: dest[i] = y[i]; inc(i)
-    # This is usually more efficient than a destroy/create pair.
-    body.add setLenSeqCall(c, t, x, y)
-    forallElements(c, t, body, x, y)
-  of attachedSink:
-    let moveCall = genBuiltin(c.g, mMove, "move", x)
-    moveCall.add y
-    doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
-    body.add moveCall
-  of attachedDestructor:
-    # destroy all elements:
-    forallElements(c, t, body, x, y)
-    body.add genBuiltin(c.g, mDestroy, "destroy", x)
-
-proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  createTypeBoundOps(c.g, c.c, t, body.info)
-  case c.kind
-  of attachedAsgn, attachedDeepCopy:
-    doAssert t.assignment != nil
-    body.add newAsgnCall(c.g, t.assignment, x, y)
-  of attachedSink:
-    # we always inline the move for better performance:
-    let moveCall = genBuiltin(c.g, mMove, "move", x)
-    moveCall.add y
-    doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
-    body.add moveCall
-    # alternatively we could do this:
-    when false:
-      doAssert t.asink != nil
-      body.add newAsgnCall(c.g, t.asink, x, y)
-  of attachedDestructor:
-    doAssert t.destructor != nil
-    body.add destructorCall(c.g, t.destructor, x)
-
-proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  case c.kind
-  of attachedAsgn, attachedDeepCopy:
-    body.add callCodegenProc(c.g, "nimAsgnStrV2", c.info, genAddr(c.g, x), y)
-  of attachedSink:
-    let moveCall = genBuiltin(c.g, mMove, "move", x)
-    moveCall.add y
-    doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
-    body.add moveCall
-  of attachedDestructor:
-    body.add genBuiltin(c.g, mDestroy, "destroy", x)
-
-proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  case c.kind
-  of attachedSink:
-    # we 'nil' y out afterwards so we *need* to take over its reference
-    # count value:
-    body.add genIf(c, x, callCodegenProc(c.g, "nimDecWeakRef", c.info, x))
-    body.add newAsgnStmt(x, y)
-  of attachedAsgn:
-    body.add genIf(c, y, callCodegenProc(c.g, "nimIncWeakRef", c.info, y))
-    body.add genIf(c, x, callCodegenProc(c.g, "nimDecWeakRef", c.info, x))
-    body.add newAsgnStmt(x, y)
-  of attachedDestructor:
-    # it's better to prepend the destruction of weak refs in order to
-    # prevent wrong "dangling refs exist" problems:
-    let des = genIf(c, x, callCodegenProc(c.g, "nimDecWeakRef", c.info, x))
-    if body.len == 0:
-      body.add des
-    else:
-      body.sons.insert(des, 0)
-  of attachedDeepCopy: assert(false, "cannot happen")
-
-proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  var actions = newNodeI(nkStmtList, c.info)
-
-  let elemType = t.lastSon
-  #fillBody(c, elemType, actions, genDeref(x), genDeref(y))
-  #var disposeCall = genBuiltin(c.g, mDispose, "dispose", x)
-
-  if isFinal(elemType):
-    addDestructorCall(c, elemType, actions, genDeref(x, nkDerefExpr))
-    actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x)
-  else:
-    addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(x, nkDerefExpr))
-    actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, x)
-
-  case c.kind
-  of attachedSink, attachedAsgn:
-    body.add genIf(c, x, actions)
-    body.add newAsgnStmt(x, y)
-  of attachedDestructor:
-    body.add genIf(c, x, actions)
-  of attachedDeepCopy: assert(false, "cannot happen")
-
-proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  if c.kind == attachedDeepCopy:
-    # a big problem is that we don't know the enviroment's type here, so we
-    # have to go through some indirection; we delegate this to the codegen:
-    let call = newNodeI(nkCall, c.info, 2)
-    call.typ = t
-    call.sons[0] = newSymNode(createMagic(c.g, "deepCopy", mDeepCopy))
-    call.sons[1] = y
-    body.add newAsgnStmt(x, call)
-  elif optNimV2 in c.g.config.globalOptions and
-      optRefCheck in c.g.config.options:
-    let xx = genBuiltin(c.g, mAccessEnv, "accessEnv", x)
-    xx.typ = getSysType(c.g, c.info, tyPointer)
-    case c.kind
-    of attachedSink:
-      # we 'nil' y out afterwards so we *need* to take over its reference
-      # count value:
-      body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
-      body.add newAsgnStmt(x, y)
-    of attachedAsgn:
-      let yy = genBuiltin(c.g, mAccessEnv, "accessEnv", y)
-      yy.typ = getSysType(c.g, c.info, tyPointer)
-      body.add genIf(c, yy, callCodegenProc(c.g, "nimIncWeakRef", c.info, yy))
-      body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
-      body.add newAsgnStmt(x, y)
-    of attachedDestructor:
-      let des = genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
-      if body.len == 0:
-        body.add des
-      else:
-        body.sons.insert(des, 0)
-    of attachedDeepCopy: assert(false, "cannot happen")
-
-proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  let xx = genBuiltin(c.g, mAccessEnv, "accessEnv", x)
-  xx.typ = getSysType(c.g, c.info, tyPointer)
-  var actions = newNodeI(nkStmtList, c.info)
-  let elemType = t.lastSon
-  #discard addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(xx))
-  actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, xx)
-  case c.kind
-  of attachedSink, attachedAsgn:
-    body.add genIf(c, xx, actions)
-    body.add newAsgnStmt(x, y)
-  of attachedDestructor:
-    body.add genIf(c, xx, actions)
-  of attachedDeepCopy: assert(false, "cannot happen")
-
-proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  case t.kind
-  of tyNone, tyEmpty, tyVoid: discard
-  of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString,
-      tyPtr, tyOpt, tyUncheckedArray:
-    defaultOp(c, t, body, x, y)
-  of tyRef:
-    if optNimV2 in c.g.config.globalOptions and
-        optRefCheck in c.g.config.options:
-      weakrefOp(c, t, body, x, y)
-    else:
-      defaultOp(c, t, body, x, y)
-  of tyProc:
-    if t.callConv == ccClosure:
-      closureOp(c, t, body, x, y)
-    else:
-      defaultOp(c, t, body, x, y)
-  of tyOwned:
-    let base = t.skipTypes(abstractInstOwned)
-    if optNimV2 in c.g.config.globalOptions:
-      case base.kind
-      of tyRef:
-        ownedRefOp(c, base, body, x, y)
-        return
-      of tyProc:
-        if base.callConv == ccClosure:
-          ownedClosureOp(c, base, body, x, y)
-          return
-      else: discard
-    defaultOp(c, base, body, x, y)
-  of tyArray:
-    if tfHasAsgn in t.flags or useNoGc(c, t):
-      forallElements(c, t, body, x, y)
-    else:
-      defaultOp(c, t, body, x, y)
-  of tySequence:
-    if useNoGc(c, t):
-      useSeqOrStrOp(c, t, body, x, y)
-    elif c.g.config.selectedGC == gcDestructors:
-      # note that tfHasAsgn is propagated so we need the check on
-      # 'selectedGC' here to determine if we have the new runtime.
-      discard considerUserDefinedOp(c, t, body, x, y)
-    elif tfHasAsgn in t.flags:
-      if c.kind != attachedDestructor:
-        body.add newSeqCall(c.g, x, y)
-      forallElements(c, t, body, x, y)
-    else:
-      defaultOp(c, t, body, x, y)
-  of tyString:
-    if useNoGc(c, t):
-      useSeqOrStrOp(c, t, body, x, y)
-    elif tfHasAsgn in t.flags:
-      discard considerUserDefinedOp(c, t, body, x, y)
-    else:
-      defaultOp(c, t, body, x, y)
-  of tyObject:
-    if not considerUserDefinedOp(c, t, body, x, y):
-      fillBodyObjT(c, t, body, x, y)
-  of tyDistinct:
-    if not considerUserDefinedOp(c, t, body, x, y):
-      fillBody(c, t.sons[0].skipTypes(skipPtrs), body, x, y)
-  of tyTuple:
-    fillBodyTup(c, t, body, x, y)
-  of tyVarargs, tyOpenArray:
-    localError(c.g.config, c.info, "cannot copy openArray")
-  of tyFromExpr, tyProxy, tyBuiltInTypeClass, tyUserTypeClass,
-     tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything,
-     tyGenericParam, tyGenericBody, tyNil, tyUntyped, tyTyped,
-     tyTypeDesc, tyGenericInvocation, tyForward:
-    #internalError(c.g.config, c.info, "assignment requested for type: " & typeToString(t))
-    discard
-  of tyVar, tyLent:
-    if c.kind != attachedDestructor:
-      fillBody(c, lastSon(t), body, x, y)
-  of tyOrdinal, tyRange, tyInferred,
-     tyGenericInst, tyStatic, tyAlias, tySink:
-    fillBody(c, lastSon(t), body, x, y)
-
-proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; info: TLineInfo): PSym =
-  assert typ.kind == tyDistinct
-  let baseType = typ[0]
-  if baseType.attachedOps[kind] == nil:
-    discard produceSym(g, c, baseType, kind, info)
-  typ.attachedOps[kind] = baseType.attachedOps[kind]
-  result = typ.attachedOps[kind]
-
-proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
-              info: TLineInfo): PSym =
-  if typ.kind == tyDistinct:
-    return produceSymDistinctType(g, c, typ, kind, info)
-
-  var a: TLiftCtx
-  a.info = info
-  a.g = g
-  a.kind = kind
-  a.c = c
-  let body = newNodeI(nkStmtList, info)
-  let procname = getIdent(g.cache, AttachedOpToStr[kind])
-
-  result = newSym(skProc, procname, typ.owner, info)
-  a.fn = result
-  a.asgnForType = typ
-
-  let dest = newSym(skParam, getIdent(g.cache, "dest"), result, info)
-  let src = newSym(skParam, getIdent(g.cache, "src"), result, info)
-  dest.typ = makeVarType(typ.owner, typ)
-  src.typ = typ
-
-  result.typ = newProcType(info, typ.owner)
-  result.typ.addParam dest
-  if kind != attachedDestructor:
-    result.typ.addParam src
-
-  # register this operation already:
-  typ.attachedOps[kind] = result
-
-  var tk: TTypeKind
-  if optNimV2 in g.config.globalOptions:
-    tk = skipTypes(typ, {tyOrdinal, tyRange, tyInferred, tyGenericInst, tyStatic, tyAlias, tySink}).kind
-  else:
-    tk = tyNone # no special casing for strings and seqs
-  case tk
-  of tySequence:
-    fillSeqOp(a, typ, body, newSymNode(dest).newDeref, newSymNode(src))
-  of tyString:
-    fillStrOp(a, typ, body, newSymNode(dest).newDeref, newSymNode(src))
-  else:
-    fillBody(a, typ, body, newSymNode(dest).newDeref, newSymNode(src))
-
-  var n = newNodeI(nkProcDef, info, bodyPos+1)
-  for i in 0 ..< n.len: n.sons[i] = newNodeI(nkEmpty, info)
-  n.sons[namePos] = newSymNode(result)
-  n.sons[paramsPos] = result.typ.n
-  n.sons[bodyPos] = body
-  result.ast = n
-  incl result.flags, sfFromGeneric
-  incl result.flags, sfGeneratedOp
-
-template liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
-  discard "now a nop"
-
-proc patchBody(g: ModuleGraph; c: PContext; n: PNode; info: TLineInfo) =
-  if n.kind in nkCallKinds:
-    if n[0].kind == nkSym and n[0].sym.magic == mDestroy:
-      let t = n[1].typ.skipTypes(abstractVar)
-      if t.destructor == nil:
-        discard produceSym(g, c, t, attachedDestructor, info)
-
-      if t.destructor != nil:
-        if t.destructor.ast[genericParamsPos].kind != nkEmpty:
-          internalError(g.config, info, "resolved destructor is generic")
-        if t.destructor.magic == mDestroy:
-          internalError(g.config, info, "patching mDestroy with mDestroy?")
-        n.sons[0] = newSymNode(t.destructor)
-  for x in n: patchBody(g, c, x, info)
-
-template inst(field, t) =
-  if field.ast != nil and field.ast[genericParamsPos].kind != nkEmpty:
-    if t.typeInst != nil:
-      var a: TLiftCtx
-      a.info = info
-      a.g = g
-      a.kind = k
-      a.c = c
-
-      field = instantiateGeneric(a, field, t, t.typeInst)
-      if field.ast != nil:
-        patchBody(g, c, field.ast, info)
-    else:
-      localError(g.config, info, "unresolved generic parameter")
-
-proc isTrival(s: PSym): bool {.inline.} = s == nil or s.ast[bodyPos].len == 0
-
-proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInfo) =
-  ## In the semantic pass this is called in strategic places
-  ## to ensure we lift assignment, destructors and moves properly.
-  ## The later 'injectdestructors' pass depends on it.
-  if orig == nil or {tfCheckedForDestructor, tfHasMeta} * orig.skipTypes({tyAlias}).flags != {}: return
-  incl orig.flags, tfCheckedForDestructor
-
-  let h = sighashes.hashType(orig, {CoType, CoConsiderOwned, CoDistinct})
-  var canon = g.canonTypes.getOrDefault(h)
-  var overwrite = false
-  if canon == nil:
-    let typ = orig.skipTypes({tyGenericInst, tyAlias})
-    g.canonTypes[h] = typ
-    canon = typ
-  elif canon != orig:
-    overwrite = true
-
-  # multiple cases are to distinguish here:
-  # 1. we don't know yet if 'typ' has a nontrival destructor.
-  # 2. we have a nop destructor. --> mDestroy
-  # 3. we have a lifted destructor.
-  # 4. We have a custom destructor.
-  # 5. We have a (custom) generic destructor.
-
-  # we generate the destructor first so that other operators can depend on it:
-  for k in attachedDestructor..attachedSink:
-    if canon.attachedOps[k] == nil:
-      discard produceSym(g, c, canon, k, info)
-    else:
-      inst(canon.attachedOps[k], canon)
-
-  if overwrite:
-    for k in attachedDestructor..attachedSink:
-      orig.attachedOps[k] = canon.attachedOps[k]
-
-  if not isTrival(orig.destructor):
-    #or not isTrival(orig.assignment) or
-    # not isTrival(orig.sink):
-    orig.flags.incl tfHasAsgn

compiler/liftlocals.nim

@@ -10,7 +10,7 @@
 ## This module implements the '.liftLocals' pragma.
 
 import
-  strutils, options, ast, msgs,
+  intsets, strutils, options, ast, astalgo, msgs,
   idents, renderer, types, lowerings, lineinfos
 
 from pragmas import getPragmaVal

compiler/lineinfos.nim

@@ -33,10 +33,10 @@ type
     warnFieldXNotSupported, warnCommentXIgnored,
     warnTypelessParam,
     warnUseBase, warnWriteToForeignHeap, warnUnsafeCode,
-    warnEachIdentIsTuple,
+    warnEachIdentIsTuple, warnShadowIdent,
     warnProveInit, warnProveField, warnProveIndex, warnGcUnsafe, warnGcUnsafe2,
     warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed,
-    warnInconsistentSpacing, warnCaseTransition, warnUser,
+    warnInconsistentSpacing, warnUser,
     hintSuccess, hintSuccessX, hintCC,
     hintLineTooLong, hintXDeclaredButNotUsed, hintConvToBaseNotNeeded,
     hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled,
@@ -44,7 +44,7 @@ type
     hintConditionAlwaysTrue, hintConditionAlwaysFalse, hintName, hintPattern,
     hintExecuting, hintLinking, hintDependency,
     hintSource, hintPerformance, hintStackTrace, hintGCStats,
-    hintGlobalVar, hintExpandMacro,
+    hintGlobalVar,
     hintUser, hintUserRaw,
     hintExtendedContext
 
@@ -65,7 +65,7 @@ const
     warnOctalEscape: "octal escape sequences do not exist; leading zero is ignored",
     warnXIsNeverRead: "'$1' is never read",
     warnXmightNotBeenInit: "'$1' might not have been initialized",
-    warnDeprecated: "$1",
+    warnDeprecated: "$1 is deprecated",
     warnConfigDeprecated: "config file '$1' is deprecated",
     warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)",
     warnUnknownMagic: "unknown magic '$1' might crash the compiler",
@@ -79,6 +79,7 @@ const
     warnWriteToForeignHeap: "write to foreign heap",
     warnUnsafeCode: "unsafe code: '$1'",
     warnEachIdentIsTuple: "each identifier is a tuple",
+    warnShadowIdent: "shadowed identifier: '$1'",
     warnProveInit: "Cannot prove that '$1' is initialized. This will become a compile time error in the future.",
     warnProveField: "cannot prove that field '$1' is accessible",
     warnProveIndex: "cannot prove index '$1' is valid",
@@ -90,7 +91,6 @@ const
     warnLockLevel: "$1",
     warnResultShadowed: "Special variable 'result' is shadowed.",
     warnInconsistentSpacing: "Number of spaces around '$#' is not consistent",
-    warnCaseTransition: "Potential object case transition, instantiate new object instead",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     hintSuccessX: "operation successful ($# lines compiled; $# sec total; $#; $#)",
@@ -108,7 +108,7 @@ const
     hintPath: "added path: '$1'",
     hintConditionAlwaysTrue: "condition is always true: '$1'",
     hintConditionAlwaysFalse: "condition is always false: '$1'",
-    hintName: "$1",
+    hintName: "name should be: '$1'",
     hintPattern: "$1",
     hintExecuting: "$1",
     hintLinking: "",
@@ -118,7 +118,6 @@ const
     hintStackTrace: "$1",
     hintGCStats: "$1",
     hintGlobalVar: "global variable declared here",
-    hintExpandMacro: "expanded macro: $1",
     hintUser: "$1",
     hintUserRaw: "$1",
     hintExtendedContext: "$1",
@@ -133,17 +132,17 @@ const
     "LanguageXNotSupported", "FieldXNotSupported",
     "CommentXIgnored",
     "TypelessParam", "UseBase", "WriteToForeignHeap",
-    "UnsafeCode", "EachIdentIsTuple",
+    "UnsafeCode", "EachIdentIsTuple", "ShadowIdent",
     "ProveInit", "ProveField", "ProveIndex", "GcUnsafe", "GcUnsafe2", "Uninit",
     "GcMem", "Destructor", "LockLevel", "ResultShadowed",
-    "Spacing", "CaseTransition", "User"]
+    "Spacing", "User"]
 
   HintsToStr* = [
     "Success", "SuccessX", "CC", "LineTooLong",
     "XDeclaredButNotUsed", "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded",
     "ExprAlwaysX", "QuitCalled", "Processing", "CodeBegin", "CodeEnd", "Conf",
     "Path", "CondTrue", "CondFalse", "Name", "Pattern", "Exec", "Link", "Dependency",
-    "Source", "Performance", "StackTrace", "GCStats", "GlobalVar", "ExpandMacro",
+    "Source", "Performance", "StackTrace", "GCStats", "GlobalVar",
     "User", "UserRaw", "ExtendedContext",
   ]
 
@@ -168,11 +167,11 @@ type
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
   result[3] = {low(TNoteKind)..high(TNoteKind)} - {}
   result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext}
-  result[1] = result[2] - {warnProveField, warnProveIndex,
+  result[1] = result[2] - {warnShadowIdent, warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
     hintSource, hintGlobalVar, hintGCStats}
   result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf,
-    hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC}
+    hintProcessing, hintPattern, hintExecuting, hintLinking}
 
 const
   NotesVerbosity* = computeNotesVerbosity()
@@ -193,7 +192,7 @@ type
                                #   used for better error messages and
                                #   embedding the original source in the
                                #   generated code
-    dirtyFile*: AbsoluteFile   # the file that is actually read into memory
+    dirtyfile*: AbsoluteFile   # the file that is actually read into memory
                                # and parsed; usually "" but is used
                                # for 'nimsuggest'
     hash*: string              # the checksum of the file
@@ -225,16 +224,16 @@ type
 
 proc `==`*(a, b: FileIndex): bool {.borrow.}
 
-proc raiseRecoverableError*(msg: string) {.noinline.} =
+proc raiseRecoverableError*(msg: string) {.noinline, noreturn.} =
   raise newException(ERecoverableError, msg)
 
 const
-  InvalidFileIdx* = FileIndex(-1)
+  InvalidFileIDX* = FileIndex(-1)
 
 proc unknownLineInfo*(): TLineInfo =
   result.line = uint16(0)
   result.col = int16(-1)
-  result.fileIndex = InvalidFileIdx
+  result.fileIndex = InvalidFileIDX
 
 type
   Severity* {.pure.} = enum ## VS Code only supports these three
@@ -250,7 +249,7 @@ type
                             ## some close token.
 
     errorOutputs*: TErrorOutputs
-    msgContext*: seq[tuple[info: TLineInfo, detail: string]]
+    msgContext*: seq[TLineInfo]
     lastError*: TLineInfo
     filenameToIndexTbl*: Table[string, FileIndex]
     fileInfos*: seq[TFileInfo]

compiler/linter.nim

@@ -7,11 +7,14 @@
 #    distribution, for details about the copyright.
 #
 
-## This module implements the style checker.
+## This module implements the code "prettifier". This is part of the toolchain
+## to convert Nim code into a consistent style.
 
-import strutils
+import
+  strutils, os, intsets, strtabs
 
-import options, ast, msgs, idents, lineinfos, wordrecg
+import options, ast, astalgo, msgs, semdata, ropes, idents,
+  lineinfos, pathutils
 
 const
   Letters* = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF', '_'}
@@ -20,6 +23,36 @@ proc identLen*(line: string, start: int): int =
   while start+result < line.len and line[start+result] in Letters:
     inc result
 
+when false:
+  import prettybase
+
+type
+  StyleCheck* {.pure.} = enum None, Warn, Auto
+
+var
+  gOverWrite* = true
+  gStyleCheck*: StyleCheck
+  gCheckExtern*, gOnlyMainfile*: bool
+
+proc overwriteFiles*(conf: ConfigRef) =
+  let doStrip = options.getConfigVar(conf, "pretty.strip").normalize == "on"
+  for i in 0 .. high(conf.m.fileInfos):
+    if conf.m.fileInfos[i].dirty and
+        (not gOnlyMainfile or FileIndex(i) == conf.projectMainIdx):
+      let newFile = if gOverWrite: conf.m.fileInfos[i].fullpath
+                    else: conf.m.fileInfos[i].fullpath.changeFileExt(".pretty.nim")
+      try:
+        var f = open(newFile.string, fmWrite)
+        for line in conf.m.fileInfos[i].lines:
+          if doStrip:
+            f.write line.strip(leading = false, trailing = true)
+          else:
+            f.write line
+          f.write(conf.m.fileInfos[i], "\L")
+        f.close
+      except IOError:
+        rawMessage(conf, errGenerated, "cannot open file: " & newFile.string)
+
 proc `=~`(s: string, a: openArray[string]): bool =
   for x in a:
     if s.startsWith(x): return true
@@ -38,9 +71,9 @@ proc beautifyName(s: string, k: TSymKind): string =
     if s =~ ["int", "uint", "cint", "cuint", "clong", "cstring", "string",
              "char", "byte", "bool", "openArray", "seq", "array", "void",
              "pointer", "float", "csize", "cdouble", "cchar", "cschar",
-             "cshort", "cu", "nil", "typedesc", "auto", "any",
-             "range", "openarray", "varargs", "set", "cfloat", "ref", "ptr",
-             "untyped", "typed", "static", "sink", "lent", "type", "owned"]:
+             "cshort", "cu", "nil", "expr", "stmt", "typedesc", "auto", "any",
+             "range", "openarray", "varargs", "set", "cfloat"
+             ]:
       result.add s[i]
     else:
       result.add toUpperAscii(s[i])
@@ -69,74 +102,75 @@ proc beautifyName(s: string, k: TSymKind): string =
       result.add s[i]
     inc i
 
-proc differ*(line: string, a, b: int, x: string): string =
-  proc substrEq(s: string, pos, last: int, substr: string): bool =
-    var i = 0
-    var length = substr.len
-    while i < length and pos+i <= last and s[pos+i] == substr[i]:
-      inc i
-    return i == length
+proc differ*(line: string, a, b: int, x: string): bool =
+  let y = line[a..b]
+  result = cmpIgnoreStyle(y, x) == 0 and y != x
 
-  let last = min(b, line.len)
+proc replaceInFile(conf: ConfigRef; info: TLineInfo; newName: string) =
+  let line = conf.m.fileInfos[info.fileIndex.int].lines[info.line.int-1]
+  var first = min(info.col.int, line.len)
+  if first < 0: return
+  #inc first, skipIgnoreCase(line, "proc ", first)
+  while first > 0 and line[first-1] in Letters: dec first
+  if first < 0: return
+  if line[first] == '`': inc first
 
-  result = ""
-  if not substrEq(line, a, b, x):
-    let y = line[a..b]
-    if cmpIgnoreStyle(y, x) == 0:
-      result = y
+  let last = first+identLen(line, first)-1
+  if differ(line, first, last, newName):
+    # last-first+1 != newName.len or
+    var x = line.substr(0, first-1) & newName & line.substr(last+1)
+    system.shallowCopy(conf.m.fileInfos[info.fileIndex.int].lines[info.line.int-1], x)
+    conf.m.fileInfos[info.fileIndex.int].dirty = true
 
 proc checkStyle(conf: ConfigRef; cache: IdentCache; info: TLineInfo, s: string, k: TSymKind; sym: PSym) =
   let beau = beautifyName(s, k)
   if s != beau:
-    lintReport(conf, info, beau, s)
+    if gStyleCheck == StyleCheck.Auto:
+      sym.name = getIdent(cache, beau)
+      replaceInFile(conf, info, beau)
+    else:
+      message(conf, info, hintName, beau)
+
+proc styleCheckDefImpl(conf: ConfigRef; cache: IdentCache; info: TLineInfo; s: PSym; k: TSymKind) =
+  # operators stay as they are:
+  if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
+  if k in {skType, skGenericParam} and sfAnon in s.flags: return
+  if {sfImportc, sfExportc} * s.flags == {} or gCheckExtern:
+    checkStyle(conf, cache, info, s.name.s, k, s)
 
 proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
   # operators stay as they are:
   if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
   if k in {skType, skGenericParam} and sfAnon in s.flags: return
-  if s.typ != nil and s.typ.kind == tyTypeDesc: return
-  if {sfImportc, sfExportc} * s.flags != {}: return
-  if optStyleCheck notin s.options: return
   let beau = beautifyName(s.name.s, k)
   if s.name.s != beau:
-    lintReport(conf, info, beau, s.name.s)
+    message(conf, info, hintName, beau)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
-  if {optStyleHint, optStyleError} * conf.globalOptions != {}:
+  if optCheckNep1 in conf.globalOptions:
     nep1CheckDefImpl(conf, info, s, k)
+  when defined(nimfix):
+    if gStyleCheck != StyleCheck.None: styleCheckDefImpl(conf, cache, info, s, k)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) =
   styleCheckDef(conf, info, s, s.kind)
 template styleCheckDef*(conf: ConfigRef; s: PSym) =
   styleCheckDef(conf, s.info, s, s.kind)
 
-proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string =
-  let line = sourceLine(conf, info)
-  var first = min(info.col.int, line.len)
-  if first < 0: return
-  #inc first, skipIgnoreCase(line, "proc ", first)
-  while first > 0 and line[first-1] in Letters: dec first
-  if first < 0: return
-  if first+1 < line.len and line[first] == '`': inc first
-
-  let last = first+identLen(line, first)-1
-  result = differ(line, first, last, newName)
-
-proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) =
+proc styleCheckUseImpl(conf: ConfigRef; info: TLineInfo; s: PSym) =
   if info.fileIndex.int < 0: return
   # we simply convert it to what it looks like in the definition
   # for consistency
 
   # operators stay as they are:
-  if s.kind == skTemp or s.name.s[0] notin Letters or sfAnon in s.flags:
+  if s.kind in {skResult, skTemp} or s.name.s[0] notin Letters:
     return
-
+  if s.kind in {skType, skGenericParam} and sfAnon in s.flags: return
   let newName = s.name.s
-  let oldName = differs(conf, info, newName)
-  if oldName.len > 0:
-    lintReport(conf, info, newName, oldName)
 
-proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
-  let wanted = canonPragmaSpelling(w)
-  if pragmaName != wanted:
-    lintReport(conf, info, wanted, pragmaName)
+  replaceInFile(conf, info, newName)
+  #if newName == "File": writeStackTrace()
+
+template styleCheckUse*(info: TLineInfo; s: PSym) =
+  when defined(nimfix):
+    if gStyleCheck != StyleCheck.None: styleCheckUseImpl(conf, info, s)

compiler/lists.nim

@@ -0,0 +1,28 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2012 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# This module is deprecated, don't use it.
+# TODO Remove this
+
+import os
+
+static:
+  echo "WARNING: imported deprecated module compiler/lists.nim, use seq ore lists from the standard library"
+
+proc appendStr*(list: var seq[string]; data: string) {.deprecated.} =
+  # just use system.add
+  list.add(data)
+
+proc includeStr(list: var seq[string]; data: string): bool {.deprecated.} =
+  if list.contains(data):
+    result = true
+  else:
+    result = false
+    list.add data
+

compiler/llstream.nim

@@ -10,14 +10,13 @@
 ## Low-level streams for high performance.
 
 import
-  pathutils
+  strutils, pathutils
 
 # support '-d:useGnuReadline' for backwards compatibility:
 when not defined(windows) and (defined(useGnuReadline) or defined(useLinenoise)):
   import rdstdin
 
 type
-  TLLRepl* = proc (s: PLLStream, buf: pointer, bufLen: int): int
   TLLStreamKind* = enum       # enum of different stream implementations
     llsNone,                  # null stream: reading and writing has no effect
     llsString,                # stream encapsulates a string
@@ -29,7 +28,6 @@ type
     s*: string
     rd*, wr*: int             # for string streams
     lineOffset*: int          # for fake stdin line numbers
-    repl*: TLLRepl            # gives stdin control to clients
 
   PLLStream* = ref TLLStream
 
@@ -52,13 +50,11 @@ proc llStreamOpen*(): PLLStream =
   new(result)
   result.kind = llsNone
 
-proc llReadFromStdin(s: PLLStream, buf: pointer, bufLen: int): int
-proc llStreamOpenStdIn*(r: TLLRepl = llReadFromStdin): PLLStream =
+proc llStreamOpenStdIn*(): PLLStream =
   new(result)
   result.kind = llsStdIn
   result.s = ""
   result.lineOffset = -1
-  result.repl = r
 
 proc llStreamClose*(s: PLLStream) =
   case s.kind
@@ -70,10 +66,10 @@ proc llStreamClose*(s: PLLStream) =
 when not declared(readLineFromStdin):
   # fallback implementation:
   proc readLineFromStdin(prompt: string, line: var string): bool =
-    stderr.write(prompt)
+    stdout.write(prompt)
     result = readLine(stdin, line)
     if not result:
-      stderr.write("\n")
+      stdout.write("\n")
       quit(0)
 
 proc endsWith*(x: string, s: set[char]): bool =
@@ -97,7 +93,7 @@ proc continueLine(line: string, inTripleString: bool): bool {.inline.} =
 
 proc countTriples(s: string): int =
   var i = 0
-  while i+2 < s.len:
+  while i < s.len:
     if s[i] == '"' and s[i+1] == '"' and s[i+2] == '"':
       inc result
       inc i, 2
@@ -131,7 +127,7 @@ proc llStreamRead*(s: PLLStream, buf: pointer, bufLen: int): int =
   of llsFile:
     result = readBuffer(s.f, buf, bufLen)
   of llsStdIn:
-    result = s.repl(s, buf, bufLen)
+    result = llReadFromStdin(s, buf, bufLen)
 
 proc llStreamReadLine*(s: PLLStream, line: var string): bool =
   setLen(line, 0)

compiler/lookups.nim

@@ -11,7 +11,7 @@
 
 import
   intsets, ast, astalgo, idents, semdata, types, msgs, options,
-  renderer, nimfix/prettybase, lineinfos, strutils
+  renderer, wordrecg, idgen, nimfix/prettybase, lineinfos, strutils
 
 proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope)
 
@@ -89,7 +89,7 @@ proc skipAlias*(s: PSym; n: PNode; conf: ConfigRef): PSym =
       prettybase.replaceDeprecated(conf, n.info, s, result)
     else:
       message(conf, n.info, warnDeprecated, "use " & result.name.s & " instead; " &
-              s.name.s & " is deprecated")
+              s.name.s)
 
 proc localSearchInScope*(c: PContext, s: PIdent): PSym =
   result = strTableGet(c.currentScope.symbols, s)
@@ -150,7 +150,7 @@ type
 
 proc getSymRepr*(conf: ConfigRef; s: PSym): string =
   case s.kind
-  of routineKinds, skType:
+  of skProc, skFunc, skMethod, skConverter, skIterator:
     result = getProcHeader(conf, s)
   else:
     result = s.name.s
@@ -168,13 +168,13 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) =
         localError(c.config, s.info, "implementation of '$1' expected" %
             getSymRepr(c.config, s))
       inc missingImpls
-    elif {sfUsed, sfExported} * s.flags == {}:
-      if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam, skEnumField}:
+    elif {sfUsed, sfExported} * s.flags == {} and optHints in s.options:
+      if s.kind notin {skForVar, skParam, skMethod, skUnknown, skGenericParam}:
         # XXX: implicit type params are currently skTypes
         # maybe they can be made skGenericParam as well.
         if s.typ != nil and tfImplicitTypeParam notin s.typ.flags and
            s.typ.kind != tyGenericParam:
-          message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s)
+          message(c.config, s.info, hintXDeclaredButNotUsed, getSymRepr(c.config, s))
     s = nextIter(it, scope.symbols)
 
 proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string;
@@ -190,7 +190,7 @@ proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) =
     wrongRedefinition(c, info, sym.name.s, conflict.info)
 
 proc addDecl*(c: PContext, sym: PSym) =
-  let conflict = strTableInclReportConflict(c.currentScope.symbols, sym, true)
+  let conflict = c.currentScope.addUniqueSym(sym)
   if conflict != nil:
     wrongRedefinition(c, sym.info, sym.name.s, conflict.info)
 
@@ -255,10 +255,9 @@ proc errorUseQualifier*(c: PContext; info: TLineInfo; s: PSym) =
   var candidate = initIdentIter(ti, c.importTable.symbols, s.name)
   var i = 0
   while candidate != nil:
-    if i == 0: err.add " -- use one of the following:\n"
-    else: err.add "\n"
-    err.add "  " & candidate.owner.name.s & "." & candidate.name.s
-    err.add ": " & typeToString(candidate.typ)
+    if i == 0: err.add " --use "
+    else: err.add " or "
+    err.add candidate.owner.name.s & "." & candidate.name.s
     candidate = nextIdentIter(ti, c.importTable.symbols)
     inc i
   localError(c.config, info, errGenerated, err)

compiler/lowerings.nim

@@ -14,26 +14,19 @@ const
 
 import ast, astalgo, types, idents, magicsys, msgs, options, modulegraphs,
   lineinfos
+from trees import getMagic
 
 proc newDeref*(n: PNode): PNode {.inline.} =
   result = newNodeIT(nkHiddenDeref, n.info, n.typ.sons[0])
   addSon(result, n)
 
 proc newTupleAccess*(g: ModuleGraph; tup: PNode, i: int): PNode =
-  if tup.kind == nkHiddenAddr:
-    result = newNodeIT(nkHiddenAddr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar}))
-    result.addSon(newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar}).sons[i]))
-    addSon(result[0], tup[0])
-    var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
-    lit.intVal = i
-    addSon(result[0], lit)
-  else:
-    result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(
-                       abstractInst).sons[i])
-    addSon(result, copyTree(tup))
-    var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
-    lit.intVal = i
-    addSon(result, lit)
+  result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(
+                     abstractInst).sons[i])
+  addSon(result, copyTree(tup))
+  var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
+  lit.intVal = i
+  addSon(result, lit)
 
 proc addVar*(father, v: PNode) =
   var vpart = newNodeI(nkIdentDefs, v.info, 3)
@@ -42,12 +35,12 @@ proc addVar*(father, v: PNode) =
   vpart.sons[2] = vpart[1]
   addSon(father, vpart)
 
-proc newAsgnStmt*(le, ri: PNode): PNode =
+proc newAsgnStmt(le, ri: PNode): PNode =
   result = newNodeI(nkAsgn, le.info, 2)
   result.sons[0] = le
   result.sons[1] = ri
 
-proc newFastAsgnStmt*(le, ri: PNode): PNode =
+proc newFastAsgnStmt(le, ri: PNode): PNode =
   result = newNodeI(nkFastAsgn, le.info, 2)
   result.sons[0] = le
   result.sons[1] = ri
@@ -71,22 +64,6 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
     if n.sons[i].kind == nkSym: v.addVar(n.sons[i])
     result.add newAsgnStmt(n.sons[i], newTupleAccess(g, tempAsNode, i))
 
-proc evalOnce*(g: ModuleGraph; value: PNode; owner: PSym): PNode =
-  ## Turns (value) into (let tmp = value; tmp) so that 'value' can be re-used
-  ## freely, multiple times. This is frequently required and such a builtin would also be
-  ## handy to have in macros.nim. The value that can be reused is 'result.lastSon'!
-  result = newNodeIT(nkStmtListExpr, value.info, value.typ)
-  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), owner, value.info, g.config.options)
-  temp.typ = skipTypes(value.typ, abstractInst)
-  incl(temp.flags, sfFromGeneric)
-
-  var v = newNodeI(nkLetSection, value.info)
-  let tempAsNode = newSymNode(temp)
-  v.addVar(tempAsNode)
-  result.add(v)
-  result.add newAsgnStmt(tempAsNode, value)
-  result.add tempAsNode
-
 proc newTupleAccessRaw*(tup: PNode, i: int): PNode =
   result = newNodeI(nkBracketExpr, tup.info)
   addSon(result, copyTree(tup))
@@ -95,13 +72,13 @@ proc newTupleAccessRaw*(tup: PNode, i: int): PNode =
   addSon(result, lit)
 
 proc newTryFinally*(body, final: PNode): PNode =
-  result = newTree(nkHiddenTryStmt, body, newTree(nkFinally, final))
+  result = newTree(nkTryStmt, body, newTree(nkFinally, final))
 
 proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   let value = n.lastSon
   result = newNodeI(nkStmtList, n.info)
 
-  var temp = newSym(skTemp, getIdent(g.cache, "_"), owner, value.info, owner.options)
+  var temp = newSym(skLet, getIdent(g.cache, "_"), owner, value.info, owner.options)
   var v = newNodeI(nkLetSection, value.info)
   let tempAsNode = newSymNode(temp) #newIdentNode(getIdent(genPrefix & $temp.id), value.info)
 
@@ -154,7 +131,6 @@ proc rawAddField*(obj: PType; field: PSym) =
   assert field.kind == skField
   field.position = sonsLen(obj.n)
   addSon(obj.n, newSymNode(field))
-  propagateToOwner(obj, field.typ)
 
 proc rawIndirectAccess*(a: PNode; field: PSym; info: TLineInfo): PNode =
   # returns a[].field as a node
@@ -179,14 +155,14 @@ proc lookupInRecord(n: PNode, id: int): PSym =
   result = nil
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
+    for i in countup(0, sonsLen(n) - 1):
       result = lookupInRecord(n.sons[i], id)
       if result != nil: return
   of nkRecCase:
     if n.sons[0].kind != nkSym: return
     result = lookupInRecord(n.sons[0], id)
     if result != nil: return
-    for i in 1 ..< sonsLen(n):
+    for i in countup(1, sonsLen(n) - 1):
       case n.sons[i].kind
       of nkOfBranch, nkElse:
         result = lookupInRecord(lastSon(n.sons[i]), id)
@@ -204,8 +180,7 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache) =
   field.id = -s.id
   let t = skipIntLit(s.typ)
   field.typ = t
-  assert t.kind != tyTyped
-  propagateToOwner(obj, t)
+  assert t.kind != tyStmt
   field.position = sonsLen(obj.n)
   addSon(obj.n, newSymNode(field))
 
@@ -217,13 +192,12 @@ proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache): PSym {.discardable
     field.id = -s.id
     let t = skipIntLit(s.typ)
     field.typ = t
-    assert t.kind != tyTyped
-    propagateToOwner(obj, t)
+    assert t.kind != tyStmt
     field.position = sonsLen(obj.n)
     addSon(obj.n, newSymNode(field))
     result = field
 
-proc newDotExpr*(obj, b: PSym): PNode =
+proc newDotExpr(obj, b: PSym): PNode =
   result = newNodeI(nkDotExpr, obj.info)
   let field = lookupInRecord(obj.typ.n, b.id)
   assert field != nil, b.name.s
@@ -254,7 +228,7 @@ proc indirectAccess*(a: PNode, b: int, info: TLineInfo): PNode =
   addSon(result, newSymNode(field))
   result.typ = field.typ
 
-proc indirectAccess*(a: PNode, b: string, info: TLineInfo; cache: IdentCache): PNode =
+proc indirectAccess(a: PNode, b: string, info: TLineInfo; cache: IdentCache): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
   deref.typ = a.typ.skipTypes(abstractInst).sons[0]
@@ -302,21 +276,20 @@ proc genAddrOf*(n: PNode): PNode =
   result.typ = newType(tyPtr, n.typ.owner)
   result.typ.rawAddSon(n.typ)
 
-proc genDeref*(n: PNode; k = nkHiddenDeref): PNode =
-  result = newNodeIT(k, n.info,
+proc genDeref*(n: PNode): PNode =
+  result = newNodeIT(nkHiddenDeref, n.info,
                      n.typ.skipTypes(abstractInst).sons[0])
   result.add n
 
-proc callCodegenProc*(g: ModuleGraph; name: string;
-                      info: TLineInfo = unknownLineInfo();
-                      arg1, arg2, arg3, optionalArgs: PNode = nil): PNode =
-  result = newNodeI(nkCall, info)
+proc callCodegenProc*(g: ModuleGraph; name: string, arg1: PNode;
+                      arg2, arg3, optionalArgs: PNode = nil): PNode =
+  result = newNodeI(nkCall, arg1.info)
   let sym = magicsys.getCompilerProc(g, name)
   if sym == nil:
-    localError(g.config, info, "system module needs: " & name)
+    localError(g.config, arg1.info, "system module needs: " & name)
   else:
     result.add newSymNode(sym)
-    if arg1 != nil: result.add arg1
+    result.add arg1
     if arg2 != nil: result.add arg2
     if arg3 != nil: result.add arg3
     if optionalArgs != nil:
@@ -324,6 +297,231 @@ proc callCodegenProc*(g: ModuleGraph; name: string;
         result.add optionalArgs[i]
     result.typ = sym.typ.sons[0]
 
+proc callProc(a: PNode): PNode =
+  result = newNodeI(nkCall, a.info)
+  result.add a
+  result.typ = a.typ.sons[0]
+
+# we have 4 cases to consider:
+# - a void proc --> nothing to do
+# - a proc returning GC'ed memory --> requires a flowVar
+# - a proc returning non GC'ed memory --> pass as hidden 'var' parameter
+# - not in a parallel environment --> requires a flowVar for memory safety
+type
+  TSpawnResult* = enum
+    srVoid, srFlowVar, srByVar
+  TFlowVarKind = enum
+    fvInvalid # invalid type T for 'FlowVar[T]'
+    fvGC      # FlowVar of a GC'ed type
+    fvBlob    # FlowVar of a blob type
+
+proc spawnResult*(t: PType; inParallel: bool): TSpawnResult =
+  if t.isEmptyType: srVoid
+  elif inParallel and not containsGarbageCollectedRef(t): srByVar
+  else: srFlowVar
+
+proc flowVarKind(t: PType): TFlowVarKind =
+  if t.skipTypes(abstractInst).kind in {tyRef, tyString, tySequence}: fvGC
+  elif containsGarbageCollectedRef(t): fvInvalid
+  else: fvBlob
+
+proc typeNeedsNoDeepCopy(t: PType): bool =
+  var t = t.skipTypes(abstractInst)
+  # for the tconvexhull example (and others) we're a bit lax here and pretend
+  # seqs and strings are *by value* only and 'shallow' doesn't exist!
+  if t.kind == tyString: return true
+  # note that seq[T] is fine, but 'var seq[T]' is not, so we need to skip 'var'
+  # for the stricter check and likewise we can skip 'seq' for a less
+  # strict check:
+  if t.kind in {tyVar, tyLent, tySequence}: t = t.lastSon
+  result = not containsGarbageCollectedRef(t)
+
+proc hoistExpr*(varSection, expr: PNode, name: PIdent, owner: PSym): PSym =
+  result = newSym(skLet, name, owner, varSection.info, owner.options)
+  result.flags.incl sfHoisted
+  result.typ = expr.typ
+
+  var varDef = newNodeI(nkIdentDefs, varSection.info, 3)
+  varDef.sons[0] = newSymNode(result)
+  varDef.sons[1] = newNodeI(nkEmpty, varSection.info)
+  varDef.sons[2] = expr
+
+  varSection.add varDef
+
+proc addLocalVar(g: ModuleGraph; varSection, varInit: PNode; owner: PSym; typ: PType;
+                 v: PNode; useShallowCopy=false): PSym =
+  result = newSym(skTemp, getIdent(g.cache, genPrefix), owner, varSection.info,
+                  owner.options)
+  result.typ = typ
+  incl(result.flags, sfFromGeneric)
+
+  var vpart = newNodeI(nkIdentDefs, varSection.info, 3)
+  vpart.sons[0] = newSymNode(result)
+  vpart.sons[1] = newNodeI(nkEmpty, varSection.info)
+  vpart.sons[2] = if varInit.isNil: v else: vpart[1]
+  varSection.add vpart
+  if varInit != nil:
+    if useShallowCopy and typeNeedsNoDeepCopy(typ):
+      varInit.add newFastAsgnStmt(newSymNode(result), v)
+    else:
+      let deepCopyCall = newNodeI(nkCall, varInit.info, 3)
+      deepCopyCall.sons[0] = newSymNode(getSysMagic(g, varSection.info, "deepCopy", mDeepCopy))
+      deepCopyCall.sons[1] = newSymNode(result)
+      deepCopyCall.sons[2] = v
+      varInit.add deepCopyCall
+
+discard """
+We generate roughly this:
+
+proc f_wrapper(thread, args) =
+  barrierEnter(args.barrier)  # for parallel statement
+  var a = args.a # thread transfer; deepCopy or shallowCopy or no copy
+                 # depending on whether we're in a 'parallel' statement
+  var b = args.b
+  var fv = args.fv
+
+  fv.owner = thread # optional
+  nimArgsPassingDone() # signal parent that the work is done
+  #
+  args.fv.blob = f(a, b, ...)
+  nimFlowVarSignal(args.fv)
+
+  # - or -
+  f(a, b, ...)
+  barrierLeave(args.barrier)  # for parallel statement
+
+stmtList:
+  var scratchObj
+  scratchObj.a = a
+  scratchObj.b = b
+
+  nimSpawn(f_wrapper, addr scratchObj)
+  scratchObj.fv # optional
+
+"""
+
+proc createWrapperProc(g: ModuleGraph; f: PNode; threadParam, argsParam: PSym;
+                       varSection, varInit, call, barrier, fv: PNode;
+                       spawnKind: TSpawnResult): PSym =
+  var body = newNodeI(nkStmtList, f.info)
+  var threadLocalBarrier: PSym
+  if barrier != nil:
+    var varSection2 = newNodeI(nkVarSection, barrier.info)
+    threadLocalBarrier = addLocalVar(g, varSection2, nil, argsParam.owner,
+                                     barrier.typ, barrier)
+    body.add varSection2
+    body.add callCodegenProc(g, "barrierEnter", threadLocalBarrier.newSymNode)
+  var threadLocalProm: PSym
+  if spawnKind == srByVar:
+    threadLocalProm = addLocalVar(g, varSection, nil, argsParam.owner, fv.typ, fv)
+  elif fv != nil:
+    internalAssert g.config, fv.typ.kind == tyGenericInst
+    threadLocalProm = addLocalVar(g, varSection, nil, argsParam.owner, fv.typ, fv)
+  body.add varSection
+  body.add varInit
+  if fv != nil and spawnKind != srByVar:
+    # generate:
+    #   fv.owner = threadParam
+    body.add newAsgnStmt(indirectAccess(threadLocalProm.newSymNode,
+      "owner", fv.info, g.cache), threadParam.newSymNode)
+
+  body.add callCodegenProc(g, "nimArgsPassingDone", threadParam.newSymNode)
+  if spawnKind == srByVar:
+    body.add newAsgnStmt(genDeref(threadLocalProm.newSymNode), call)
+  elif fv != nil:
+    let fk = fv.typ.sons[1].flowVarKind
+    if fk == fvInvalid:
+      localError(g.config, f.info, "cannot create a flowVar of type: " &
+        typeToString(fv.typ.sons[1]))
+    body.add newAsgnStmt(indirectAccess(threadLocalProm.newSymNode,
+      if fk == fvGC: "data" else: "blob", fv.info, g.cache), call)
+    if fk == fvGC:
+      let incRefCall = newNodeI(nkCall, fv.info, 2)
+      incRefCall.sons[0] = newSymNode(getSysMagic(g, fv.info, "GCref", mGCref))
+      incRefCall.sons[1] = indirectAccess(threadLocalProm.newSymNode,
+                                          "data", fv.info, g.cache)
+      body.add incRefCall
+    if barrier == nil:
+      # by now 'fv' is shared and thus might have beeen overwritten! we need
+      # to use the thread-local view instead:
+      body.add callCodegenProc(g, "nimFlowVarSignal", threadLocalProm.newSymNode)
+  else:
+    body.add call
+  if barrier != nil:
+    body.add callCodegenProc(g, "barrierLeave", threadLocalBarrier.newSymNode)
+
+  var params = newNodeI(nkFormalParams, f.info)
+  params.add newNodeI(nkEmpty, f.info)
+  params.add threadParam.newSymNode
+  params.add argsParam.newSymNode
+
+  var t = newType(tyProc, threadParam.owner)
+  t.rawAddSon nil
+  t.rawAddSon threadParam.typ
+  t.rawAddSon argsParam.typ
+  t.n = newNodeI(nkFormalParams, f.info)
+  t.n.add newNodeI(nkEffectList, f.info)
+  t.n.add threadParam.newSymNode
+  t.n.add argsParam.newSymNode
+
+  let name = (if f.kind == nkSym: f.sym.name.s else: genPrefix) & "Wrapper"
+  result = newSym(skProc, getIdent(g.cache, name), argsParam.owner, f.info,
+                  argsParam.options)
+  let emptyNode = newNodeI(nkEmpty, f.info)
+  result.ast = newProcNode(nkProcDef, f.info, body = body,
+      params = params, name = newSymNode(result), pattern = emptyNode,
+      genericParams = emptyNode, pragmas = emptyNode,
+      exceptions = emptyNode)
+  result.typ = t
+
+proc createCastExpr(argsParam: PSym; objType: PType): PNode =
+  result = newNodeI(nkCast, argsParam.info)
+  result.add newNodeI(nkEmpty, argsParam.info)
+  result.add newSymNode(argsParam)
+  result.typ = newType(tyPtr, objType.owner)
+  result.typ.rawAddSon(objType)
+
+proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType; scratchObj: PSym,
+                             castExpr, call,
+                             varSection, varInit, result: PNode) =
+  let formals = n[0].typ.n
+  let tmpName = getIdent(g.cache, genPrefix)
+  for i in 1 ..< n.len:
+    # we pick n's type here, which hopefully is 'tyArray' and not
+    # 'tyOpenArray':
+    var argType = n[i].typ.skipTypes(abstractInst)
+    if i < formals.len and formals[i].typ.kind in {tyVar, tyLent}:
+      localError(g.config, n[i].info, "'spawn'ed function cannot have a 'var' parameter")
+    #elif containsTyRef(argType):
+    #  localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure")
+
+    let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
+    var field = newSym(skField, fieldname, objType.owner, n.info, g.config.options)
+    field.typ = argType
+    objType.addField(field, g.cache)
+    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[i])
+
+    let temp = addLocalVar(g, varSection, varInit, objType.owner, argType,
+                           indirectAccess(castExpr, field, n.info))
+    call.add(newSymNode(temp))
+
+proc getRoot*(n: PNode): PSym =
+  ## ``getRoot`` takes a *path* ``n``. A path is an lvalue expression
+  ## like ``obj.x[i].y``. The *root* of a path is the symbol that can be
+  ## determined as the owner; ``obj`` in the example.
+  case n.kind
+  of nkSym:
+    if n.sym.kind in {skVar, skResult, skTemp, skLet, skForVar}:
+      result = n.sym
+  of nkDotExpr, nkBracketExpr, nkHiddenDeref, nkDerefExpr,
+      nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
+    result = getRoot(n.sons[0])
+  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+    result = getRoot(n.sons[1])
+  of nkCallKinds:
+    if getMagic(n) == mSlice: result = getRoot(n.sons[1])
+  else: discard
+
 proc newIntLit*(g: ModuleGraph; info: TLineInfo; value: BiggestInt): PNode =
   result = nkIntLit.newIntNode(value)
   result.typ = getSysType(g, info, tyInt)
@@ -337,23 +535,195 @@ proc genHigh*(g: ModuleGraph; n: PNode): PNode =
     result.sons[0] = newSymNode(getSysMagic(g, n.info, "high", mHigh))
     result.sons[1] = n
 
-proc genLen*(g: ModuleGraph; n: PNode): PNode =
-  if skipTypes(n.typ, abstractVar).kind == tyArray:
-    result = newIntLit(g, n.info, lastOrd(g.config, skipTypes(n.typ, abstractVar)) + 1)
+proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; scratchObj: PSym;
+                             castExpr, call,
+                             varSection, varInit, result: PNode) =
+  let formals = n[0].typ.n
+  let tmpName = getIdent(g.cache, genPrefix)
+  # we need to copy the foreign scratch object fields into local variables
+  # for correctness: These are called 'threadLocal' here.
+  for i in 1 ..< n.len:
+    let n = n[i]
+    let argType = skipTypes(if i < formals.len: formals[i].typ else: n.typ,
+                            abstractInst)
+    #if containsTyRef(argType):
+    #  localError(n.info, "'spawn'ed function cannot refer to 'ref'/closure")
+
+    let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
+    var field = newSym(skField, fieldname, objType.owner, n.info, g.config.options)
+
+    if argType.kind in {tyVarargs, tyOpenArray}:
+      # important special case: we always create a zero-copy slice:
+      let slice = newNodeI(nkCall, n.info, 4)
+      slice.typ = n.typ
+      slice.sons[0] = newSymNode(createMagic(g, "slice", mSlice))
+      slice.sons[0].typ = getSysType(g, n.info, tyInt) # fake type
+      var fieldB = newSym(skField, tmpName, objType.owner, n.info, g.config.options)
+      fieldB.typ = getSysType(g, n.info, tyInt)
+      objType.addField(fieldB, g.cache)
+
+      if getMagic(n) == mSlice:
+        let a = genAddrOf(n[1])
+        field.typ = a.typ
+        objType.addField(field, g.cache)
+        result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
+
+        var fieldA = newSym(skField, tmpName, objType.owner, n.info, g.config.options)
+        fieldA.typ = getSysType(g, n.info, tyInt)
+        objType.addField(fieldA, g.cache)
+        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldA), n[2])
+        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), n[3])
+
+        let threadLocal = addLocalVar(g, varSection,nil, objType.owner, fieldA.typ,
+                                      indirectAccess(castExpr, fieldA, n.info),
+                                      useShallowCopy=true)
+        slice.sons[2] = threadLocal.newSymNode
+      else:
+        let a = genAddrOf(n)
+        field.typ = a.typ
+        objType.addField(field, g.cache)
+        result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
+        result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), genHigh(g, n))
+
+        slice.sons[2] = newIntLit(g, n.info, 0)
+      # the array itself does not need to go through a thread local variable:
+      slice.sons[1] = genDeref(indirectAccess(castExpr, field, n.info))
+
+      let threadLocal = addLocalVar(g, varSection,nil, objType.owner, fieldB.typ,
+                                    indirectAccess(castExpr, fieldB, n.info),
+                                    useShallowCopy=true)
+      slice.sons[3] = threadLocal.newSymNode
+      call.add slice
+    elif (let size = computeSize(g.config, argType); size < 0 or size > 16) and
+        n.getRoot != nil:
+      # it is more efficient to pass a pointer instead:
+      let a = genAddrOf(n)
+      field.typ = a.typ
+      objType.addField(field, g.cache)
+      result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
+      let threadLocal = addLocalVar(g, varSection,nil, objType.owner, field.typ,
+                                    indirectAccess(castExpr, field, n.info),
+                                    useShallowCopy=true)
+      call.add(genDeref(threadLocal.newSymNode))
+    else:
+      # boring case
+      field.typ = argType
+      objType.addField(field, g.cache)
+      result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n)
+      let threadLocal = addLocalVar(g, varSection, varInit,
+                                    objType.owner, field.typ,
+                                    indirectAccess(castExpr, field, n.info),
+                                    useShallowCopy=true)
+      call.add(threadLocal.newSymNode)
+
+proc wrapProcForSpawn*(g: ModuleGraph; owner: PSym; spawnExpr: PNode; retType: PType;
+                       barrier, dest: PNode = nil): PNode =
+  # if 'barrier' != nil, then it is in a 'parallel' section and we
+  # generate quite different code
+  let n = spawnExpr[^2]
+  let spawnKind = spawnResult(retType, barrier!=nil)
+  case spawnKind
+  of srVoid:
+    internalAssert g.config, dest == nil
+    result = newNodeI(nkStmtList, n.info)
+  of srFlowVar:
+    internalAssert g.config, dest == nil
+    result = newNodeIT(nkStmtListExpr, n.info, retType)
+  of srByVar:
+    if dest == nil: localError(g.config, n.info, "'spawn' must not be discarded")
+    result = newNodeI(nkStmtList, n.info)
+
+  if n.kind notin nkCallKinds:
+    localError(g.config, n.info, "'spawn' takes a call expression")
+    return
+  if optThreadAnalysis in g.config.globalOptions:
+    if {tfThread, tfNoSideEffect} * n[0].typ.flags == {}:
+      localError(g.config, n.info, "'spawn' takes a GC safe call expression")
+  var
+    threadParam = newSym(skParam, getIdent(g.cache, "thread"), owner, n.info, g.config.options)
+    argsParam = newSym(skParam, getIdent(g.cache, "args"), owner, n.info, g.config.options)
+  block:
+    let ptrType = getSysType(g, n.info, tyPointer)
+    threadParam.typ = ptrType
+    argsParam.typ = ptrType
+    argsParam.position = 1
+
+  var objType = createObj(g, owner, n.info)
+  incl(objType.flags, tfFinal)
+  let castExpr = createCastExpr(argsParam, objType)
+
+  var scratchObj = newSym(skVar, getIdent(g.cache, "scratch"), owner, n.info, g.config.options)
+  block:
+    scratchObj.typ = objType
+    incl(scratchObj.flags, sfFromGeneric)
+    var varSectionB = newNodeI(nkVarSection, n.info)
+    varSectionB.addVar(scratchObj.newSymNode)
+    result.add varSectionB
+
+  var call = newNodeIT(nkCall, n.info, n.typ)
+  var fn = n.sons[0]
+  # templates and macros are in fact valid here due to the nature of
+  # the transformation:
+  if fn.kind == nkClosure:
+    localError(g.config, n.info, "closure in spawn environment is not allowed")
+  if not (fn.kind == nkSym and fn.sym.kind in {skProc, skTemplate, skMacro,
+                                               skFunc, skMethod, skConverter}):
+    # for indirect calls we pass the function pointer in the scratchObj
+    var argType = n[0].typ.skipTypes(abstractInst)
+    var field = newSym(skField, getIdent(g.cache, "fn"), owner, n.info, g.config.options)
+    field.typ = argType
+    objType.addField(field, g.cache)
+    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0])
+    fn = indirectAccess(castExpr, field, n.info)
+  elif fn.kind == nkSym and fn.sym.kind == skIterator:
+    localError(g.config, n.info, "iterator in spawn environment is not allowed")
+  elif fn.typ.callConv == ccClosure:
+    localError(g.config, n.info, "closure in spawn environment is not allowed")
+
+  call.add(fn)
+  var varSection = newNodeI(nkVarSection, n.info)
+  var varInit = newNodeI(nkStmtList, n.info)
+  if barrier.isNil:
+    setupArgsForConcurrency(g, n, objType, scratchObj, castExpr, call,
+                            varSection, varInit, result)
   else:
-    result = newNodeI(nkCall, n.info, 2)
-    result.typ = getSysType(g, n.info, tyInt)
-    result.sons[0] = newSymNode(getSysMagic(g, n.info, "len", mLengthSeq))
-    result.sons[1] = n
+    setupArgsForParallelism(g, n, objType, scratchObj, castExpr, call,
+                            varSection, varInit, result)
 
-proc hoistExpr*(varSection, expr: PNode, name: PIdent, owner: PSym): PSym =
-  result = newSym(skLet, name, owner, varSection.info, owner.options)
-  result.flags.incl sfHoisted
-  result.typ = expr.typ
+  var barrierAsExpr: PNode = nil
+  if barrier != nil:
+    let typ = newType(tyPtr, owner)
+    typ.rawAddSon(magicsys.getCompilerProc(g, "Barrier").typ)
+    var field = newSym(skField, getIdent(g.cache, "barrier"), owner, n.info, g.config.options)
+    field.typ = typ
+    objType.addField(field, g.cache)
+    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), barrier)
+    barrierAsExpr = indirectAccess(castExpr, field, n.info)
 
-  var varDef = newNodeI(nkIdentDefs, varSection.info, 3)
-  varDef.sons[0] = newSymNode(result)
-  varDef.sons[1] = newNodeI(nkEmpty, varSection.info)
-  varDef.sons[2] = expr
+  var fvField, fvAsExpr: PNode = nil
+  if spawnKind == srFlowVar:
+    var field = newSym(skField, getIdent(g.cache, "fv"), owner, n.info, g.config.options)
+    field.typ = retType
+    objType.addField(field, g.cache)
+    fvField = newDotExpr(scratchObj, field)
+    fvAsExpr = indirectAccess(castExpr, field, n.info)
+    # create flowVar:
+    result.add newFastAsgnStmt(fvField, callProc(spawnExpr[^1]))
+    if barrier == nil:
+      result.add callCodegenProc(g, "nimFlowVarCreateSemaphore", fvField)
 
-  varSection.add varDef
+  elif spawnKind == srByVar:
+    var field = newSym(skField, getIdent(g.cache, "fv"), owner, n.info, g.config.options)
+    field.typ = newType(tyPtr, objType.owner)
+    field.typ.rawAddSon(retType)
+    objType.addField(field, g.cache)
+    fvAsExpr = indirectAccess(castExpr, field, n.info)
+    result.add newFastAsgnStmt(newDotExpr(scratchObj, field), genAddrOf(dest))
+
+  let wrapper = createWrapperProc(g, fn, threadParam, argsParam,
+                                  varSection, varInit, call,
+                                  barrierAsExpr, fvAsExpr, spawnKind)
+  result.add callCodegenProc(g, "nimSpawn" & $spawnExpr.len, wrapper.newSymNode,
+                             genAddrOf(scratchObj.newSymNode), nil, spawnExpr)
+
+  if spawnKind == srFlowVar: result.add fvField

compiler/macrocacheimpl.nim

@@ -9,7 +9,7 @@
 
 ## This module implements helpers for the macro cache.
 
-import lineinfos, ast, modulegraphs, vmdef
+import lineinfos, ast, modulegraphs, vmdef, magicsys
 
 proc recordInc*(c: PCtx; info: TLineInfo; key: string; by: BiggestInt) =
   var recorded = newNodeI(nkCommentStmt, info)

compiler/magicsys.nim

@@ -10,7 +10,7 @@
 # Built-in types and compilerprocs are registered here.
 
 import
-  ast, astalgo, msgs, platform, idents,
+  ast, astalgo, hashes, msgs, platform, nversion, times, idents,
   modulegraphs, lineinfos
 
 export createMagic

compiler/main.nim

@@ -13,24 +13,25 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  llstream, strutils, ast, lexer, syntaxes, options, msgs,
-  condsyms, times,
-  sem, idents, passes, extccomp,
-  cgen, json, nversion,
-  platform, nimconf, passaux, depends, vm, idgen,
-  parser, modules,
+  llstream, strutils, ast, astalgo, lexer, syntaxes, renderer, options, msgs,
+  os, condsyms, times,
+  wordrecg, sem, semdata, idents, passes, docgen, extccomp,
+  cgen, jsgen, json, nversion,
+  platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
+  docgen2, parser, modules, ccgutils, sigmatch, ropes,
   modulegraphs, tables, rod, lineinfos, pathutils
 
-when not defined(leanCompiler):
-  import jsgen, docgen, docgen2
+from magicsys import resetSysTypes
+
+proc codegenPass(g: ModuleGraph) =
+  registerPass g, cgenPass
 
 proc semanticPasses(g: ModuleGraph) =
   registerPass g, verbosePass
   registerPass g, semPass
 
-proc writeDepsFile(g: ModuleGraph) =
-  let fname = g.config.nimcacheDir / RelativeFile(g.config.projectName & ".deps")
-  let f = open(fname.string, fmWrite)
+proc writeDepsFile(g: ModuleGraph; project: AbsoluteFile) =
+  let f = open(changeFileExt(project, "deps").string, fmWrite)
   for m in g.modules:
     if m != nil:
       f.writeLine(toFullPath(g.config, m.position.FileIndex))
@@ -44,7 +45,7 @@ proc commandGenDepend(graph: ModuleGraph) =
   registerPass(graph, gendependPass)
   compileProject(graph)
   let project = graph.config.projectFull
-  writeDepsFile(graph)
+  writeDepsFile(graph, project)
   generateDot(graph, project)
   execExternalProgram(graph.config, "dot -Tpng -o" &
       changeFileExt(project, "png").string &
@@ -56,72 +57,46 @@ proc commandCheck(graph: ModuleGraph) =
   semanticPasses(graph)  # use an empty backend for semantic checking only
   compileProject(graph)
 
-when not defined(leanCompiler):
-  proc commandDoc2(graph: ModuleGraph; json: bool) =
-    handleDocOutputOptions graph.config
-    graph.config.errorMax = high(int)  # do not stop after first error
-    semanticPasses(graph)
-    if json: registerPass(graph, docgen2JsonPass)
-    else: registerPass(graph, docgen2Pass)
-    compileProject(graph)
-    finishDoc2Pass(graph.config.projectName)
+proc commandDoc2(graph: ModuleGraph; json: bool) =
+  graph.config.errorMax = high(int)  # do not stop after first error
+  semanticPasses(graph)
+  if json: registerPass(graph, docgen2JsonPass)
+  else: registerPass(graph, docgen2Pass)
+  compileProject(graph)
+  finishDoc2Pass(graph.config.projectName)
 
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
-
-  if conf.outDir.isEmpty:
-    conf.outDir = conf.projectPath
-  if conf.outFile.isEmpty:
-    let targetName = if optGenDynLib in conf.globalOptions:
-      platform.OS[conf.target.targetOS].dllFrmt % conf.projectName
-    else:
-      conf.projectName & platform.OS[conf.target.targetOS].exeExt
-    conf.outFile = RelativeFile targetName
-
   extccomp.initVars(conf)
   semanticPasses(graph)
   registerPass(graph, cgenPass)
 
   compileProject(graph)
-  if graph.config.errorCounter > 0:
-    return # issue #9933
   cgenWriteModules(graph.backend, conf)
   if conf.cmd != cmdRun:
-    extccomp.callCCompiler(conf)
-    # for now we do not support writing out a .json file with the build instructions when HCR is on
-    if not conf.hcrOn:
-      extccomp.writeJsonBuildInstructions(conf)
+    let proj = changeFileExt(conf.projectFull, "")
+    extccomp.callCCompiler(conf, proj)
+    extccomp.writeJsonBuildInstructions(conf, proj)
     if optGenScript in graph.config.globalOptions:
-      writeDepsFile(graph)
+      writeDepsFile(graph, toGeneratedFile(conf, proj, ""))
 
 proc commandJsonScript(graph: ModuleGraph) =
   let proj = changeFileExt(graph.config.projectFull, "")
   extccomp.runJsonBuildInstructions(graph.config, proj)
 
-when not defined(leanCompiler):
-  proc commandCompileToJS(graph: ModuleGraph) =
-    let conf = graph.config
-
-    if conf.outDir.isEmpty:
-      conf.outDir = conf.projectPath
-    if conf.outFile.isEmpty:
-      conf.outFile = RelativeFile(conf.projectName & ".js")
-
-    #incl(gGlobalOptions, optSafeCode)
-    setTarget(graph.config.target, osJS, cpuJS)
-    #initDefines()
-    defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility
-    defineSymbol(graph.config.symbols, "js")
-    semanticPasses(graph)
-    registerPass(graph, JSgenPass)
-    compileProject(graph)
-    if optGenScript in graph.config.globalOptions:
-      writeDepsFile(graph)
+proc commandCompileToJS(graph: ModuleGraph) =
+  #incl(gGlobalOptions, optSafeCode)
+  setTarget(graph.config.target, osJS, cpuJS)
+  #initDefines()
+  defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility
+  defineSymbol(graph.config.symbols, "js")
+  semanticPasses(graph)
+  registerPass(graph, JSgenPass)
+  compileProject(graph)
 
 proc interactivePasses(graph: ModuleGraph) =
   initDefines(graph.config.symbols)
   defineSymbol(graph.config.symbols, "nimscript")
-  # note: seems redundant with -d:nimHasLibFFI
   when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
   registerPass(graph, verbosePass)
   registerPass(graph, semPass)
@@ -185,7 +160,6 @@ proc mainCommand*(graph: ModuleGraph) =
   of "c", "cc", "compile", "compiletoc":
     # compile means compileToC currently
     conf.cmd = cmdCompileToC
-    defineSymbol(graph.config.symbols, "c")
     commandCompileToC(graph)
   of "cpp", "compiletocpp":
     conf.cmd = cmdCompileToCpp
@@ -203,82 +177,49 @@ proc mainCommand*(graph: ModuleGraph) =
     else:
       rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
   of "js", "compiletojs":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with JS code generator"
-    else:
-      conf.cmd = cmdCompileToJS
-      if conf.hcrOn:
-        # XXX: At the moment, system.nim cannot be compiled in JS mode
-        # with "-d:useNimRtl". The HCR option has been processed earlier
-        # and it has added this define implictly, so we must undo that here.
-        # A better solution might be to fix system.nim
-        undefSymbol(conf.symbols, "useNimRtl")
-      commandCompileToJS(graph)
+    conf.cmd = cmdCompileToJS
+    commandCompileToJS(graph)
   of "doc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandDoc(cache, conf)
+    wantMainModule(conf)
+    conf.cmd = cmdDoc
+    loadConfigs(DocConfig, cache, conf)
+    commandDoc(cache, conf)
   of "doc2", "doc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandDoc2(graph, false)
+    conf.cmd = cmdDoc
+    loadConfigs(DocConfig, cache, conf)
+    defineSymbol(conf.symbols, "nimdoc")
+    commandDoc2(graph, false)
   of "rst2html":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdRst2html
-      loadConfigs(DocConfig, cache, conf)
-      commandRst2Html(cache, conf)
+    conf.cmd = cmdRst2html
+    loadConfigs(DocConfig, cache, conf)
+    commandRst2Html(cache, conf)
   of "rst2tex":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdRst2tex
-      loadConfigs(DocTexConfig, cache, conf)
-      commandRst2TeX(cache, conf)
+    conf.cmd = cmdRst2tex
+    loadConfigs(DocTexConfig, cache, conf)
+    commandRst2TeX(cache, conf)
   of "jsondoc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandJson(cache, conf)
+    wantMainModule(conf)
+    conf.cmd = cmdDoc
+    loadConfigs(DocConfig, cache, conf)
+    wantMainModule(conf)
+    defineSymbol(conf.symbols, "nimdoc")
+    commandJson(cache, conf)
   of "jsondoc2", "jsondoc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandDoc2(graph, true)
+    conf.cmd = cmdDoc
+    loadConfigs(DocConfig, cache, conf)
+    wantMainModule(conf)
+    defineSymbol(conf.symbols, "nimdoc")
+    commandDoc2(graph, true)
   of "ctags":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandTags(cache, conf)
+    wantMainModule(conf)
+    conf.cmd = cmdDoc
+    loadConfigs(DocConfig, cache, conf)
+    defineSymbol(conf.symbols, "nimdoc")
+    commandTags(cache, conf)
   of "buildindex":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandBuildIndex(cache, conf)
+    conf.cmd = cmdDoc
+    loadConfigs(DocConfig, cache, conf)
+    commandBuildIndex(cache, conf)
   of "gendepend":
     conf.cmd = cmdGenDepend
     commandGenDepend(graph)
@@ -293,25 +234,11 @@ proc mainCommand*(graph: ModuleGraph) =
       var libpaths = newJArray()
       for dir in conf.searchPaths: libpaths.elems.add(%dir.string)
 
-      var hints = newJObject() # consider factoring with `listHints`
-      for a in hintMin..hintMax:
-        let key = lineinfos.HintsToStr[ord(a) - ord(hintMin)]
-        hints[key] = %(a in conf.notes)
-      var warnings = newJObject()
-      for a in warnMin..warnMax:
-        let key = lineinfos.WarningsToStr[ord(a) - ord(warnMin)]
-        warnings[key] = %(a in conf.notes)
-
-      var dumpdata = %[
+      var dumpdata = % [
         (key: "version", val: %VersionAsString),
         (key: "project_path", val: %conf.projectFull.string),
         (key: "defined_symbols", val: definedSymbols),
-        (key: "lib_paths", val: %libpaths),
-        (key: "outdir", val: %conf.outDir.string),
-        (key: "out", val: %conf.outFile.string),
-        (key: "nimcache", val: %getNimcacheDir(conf).string),
-        (key: "hints", val: hints),
-        (key: "warnings", val: warnings),
+        (key: "lib_paths", val: libpaths)
       ]
 
       msgWriteln(conf, $dumpdata, {msgStdout, msgSkipHook})
@@ -358,8 +285,7 @@ proc mainCommand*(graph: ModuleGraph) =
     rawMessage(conf, hintSuccessX, [$conf.linesCompiled,
                formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3),
                usedMem,
-               if isDefined(conf, "danger"): "Dangerous Release Build"
-               elif isDefined(conf, "release"): "Release Build"
+               if isDefined(conf, "release"): "Release Build"
                else: "Debug Build"])
 
   when PrintRopeCacheStats:

compiler/modulegraphs.nim

@@ -26,16 +26,13 @@
 ##
 
 import ast, intsets, tables, options, lineinfos, hashes, idents,
-  incremental, btrees, md5
+  incremental, btrees
 
 type
-  SigHash* = distinct MD5Digest
-
   ModuleGraph* = ref object
     modules*: seq[PSym]  ## indexed by int32 fileIdx
     packageSyms*: TStrTable
     deps*: IntSet # the dependency graph or potentially its transitive closure.
-    importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies
     suggestMode*: bool # whether we are in nimsuggest mode or not.
     invalidTransitiveClosure: bool
     inclToMod*: Table[FileIndex, FileIndex] # mapping of include file to the
@@ -50,7 +47,7 @@ type
     doStopCompile*: proc(): bool {.closure.}
     usageSym*: PSym # for nimsuggest
     owners*: seq[PSym]
-    methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization!
+    methods*: seq[tuple[methods: TSymSeq, dispatcher: PSym]] # needs serialization!
     systemModule*: PSym
     sysTypes*: array[TTypeKind, PType]
     compilerprocs*: TStrTable
@@ -59,8 +56,6 @@ type
     opContains*, opNot*: PSym
     emptyNode*: PNode
     incr*: IncrementalCtx
-    canonTypes*: Table[SigHash, PType]
-    symBodyHashes*: Table[int, SigHash] # symId to digest mapping
     importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.}
     includeFileCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PNode {.nimcall.}
     recordStmt*: proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.}
@@ -68,10 +63,6 @@ type
     cacheCounters*: Table[string, BiggestInt]
     cacheTables*: Table[string, BTree[string, PNode]]
     passes*: seq[TPass]
-    onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
-    onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
-    onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
-    globalDestructors*: seq[PNode]
 
   TPassContext* = object of RootObj # the pass's context
   PPassContext* = ref TPassContext
@@ -85,80 +76,8 @@ type
                  close: TPassClose,
                  isFrontend: bool]
 
-
-const
-  cb64 = [
-    "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N",
-    "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
-    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n",
-    "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
-    "0", "1", "2", "3", "4", "5", "6", "7", "8", "9a",
-    "9b", "9c"]
-
-proc toBase64a(s: cstring, len: int): string =
-  ## encodes `s` into base64 representation.
-  result = newStringOfCap(((len + 2) div 3) * 4)
-  result.add '_'
-  var i = 0
-  while i < len - 2:
-    let a = ord(s[i])
-    let b = ord(s[i+1])
-    let c = ord(s[i+2])
-    result.add cb64[a shr 2]
-    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
-    result.add cb64[((b and 0x0F) shl 2) or ((c and 0xC0) shr 6)]
-    result.add cb64[c and 0x3F]
-    inc(i, 3)
-  if i < len-1:
-    let a = ord(s[i])
-    let b = ord(s[i+1])
-    result.add cb64[a shr 2]
-    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
-    result.add cb64[((b and 0x0F) shl 2)]
-  elif i < len:
-    let a = ord(s[i])
-    result.add cb64[a shr 2]
-    result.add cb64[(a and 3) shl 4]
-
-proc `$`*(u: SigHash): string =
-  toBase64a(cast[cstring](unsafeAddr u), sizeof(u))
-
-proc `==`*(a, b: SigHash): bool =
-  result = equalMem(unsafeAddr a, unsafeAddr b, sizeof(a))
-
-proc hash*(u: SigHash): Hash =
-  result = 0
-  for x in 0..3:
-    result = (result shl 8) or u.MD5Digest[x].int
-
 proc hash*(x: FileIndex): Hash {.borrow.}
 
-when defined(nimfind):
-  template onUse*(info: TLineInfo; s: PSym) =
-    when compiles(c.c.graph):
-      if c.c.graph.onUsage != nil: c.c.graph.onUsage(c.c.graph, s, info)
-    else:
-      if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info)
-
-  template onDef*(info: TLineInfo; s: PSym) =
-    when compiles(c.c.graph):
-      if c.c.graph.onDefinition != nil: c.c.graph.onDefinition(c.c.graph, s, info)
-    else:
-      if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info)
-
-  template onDefResolveForward*(info: TLineInfo; s: PSym) =
-    when compiles(c.c.graph):
-      if c.c.graph.onDefinitionResolveForward != nil:
-        c.c.graph.onDefinitionResolveForward(c.c.graph, s, info)
-    else:
-      if c.graph.onDefinitionResolveForward != nil:
-        c.graph.onDefinitionResolveForward(c.graph, s, info)
-
-else:
-  template onUse*(info: TLineInfo; s: PSym) = discard
-  template onDef*(info: TLineInfo; s: PSym) = discard
-  template onDefResolveForward*(info: TLineInfo; s: PSym) = discard
-
 proc stopCompile*(g: ModuleGraph): bool {.inline.} =
   result = g.doStopCompile != nil and g.doStopCompile()
 
@@ -170,7 +89,6 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result = ModuleGraph()
   initStrTable(result.packageSyms)
   result.deps = initIntSet()
-  result.importDeps = initTable[FileIndex, seq[FileIndex]]()
   result.modules = @[]
   result.importStack = @[]
   result.inclToMod = initTable[FileIndex, FileIndex]()
@@ -189,8 +107,6 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.cacheSeqs = initTable[string, PNode]()
   result.cacheCounters = initTable[string, BiggestInt]()
   result.cacheTables = initTable[string, BTree[string, PNode]]()
-  result.canonTypes = initTable[SigHash, PType]()
-  result.symBodyHashes = initTable[int, SigHash]()
 
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.packageSyms)

compiler/modulepaths.nim

@@ -8,7 +8,7 @@
 #
 
 import ast, renderer, strutils, msgs, options, idents, os, lineinfos,
-  pathutils
+  pathutils, nimblecmd
 
 when false:
   const
@@ -114,6 +114,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
     try:
       result =
         pathSubs(conf, n.strVal, toFullPath(conf, n.info).splitFile().dir)
+          .replace(" ")
     except ValueError:
       localError(conf, n.info, "invalid path: " & n.strVal)
       result = n.strVal
@@ -146,7 +147,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
     # hacky way to implement 'x / y /../ z':
     result = renderTree(n, {renderNoComments}).replace(" ")
   of nkDotExpr:
-    localError(conf, n.info, warnDeprecated, "using '.' instead of '/' in import paths is deprecated")
+    localError(conf, n.info, warnDeprecated, "using '.' instead of '/' in import paths")
     result = renderTree(n, {renderNoComments}).replace(".", "/")
   of nkImportAs:
     result = getModuleName(conf, n.sons[0])
@@ -162,6 +163,6 @@ proc checkModuleName*(conf: ConfigRef; n: PNode; doLocalError=true): FileIndex =
     if doLocalError:
       let m = if modulename.len > 0: modulename else: $n
       localError(conf, n.info, "cannot open file: " & m)
-    result = InvalidFileIdx
+    result = InvalidFileIDX
   else:
     result = fileInfoIdx(conf, fullPath)

compiler/modules.nim

@@ -10,79 +10,63 @@
 ## Implements the module handling, including the caching of modules.
 
 import
-  ast, astalgo, magicsys, msgs, options,
-  idents, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod,
-  lineinfos, pathutils, tables
+  ast, astalgo, magicsys, std / sha1, msgs, cgendata, sigmatch, options,
+  idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod,
+  lineinfos, pathutils
 
 proc resetSystemArtifacts*(g: ModuleGraph) =
   magicsys.resetSysTypes(g)
 
-proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
-  let
-    pck = getPackageName(graph.config, filename.string)
-    pck2 = if pck.len > 0: pck else: "unknown"
-    pack = getIdent(graph.cache, pck2)
-  var packSym = graph.packageSyms.strTableGet(pack)
-  if packSym == nil:
-    packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, result.info)
-    initStrTable(packSym.tab)
-    graph.packageSyms.strTableAdd(packSym)
-  else:
-    let existing = strTableGet(packSym.tab, result.name)
-    if existing != nil and existing.info.fileIndex != result.info.fileIndex:
-      when false:
-        # we used to produce an error:
-        localError(graph.config, result.info,
-          "module names need to be unique per Nimble package; module clashes with " &
-            toFullPath(graph.config, existing.info.fileIndex))
-      else:
-        # but starting with version 0.20 we now produce a fake Nimble package instead
-        # to resolve the conflicts:
-        let pck3 = fakePackageName(graph.config, filename)
-        packSym = newSym(skPackage, getIdent(graph.cache, pck3), nil, result.info)
-        initStrTable(packSym.tab)
-        graph.packageSyms.strTableAdd(packSym)
-
-  result.owner = packSym
-  result.position = int fileIdx
-
-  if int(fileIdx) >= graph.modules.len:
-    setLen(graph.modules, int(fileIdx) + 1)
-  graph.modules[result.position] = result
-
-  incl(result.flags, sfUsed)
-  initStrTable(result.tab)
-  strTableAdd(result.tab, result) # a module knows itself
-  strTableAdd(packSym.tab, result)
-
 proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID.
   new(result)
   result.id = -1             # for better error checking
   result.kind = skModule
-  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+  let filename = toFullPath(graph.config, fileIdx)
   result.name = getIdent(graph.cache, splitFile(filename).name)
   if not isNimIdentifier(result.name.s):
     rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
+
   result.info = newLineInfo(fileIdx, 1, 1)
-  partialInitModule(result, graph, fileIdx, filename)
+  let
+    pck = getPackageName(graph.config, filename)
+    pck2 = if pck.len > 0: pck else: "unknown"
+    pack = getIdent(graph.cache, pck2)
+  var packSym = graph.packageSyms.strTableGet(pack)
+  if packSym == nil:
+    packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, result.info)
+    initStrTable(packSym.tab)
+    graph.packageSyms.strTableAdd(packSym)
+
+  result.owner = packSym
+  result.position = int fileIdx
+
+  if int(fileIdx) >= graph.modules.len:
+    setLen(graph.modules, int(fileIdx) + 1)
+  #growCache graph.modules, int fileIdx
+  graph.modules[result.position] = result
+
+  incl(result.flags, sfUsed)
+  initStrTable(result.tab)
+  strTableAdd(result.tab, result) # a module knows itself
+  let existing = strTableGet(packSym.tab, result.name)
+  if existing != nil and existing.info.fileIndex != result.info.fileIndex:
+    localError(graph.config, result.info,
+      "module names need to be unique per Nimble package; module clashes with " &
+        toFullPath(graph.config, existing.info.fileIndex))
+  # strTableIncl() for error corrections:
+  discard strTableIncl(packSym.tab, result)
 
 proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym =
   result = graph.getModule(fileIdx)
   if result == nil:
-    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-    let (r, id) = loadModuleSym(graph, fileIdx, filename)
-    result = r
-    if result == nil:
-      result = newModule(graph, fileIdx)
-      result.flags = result.flags + flags
-      result.id = id
-      registerModule(graph, result)
-    else:
-      partialInitModule(result, graph, fileIdx, filename)
-      result.id = id
-      assert result.id < 0
+    result = newModule(graph, fileIdx)
+    result.flags = result.flags + flags
+    if sfMainModule in result.flags:
+      graph.config.mainPackageId = result.owner.id
+
+    result.id = getModuleId(graph, fileIdx, AbsoluteFile toFullPath(graph.config, fileIdx))
     discard processModule(graph, result,
       if sfMainModule in flags and graph.config.projectIsStdin: stdin.llStreamOpen else: nil)
   elif graph.isDirty(result):
@@ -99,14 +83,11 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym {.proc
   assert graph.config != nil
   result = compileModule(graph, fileIdx, {})
   graph.addDep(s, fileIdx)
-  # keep track of import relationships
-  if graph.config.hcrOn:
-    graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx)
   #if sfSystemModule in result.flags:
   #  localError(result.info, errAttemptToRedefine, result.name.s)
   # restore the notes for outer module:
   graph.config.notes =
-    if s.owner.id == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
+    if s.owner.id == graph.config.mainPackageId: graph.config.mainPackageNotes
     else: graph.config.foreignPackageNotes
 
 proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode {.procvar.} =
@@ -131,12 +112,12 @@ proc wantMainModule*(conf: ConfigRef) =
         "command expects a filename")
   conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt))
 
-proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
+proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIDX) =
   connectCallbacks(graph)
   let conf = graph.config
   wantMainModule(conf)
   let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
-  let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
+  let projectFile = if projectFileIdx == InvalidFileIDX: conf.projectMainIdx else: projectFileIdx
   graph.importStack.add projectFile
   if projectFile == systemFileIdx:
     discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})
@@ -147,7 +128,6 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
 proc makeModule*(graph: ModuleGraph; filename: AbsoluteFile): PSym =
   result = graph.newModule(fileInfoIdx(graph.config, filename))
   result.id = getID()
-  registerModule(graph, result)
 
 proc makeModule*(graph: ModuleGraph; filename: string): PSym =
   result = makeModule(graph, AbsoluteFile filename)

compiler/msgs.nim

@@ -8,7 +8,7 @@
 #
 
 import
-  options, strutils, os, tables, ropes, terminal, macros,
+  options, strutils, os, tables, ropes, platform, terminal, macros,
   lineinfos, pathutils
 
 proc toCChar*(c: char; result: var string) =
@@ -27,7 +27,7 @@ proc makeCString*(s: string): Rope =
   result = nil
   var res = newStringOfCap(int(s.len.toFloat * 1.1) + 1)
   add(res, "\"")
-  for i in 0 ..< len(s):
+  for i in countup(0, len(s) - 1):
     if (i + 1) mod MaxLineLength == 0:
       add(res, "\"\L\"")
     toCChar(s[i], res)
@@ -40,8 +40,9 @@ proc newFileInfo(fullPath: AbsoluteFile, projPath: RelativeFile): TFileInfo =
   #shallow(result.fullPath)
   result.projPath = projPath
   #shallow(result.projPath)
-  result.shortName = fullPath.extractFilename
-  result.quotedName = result.shortName.makeCString
+  let fileName = fullPath.extractFilename
+  result.shortName = fileName.changeFileExt("")
+  result.quotedName = fileName.makeCString
   result.quotedFullName = fullPath.string.makeCString
   result.lines = @[]
   when defined(nimpretty):
@@ -95,20 +96,14 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile): FileIndex =
 
 proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo =
   result.fileIndex = fileInfoIdx
-  if line < int high(uint16):
-    result.line = uint16(line)
-  else:
-    result.line = high(uint16)
-  if col < int high(int16):
-    result.col = int16(col)
-  else:
-    result.col = -1
+  result.line = uint16(line)
+  result.col = int16(col)
 
 proc newLineInfo*(conf: ConfigRef; filename: AbsoluteFile, line, col: int): TLineInfo {.inline.} =
   result = newLineInfo(fileInfoIdx(conf, filename), line, col)
 
 
-proc concat(strings: openArray[string]): string =
+proc concat(strings: openarray[string]): string =
   var totalLen = 0
   for s in strings: totalLen += s.len
   result = newStringOfCap totalLen
@@ -140,16 +135,12 @@ const
   WarningColor = fgYellow
   HintTitle    = "Hint: "
   HintColor    = fgGreen
-  # NOTE: currently line info line numbers start with 1,
-  # but column numbers start with 0, however most editors expect
-  # first column to be 1, so we need to +1 here
-  ColOffset*   = 1
 
 proc getInfoContextLen*(conf: ConfigRef): int = return conf.m.msgContext.len
 proc setInfoContextLen*(conf: ConfigRef; L: int) = setLen(conf.m.msgContext, L)
 
-proc pushInfoContext*(conf: ConfigRef; info: TLineInfo; detail: string = "") =
-  conf.m.msgContext.add((info, detail))
+proc pushInfoContext*(conf: ConfigRef; info: TLineInfo) =
+  conf.m.msgContext.add(info)
 
 proc popInfoContext*(conf: ConfigRef) =
   setLen(conf.m.msgContext, len(conf.m.msgContext) - 1)
@@ -158,17 +149,13 @@ proc getInfoContext*(conf: ConfigRef; index: int): TLineInfo =
   let L = conf.m.msgContext.len
   let i = if index < 0: L + index else: index
   if i >=% L: result = unknownLineInfo()
-  else: result = conf.m.msgContext[i].info
+  else: result = conf.m.msgContext[i]
 
 template toFilename*(conf: ConfigRef; fileIdx: FileIndex): string =
   if fileIdx.int32 < 0 or conf == nil:
     "???"
   else:
-    conf.m.fileInfos[fileIdx.int32].shortName
-
-proc toProjPath*(conf: ConfigRef; fileIdx: FileIndex): string =
-  if fileIdx.int32 < 0 or conf == nil: "???"
-  else: conf.m.fileInfos[fileIdx.int32].projPath.string
+    conf.m.fileInfos[fileIdx.int32].projPath.string
 
 proc toFullPath*(conf: ConfigRef; fileIdx: FileIndex): string =
   if fileIdx.int32 < 0 or conf == nil: result = "???"
@@ -177,7 +164,6 @@ proc toFullPath*(conf: ConfigRef; fileIdx: FileIndex): string =
 proc setDirtyFile*(conf: ConfigRef; fileIdx: FileIndex; filename: AbsoluteFile) =
   assert fileIdx.int32 >= 0
   conf.m.fileInfos[fileIdx.int32].dirtyFile = filename
-  setLen conf.m.fileInfos[fileIdx.int32].lines, 0
 
 proc setHash*(conf: ConfigRef; fileIdx: FileIndex; hash: string) =
   assert fileIdx.int32 >= 0
@@ -198,28 +184,19 @@ proc toFullPathConsiderDirty*(conf: ConfigRef; fileIdx: FileIndex): AbsoluteFile
 template toFilename*(conf: ConfigRef; info: TLineInfo): string =
   toFilename(conf, info.fileIndex)
 
-template toProjPath*(conf: ConfigRef; info: TLineInfo): string =
-  toProjPath(conf, info.fileIndex)
-
 template toFullPath*(conf: ConfigRef; info: TLineInfo): string =
   toFullPath(conf, info.fileIndex)
 
-template toFullPathConsiderDirty*(conf: ConfigRef; info: TLineInfo): string =
-  string toFullPathConsiderDirty(conf, info.fileIndex)
-
-proc toMsgFilename*(conf: ConfigRef; info: FileIndex): string =
-  let
-    absPath = toFullPath(conf, info)
-    relPath = toProjPath(conf, info)
-  result = if (optListFullPaths in conf.globalOptions) or
-              (relPath.len > absPath.len) or
-              (relPath.count("..") > 2):
-             absPath
-           else:
-             relPath
-
-template toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
-  toMsgFilename(conf, info.fileIndex)
+proc toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
+  if info.fileIndex.int32 < 0:
+    result = "???"
+    return
+  let absPath = conf.m.fileInfos[info.fileIndex.int32].fullPath.string
+  if optListFullPaths in conf.globalOptions:
+    result = absPath
+  else:
+    let relPath = conf.m.fileInfos[info.fileIndex.int32].projPath.string
+    result = if relPath.count("..") > 2: absPath else: relPath
 
 proc toLinenumber*(info: TLineInfo): int {.inline.} =
   result = int info.line
@@ -227,10 +204,11 @@ proc toLinenumber*(info: TLineInfo): int {.inline.} =
 proc toColumn*(info: TLineInfo): int {.inline.} =
   result = info.col
 
+proc toFileLine*(conf: ConfigRef; info: TLineInfo): string {.inline.} =
+  result = toFilename(conf, info) & ":" & $info.line
+
 proc toFileLineCol*(conf: ConfigRef; info: TLineInfo): string {.inline.} =
-  # consider calling `helpers.lineInfoToString` instead
-  result = toMsgFilename(conf, info) & "(" & $info.line & ", " &
-    $(info.col + ColOffset) & ")"
+  result = toFilename(conf, info) & "(" & $info.line & ", " & $info.col & ")"
 
 proc `$`*(conf: ConfigRef; info: TLineInfo): string = toFileLineCol(conf, info)
 
@@ -333,15 +311,14 @@ proc log*(s: string) {.procvar.} =
     f.writeLine(s)
     close(f)
 
-proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} =
+proc quit(conf: ConfigRef; msg: TMsgKind) =
   if defined(debug) or msg == errInternal or hintStackTrace in conf.notes:
-    {.gcsafe.}:
-      if stackTraceAvailable() and isNil(conf.writelnHook):
-        writeStackTrace()
-      else:
-        styledMsgWriteln(fgRed, "No stack traceback available\n" &
-            "To create a stacktrace, rerun compilation with ./koch temp " &
-            conf.command & " <file>")
+    if stackTraceAvailable() and isNil(conf.writelnHook):
+      writeStackTrace()
+    else:
+      styledMsgWriteln(fgRed, "No stack traceback available\n" &
+          "To create a stacktrace, rerun compilation with ./koch temp " &
+          conf.command & " <file>")
   quit 1
 
 proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) =
@@ -366,26 +343,20 @@ proc exactEquals*(a, b: TLineInfo): bool =
 
 proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
   const instantiationFrom = "template/generic instantiation from here"
-  const instantiationOfFrom = "template/generic instantiation of `$1` from here"
   var info = lastinfo
   for i in 0 ..< len(conf.m.msgContext):
-    let context = conf.m.msgContext[i]
-    if context.info != lastinfo and context.info != info:
+    if conf.m.msgContext[i] != lastinfo and conf.m.msgContext[i] != info:
       if conf.structuredErrorHook != nil:
-        conf.structuredErrorHook(conf, context.info, instantiationFrom,
-                                 Severity.Error)
+        conf.structuredErrorHook(conf, conf.m.msgContext[i], instantiationFrom,
+                                  Severity.Error)
       else:
-        let message = if context.detail == "":
-          instantiationFrom
-        else:
-          instantiationOfFrom.format(context.detail)
         styledMsgWriteln(styleBright,
-                         PosFormat % [toMsgFilename(conf, context.info),
-                                      coordToStr(context.info.line.int),
-                                      coordToStr(context.info.col+ColOffset)],
+                         PosFormat % [toMsgFilename(conf, conf.m.msgContext[i]),
+                                      coordToStr(conf.m.msgContext[i].line.int),
+                                      coordToStr(conf.m.msgContext[i].col+1)],
                          resetStyle,
-                         message)
-    info = context.info
+                         instantiationFrom)
+    info = conf.m.msgContext[i]
 
 proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
   msg >= errGenerated and conf.cmd == cmdIdeTools and optIdeDebug notin conf.globalOptions
@@ -448,7 +419,7 @@ proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
 
   if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
     try:
-      for line in lines(toFullPathConsiderDirty(conf, i)):
+      for line in lines(toFullPath(conf, i)):
         addSourceLine conf, i.fileIndex, line.string
     except IOError:
       discard
@@ -461,8 +432,7 @@ proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
 proc writeSurroundingSrc(conf: ConfigRef; info: TLineInfo) =
   const indent = "  "
   msgWriteln(conf, indent & $sourceLine(conf, info))
-  if info.col >= 0:
-    msgWriteln(conf, indent & spaces(info.col) & '^')
+  msgWriteln(conf, indent & spaces(info.col) & '^')
 
 proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): string =
   let title = case msg
@@ -470,7 +440,7 @@ proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): s
               of hintMin..hintMax: HintTitle
               else: ErrorTitle
   result = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                        coordToStr(info.col+ColOffset)] &
+                        coordToStr(info.col+1)] &
            title &
            getMessageStr(msg, arg)
 
@@ -507,8 +477,11 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
     color = HintColor
     if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
     inc(conf.hintCounter)
+  # NOTE: currently line info line numbers start with 1,
+  # but column numbers start with 0, however most editors expect
+  # first column to be 1, so we need to +1 here
   let x = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                       coordToStr(info.col+ColOffset)]
+                       coordToStr(info.col+1)]
   let s = getMessageStr(msg, arg)
 
   if not ignoreMsg:
@@ -542,7 +515,7 @@ proc localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
 proc localError*(conf: ConfigRef; info: TLineInfo, arg: string) =
   liMessage(conf, info, errGenerated, arg, doNothing)
 
-proc localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openArray[string]) =
+proc localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openarray[string]) =
   localError(conf, info, format % params)
 
 proc message*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
@@ -566,9 +539,8 @@ template internalAssert*(conf: ConfigRef, e: bool) =
   if not e: internalError(conf, $instantiationInfo())
 
 proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
-  if i.fileIndex.int32 < 0:
-    result = makeCString "???"
-  elif optExcessiveStackTrace in conf.globalOptions:
+  assert i.fileIndex.int32 >= 0
+  if optExcessiveStackTrace in conf.globalOptions:
     result = conf.m.fileInfos[i.fileIndex.int32].quotedFullName
   else:
     result = conf.m.fileInfos[i.fileIndex.int32].quotedName
@@ -588,10 +560,3 @@ proc listHints*(conf: ConfigRef) =
       if hint in conf.notes: "x" else: " ",
       lineinfos.HintsToStr[ord(hint) - ord(hintMin)]
     ])
-
-proc lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string) =
-  let m = "'$2' should be: '$1'" % [beau, got]
-  if optStyleError in conf.globalOptions:
-    localError(conf, info, m)
-  else:
-    message(conf, info, hintName, m)

compiler/ndi.nim

@@ -20,9 +20,9 @@ type
 
 proc doWrite(f: var NdiFile; s: PSym; conf: ConfigRef) =
   f.buf.setLen 0
-  f.buf.addInt s.info.line.int
+  f.buf.add s.info.line.int
   f.buf.add "\t"
-  f.buf.addInt s.info.col.int
+  f.buf.add s.info.col.int
   f.f.write(s.name.s, "\t")
   f.f.writeRope(s.loc.r)
   f.f.writeLine("\t", toFullPath(conf, s.info), "\t", f.buf)

compiler/nim.cfg

@@ -12,15 +12,6 @@ define:nimcore
 @end
 
 define:useStdoutAsStdmsg
-define:nimOldCaseObjects
-
-@if nimHasStyleChecks:
-  styleCheck:error
-@end
 
 #define:useNodeIds
 #gc:markAndSweep
-
-@if nimHasWarningCaseTransition:
-warning[CaseTransition]:off
-@end

compiler/nim.nim

@@ -9,23 +9,21 @@
 
 when defined(gcc) and defined(windows):
   when defined(x86):
-    {.link: "../icons/nim.res".}
+    {.link: "icons/nim.res".}
   else:
-    {.link: "../icons/nim_icon.o".}
+    {.link: "icons/nim_icon.o".}
 
 when defined(amd64) and defined(windows) and defined(vcc):
-  {.link: "../icons/nim-amd64-windows-vcc.res".}
+  {.link: "icons/nim-amd64-windows-vcc.res".}
 when defined(i386) and defined(windows) and defined(vcc):
-  {.link: "../icons/nim-i386-windows-vcc.res".}
+  {.link: "icons/nim-i386-windows-vcc.res".}
 
 import
-  commands, options, msgs,
-  extccomp, strutils, os, main, parseopt,
-  idents, lineinfos, cmdlinehelper,
+  commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes,
+  extccomp, strutils, os, osproc, platform, main, parseopt,
+  nodejs, scriptconfig, idents, modulegraphs, lineinfos, cmdlinehelper,
   pathutils
 
-include nodejs
-
 when hasTinyCBackend:
   import tccgen
 
@@ -47,7 +45,7 @@ proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
     parseopt.next(p)
     case p.kind
     of cmdEnd: break
-    of cmdLongOption, cmdShortOption:
+    of cmdLongoption, cmdShortOption:
       if p.key == " ":
         p.key = "-"
         if processArgument(pass, p, argsCount, config): break
@@ -68,7 +66,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
   )
   self.initDefinesProg(conf, "nim_compiler")
   if paramCount() == 0:
-    writeCommandLineUsage(conf)
+    writeCommandLineUsage(conf, conf.helpWritten)
     return
 
   self.processCmdLineAndProjectPath(conf)
@@ -80,10 +78,23 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
     if conf.cmd == cmdRun:
       tccgen.run(conf.arguments)
   if optRun in conf.globalOptions:
-    var ex = quoteShell conf.absOutFile
     if conf.cmd == cmdCompileToJS:
+      var ex: string
+      if not conf.outFile.isEmpty:
+        ex = conf.outFile.prependCurDir.quoteShell
+      else:
+        ex = quoteShell(
+          completeCFilePath(conf, changeFileExt(conf.projectFull, "js").prependCurDir))
       execExternalProgram(conf, findNodeJs() & " " & ex & ' ' & conf.arguments)
     else:
+      var binPath: AbsoluteFile
+      if not conf.outFile.isEmpty:
+        # If the user specified an outFile path, use that directly.
+        binPath = conf.outFile.prependCurDir
+      else:
+        # Figure out ourselves a valid binary name.
+        binPath = changeFileExt(conf.projectFull, ExeExt).prependCurDir
+      var ex = quoteShell(binPath)
       execExternalProgram(conf, ex & ' ' & conf.arguments)
 
 when declared(GC_setMaxPause):

compiler/nimconf.nim

@@ -10,7 +10,7 @@
 # This module handles the reading of the config file.
 
 import
-  llstream, commands, os, strutils, msgs, lexer,
+  llstream, nversion, commands, os, strutils, msgs, platform, condsyms, lexer,
   options, idents, wordrecg, strtabs, lineinfos, pathutils
 
 # ---------------- configuration file parser -----------------------------
@@ -28,7 +28,7 @@ proc parseAtom(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
     result = parseExpr(L, tok, config)
     if tok.tokType == tkParRi: ppGetTok(L, tok)
     else: lexMessage(L, errGenerated, "expected closing ')'")
-  elif tok.tokType == tkNot:
+  elif tok.ident.id == ord(wNot):
     ppGetTok(L, tok)
     result = not parseAtom(L, tok, config)
   else:
@@ -37,14 +37,14 @@ proc parseAtom(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
 
 proc parseAndExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
   result = parseAtom(L, tok, config)
-  while tok.tokType == tkAnd:
+  while tok.ident.id == ord(wAnd):
     ppGetTok(L, tok)          # skip "and"
     var b = parseAtom(L, tok, config)
     result = result and b
 
 proc parseExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
   result = parseAndExpr(L, tok, config)
-  while tok.tokType == tkOr:
+  while tok.ident.id == ord(wOr):
     ppGetTok(L, tok)          # skip "or"
     var b = parseAndExpr(L, tok, config)
     result = result or b
@@ -123,31 +123,31 @@ proc parseDirective(L: var TLexer, tok: var TToken; config: ConfigRef; condStack
   of wEnd: doEnd(L, tok, condStack)
   of wWrite:
     ppGetTok(L, tok)
-    msgs.msgWriteln(config, strtabs.`%`($tok, config.configVars,
+    msgs.msgWriteln(config, strtabs.`%`(tokToStr(tok), config.configVars,
                                 {useEnvironment, useKey}))
     ppGetTok(L, tok)
   else:
     case tok.ident.s.normalize
     of "putenv":
       ppGetTok(L, tok)
-      var key = $tok
+      var key = tokToStr(tok)
       ppGetTok(L, tok)
-      os.putEnv(key, $tok)
+      os.putEnv(key, tokToStr(tok))
       ppGetTok(L, tok)
     of "prependenv":
       ppGetTok(L, tok)
-      var key = $tok
+      var key = tokToStr(tok)
       ppGetTok(L, tok)
-      os.putEnv(key, $tok & os.getEnv(key))
+      os.putEnv(key, tokToStr(tok) & os.getEnv(key))
       ppGetTok(L, tok)
     of "appendenv":
       ppGetTok(L, tok)
-      var key = $tok
+      var key = tokToStr(tok)
       ppGetTok(L, tok)
-      os.putEnv(key, os.getEnv(key) & $tok)
+      os.putEnv(key, os.getEnv(key) & tokToStr(tok))
       ppGetTok(L, tok)
     else:
-      lexMessage(L, errGenerated, "invalid directive: '$1'" % $tok)
+      lexMessage(L, errGenerated, "invalid directive: '$1'" % tokToStr(tok))
 
 proc confTok(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var seq[bool]) =
   ppGetTok(L, tok)
@@ -156,7 +156,7 @@ proc confTok(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var s
 
 proc checkSymbol(L: TLexer, tok: TToken) =
   if tok.tokType notin {tkSymbol..tkInt64Lit, tkStrLit..tkTripleStrLit}:
-    lexMessage(L, errGenerated, "expected identifier, but got: " & $tok)
+    lexMessage(L, errGenerated, "expected identifier, but got: " & tokToStr(tok))
 
 proc parseAssignment(L: var TLexer, tok: var TToken;
                      config: ConfigRef; condStack: var seq[bool]) =
@@ -164,21 +164,21 @@ proc parseAssignment(L: var TLexer, tok: var TToken;
     confTok(L, tok, config, condStack)           # skip unnecessary prefix
   var info = getLineInfo(L, tok) # save for later in case of an error
   checkSymbol(L, tok)
-  var s = $tok
+  var s = tokToStr(tok)
   confTok(L, tok, config, condStack)             # skip symbol
   var val = ""
   while tok.tokType == tkDot:
     add(s, '.')
     confTok(L, tok, config, condStack)
     checkSymbol(L, tok)
-    add(s, $tok)
+    add(s, tokToStr(tok))
     confTok(L, tok, config, condStack)
   if tok.tokType == tkBracketLe:
     # BUGFIX: val, not s!
     confTok(L, tok, config, condStack)
     checkSymbol(L, tok)
     add(val, '[')
-    add(val, $tok)
+    add(val, tokToStr(tok))
     confTok(L, tok, config, condStack)
     if tok.tokType == tkBracketRi: confTok(L, tok, config, condStack)
     else: lexMessage(L, errGenerated, "expected closing ']'")
@@ -188,12 +188,12 @@ proc parseAssignment(L: var TLexer, tok: var TToken;
     if len(val) > 0: add(val, ':')
     confTok(L, tok, config, condStack)           # skip ':' or '=' or '%'
     checkSymbol(L, tok)
-    add(val, $tok)
+    add(val, tokToStr(tok))
     confTok(L, tok, config, condStack)           # skip symbol
     while tok.ident != nil and tok.ident.s == "&":
       confTok(L, tok, config, condStack)
       checkSymbol(L, tok)
-      add(val, $tok)
+      add(val, tokToStr(tok))
       confTok(L, tok, config, condStack)
   if percent:
     processSwitch(s, strtabs.`%`(val, config.configVars,

compiler/nimeval.nim

@@ -11,8 +11,7 @@
 import
   ast, astalgo, modules, passes, condsyms,
   options, sem, semdata, llstream, vm, vmdef,
-  modulegraphs, idents, os, pathutils, passaux,
-  scriptconfig
+  modulegraphs, idents, os, pathutils
 
 type
   Interpreter* = ref object ## Use Nim as an interpreter with this object
@@ -90,13 +89,6 @@ proc findNimStdLib*(): string =
   except OSError, ValueError:
     return ""
 
-proc findNimStdLibCompileTime*(): string =
-  ## Same as ``findNimStdLib`` but uses source files used at compile time,
-  ## and asserts on error.
-  const sourcePath = currentSourcePath()
-  result = sourcePath.parentDir.parentDir / "lib"
-  doAssert fileExists(result / "system.nim"), "result:" & result
-
 proc createInterpreter*(scriptName: string;
                         searchPaths: openArray[string];
                         flags: TSandboxFlags = {}): Interpreter =
@@ -126,29 +118,3 @@ proc createInterpreter*(scriptName: string;
 proc destroyInterpreter*(i: Interpreter) =
   ## destructor.
   discard "currently nothing to do."
-
-proc runRepl*(r: TLLRepl;
-              searchPaths: openArray[string];
-              supportNimscript: bool) =
-  var conf = newConfigRef()
-  var cache = newIdentCache()
-  var graph = newModuleGraph(cache, conf)
-
-  for p in searchPaths:
-    conf.searchPaths.add(AbsoluteDir p)
-    if conf.libpath.isEmpty: conf.libpath = AbsoluteDir p
-
-  conf.cmd = cmdInteractive
-  conf.errorMax = high(int)
-  initDefines(conf.symbols)
-  defineSymbol(conf.symbols, "nimscript")
-  if supportNimscript: defineSymbol(conf.symbols, "nimconfig")
-  when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
-  registerPass(graph, verbosePass)
-  registerPass(graph, semPass)
-  registerPass(graph, evalPass)
-  var m = graph.makeStdinModule()
-  incl(m.flags, sfMainModule)
-  if supportNimscript: graph.vm = setupVM(m, cache, "stdin", graph)
-  graph.compileSystemModule()
-  processModule(graph, m, llStreamOpenStdIn(r))

compiler/nimfix/prettybase.nim

@@ -8,7 +8,9 @@
 #
 
 import strutils except Letters
+import lexbase, streams
 import ".." / [ast, msgs, lineinfos, idents, options, linter]
+from os import splitFile
 
 proc replaceDeprecated*(conf: ConfigRef; info: TLineInfo; oldSym, newSym: PIdent) =
   let line = sourceLine(conf, info)

compiler/nimlexbase.nim

@@ -39,7 +39,8 @@ const
 type
   TBaseLexer* = object of RootObj
     bufpos*: int
-    buf*: string
+    buf*: cstring
+    bufLen*: int              # length of buffer in characters
     stream*: PLLStream        # we read from this stream
     lineNumber*: int          # the current line number
                               # private data:
@@ -64,7 +65,11 @@ proc handleLF*(L: var TBaseLexer, pos: int): int
   # of the LF.
 # implementation
 
+const
+  chrSize = sizeof(char)
+
 proc closeBaseLexer(L: var TBaseLexer) =
+  dealloc(L.buf)
   llStreamClose(L.stream)
 
 proc fillBuffer(L: var TBaseLexer) =
@@ -75,13 +80,14 @@ proc fillBuffer(L: var TBaseLexer) =
     oldBufLen: int
   # we know here that pos == L.sentinel, but not if this proc
   # is called the first time by initBaseLexer()
-  assert(L.sentinel < L.buf.len)
-  toCopy = L.buf.len - L.sentinel - 1
+  assert(L.sentinel < L.bufLen)
+  toCopy = L.bufLen - L.sentinel - 1
   assert(toCopy >= 0)
   if toCopy > 0:
-    moveMem(addr L.buf[0], addr L.buf[L.sentinel + 1], toCopy)
+    moveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize)
     # "moveMem" handles overlapping regions
-  charsRead = llStreamRead(L.stream, addr L.buf[toCopy], L.sentinel + 1)
+  charsRead = llStreamRead(L.stream, addr(L.buf[toCopy]),
+                           (L.sentinel + 1) * chrSize) div chrSize
   s = toCopy + charsRead
   if charsRead < L.sentinel + 1:
     L.buf[s] = EndOfFile      # set end marker
@@ -90,7 +96,7 @@ proc fillBuffer(L: var TBaseLexer) =
     # compute sentinel:
     dec(s)                    # BUGFIX (valgrind)
     while true:
-      assert(s < L.buf.len)
+      assert(s < L.bufLen)
       while (s >= 0) and not (L.buf[s] in NewLines): dec(s)
       if s >= 0:
         # we found an appropriate character for a sentinel:
@@ -99,16 +105,17 @@ proc fillBuffer(L: var TBaseLexer) =
       else:
         # rather than to give up here because the line is too long,
         # double the buffer's size and try again:
-        oldBufLen = L.buf.len
-        L.buf.setLen(L.buf.len * 2)
-        assert(L.buf.len - oldBufLen == oldBufLen)
+        oldBufLen = L.bufLen
+        L.bufLen = L.bufLen * 2
+        L.buf = cast[cstring](realloc(L.buf, L.bufLen * chrSize))
+        assert(L.bufLen - oldBufLen == oldBufLen)
         charsRead = llStreamRead(L.stream, addr(L.buf[oldBufLen]),
-                                 oldBufLen)
+                                 oldBufLen * chrSize) div chrSize
         if charsRead < oldBufLen:
           L.buf[oldBufLen + charsRead] = EndOfFile
           L.sentinel = oldBufLen + charsRead
           break
-        s = L.buf.len - 1
+        s = L.bufLen - 1
 
 proc fillBaseLexer(L: var TBaseLexer, pos: int): int =
   assert(pos <= L.sentinel)
@@ -142,7 +149,8 @@ proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen = 8192) =
   assert(bufLen > 0)
   L.bufpos = 0
   L.offsetBase = 0
-  L.buf = newString(bufLen)
+  L.bufLen = bufLen
+  L.buf = cast[cstring](alloc(bufLen * chrSize))
   L.sentinel = bufLen - 1
   L.lineStart = 0
   L.lineNumber = 1            # lines start at 1

compiler/nimsets.nim

@@ -10,14 +10,15 @@
 # this unit handles Nim sets; it implements symbolic sets
 
 import
-  ast, astalgo, lineinfos, bitsets, types, options
+  ast, astalgo, trees, nversion, lineinfos, platform, bitsets, types, renderer,
+  options
 
 proc inSet*(s: PNode, elem: PNode): bool =
   assert s.kind == nkCurly
   if s.kind != nkCurly:
     #internalError(s.info, "inSet")
     return false
-  for i in 0 ..< sonsLen(s):
+  for i in countup(0, sonsLen(s) - 1):
     if s.sons[i].kind == nkRange:
       if leValue(s.sons[i].sons[0], elem) and
           leValue(elem, s.sons[i].sons[1]):
@@ -47,7 +48,7 @@ proc someInSet*(s: PNode, a, b: PNode): bool =
   if s.kind != nkCurly:
     #internalError(s.info, "SomeInSet")
     return false
-  for i in 0 ..< sonsLen(s):
+  for i in countup(0, sonsLen(s) - 1):
     if s.sons[i].kind == nkRange:
       if leValue(s.sons[i].sons[0], b) and leValue(b, s.sons[i].sons[1]) or
           leValue(s.sons[i].sons[0], a) and leValue(a, s.sons[i].sons[1]):
@@ -62,14 +63,14 @@ proc toBitSet*(conf: ConfigRef; s: PNode, b: var TBitSet) =
   var first, j: BiggestInt
   first = firstOrd(conf, s.typ.sons[0])
   bitSetInit(b, int(getSize(conf, s.typ)))
-  for i in 0 ..< sonsLen(s):
+  for i in countup(0, sonsLen(s) - 1):
     if s.sons[i].kind == nkRange:
-      j = getOrdValue(s.sons[i].sons[0], first)
-      while j <= getOrdValue(s.sons[i].sons[1], first):
+      j = getOrdValue(s.sons[i].sons[0])
+      while j <= getOrdValue(s.sons[i].sons[1]):
         bitSetIncl(b, j - first)
         inc(j)
     else:
-      bitSetIncl(b, getOrdValue(s.sons[i], first) - first)
+      bitSetIncl(b, getOrdValue(s.sons[i]) - first)
 
 proc toTreeSet*(conf: ConfigRef; s: TBitSet, settype: PType, info: TLineInfo): PNode =
   var
@@ -132,7 +133,7 @@ proc equalSets*(conf: ConfigRef; a, b: PNode): bool =
 proc complement*(conf: ConfigRef; a: PNode): PNode =
   var x: TBitSet
   toBitSet(conf, a, x)
-  for i in 0 .. high(x): x[i] = not x[i]
+  for i in countup(0, high(x)): x[i] = not x[i]
   result = toTreeSet(conf, x, a.typ, a.info)
 
 proc deduplicate*(conf: ConfigRef; a: PNode): PNode =
@@ -149,7 +150,7 @@ proc setHasRange*(s: PNode): bool =
   assert s.kind == nkCurly
   if s.kind != nkCurly:
     return false
-  for i in 0 ..< sonsLen(s):
+  for i in countup(0, sonsLen(s) - 1):
     if s.sons[i].kind == nkRange:
       return true
   result = false

compiler/nodejs.nim

@@ -1,7 +1,8 @@
 import os
 
-proc findNodeJs*(): string {.inline.} =
-  ## Find NodeJS executable and return it as a string.
+proc findNodeJs*(): string =
   result = findExe("nodejs")
   if result == "":
     result = findExe("node")
+  if result == "":
+    result = findExe("iojs")

compiler/options.nim

@@ -8,7 +8,7 @@
 #
 
 import
-  os, strutils, strtabs, sets, lineinfos, platform,
+  os, strutils, strtabs, osproc, sets, lineinfos, platform,
   prefixmatches, pathutils
 
 from terminal import isatty
@@ -18,15 +18,15 @@ const
   hasTinyCBackend* = defined(tinyc)
   useEffectSystem* = true
   useWriteTracking* = false
-  hasFFI* = defined(nimHasLibFFI)
-  copyrightYear* = "2019"
+  hasFFI* = defined(useFFI)
+  copyrightYear* = "2018"
 
 type                          # please make sure we have under 32 options
                               # (improves code efficiency a lot!)
   TOption* = enum             # **keep binary compatible**
     optNone, optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck,
-    optOverflowCheck, optNilCheck, optRefCheck,
-    optNaNCheck, optInfCheck, optStyleCheck,
+    optOverflowCheck, optNilCheck,
+    optNaNCheck, optInfCheck, optMoveCheck,
     optAssert, optLineDir, optWarns, optHints,
     optOptimizeSpeed, optOptimizeSize, optStackTrace, # stack tracing support
     optLineTrace,             # line tracing support (includes stack tracing)
@@ -36,11 +36,11 @@ type                          # please make sure we have under 32 options
     optProfiler,              # profiler turned on
     optImplicitStatic,        # optimization: implicit at compile time
                               # evaluation
-    optTrMacros,              # en/disable pattern matching
+    optPatterns,              # en/disable pattern matching
     optMemTracker,
+    optHotCodeReloading,
     optLaxStrings,
-    optNilSeqs,
-    optOldAst
+    optNilSeqs
 
   TOptions* = set[TOption]
   TGlobalOption* = enum       # **keep binary compatible**
@@ -54,8 +54,7 @@ type                          # please make sure we have under 32 options
     optGenScript,             # generate a script file to compile the *.c files
     optGenMapping,            # generate a mapping file
     optRun,                   # run the compiled project
-    optStyleHint,             # check that the names adhere to NEP-1
-    optStyleError,            # enforce that the names adhere to NEP-1
+    optCheckNep1,             # check that the names adhere to NEP-1
     optSkipSystemConfigFile,  # skip the system's cfg/nims config file
     optSkipProjConfigFile,    # skip the project's cfg/nims config file
     optSkipUserConfigFile,    # skip the users's cfg/nims config file
@@ -74,16 +73,11 @@ type                          # please make sure we have under 32 options
     optIdeTerse               # idetools: use terse descriptions
     optNoCppExceptions        # use C exception handling even with CPP
     optExcessiveStackTrace    # fully qualified module filenames
-    optShowAllMismatches      # show all overloading resolution candidates
     optWholeProject           # for 'doc2': output any dependency
-    optDocInternal            # generate documentation for non-exported symbols
     optMixedMode              # true if some module triggered C++ codegen
-    optListFullPaths          # use full paths in toMsgFilename
+    optListFullPaths
     optNoNimblePath
-    optHotCodeReloading
     optDynlibOverrideAll
-    optNimV2
-    optMultiMethods
 
   TGlobalOptions* = set[TGlobalOption]
 
@@ -110,10 +104,8 @@ type
     cmdJsonScript             # compile a .json build file
   TStringSeq* = seq[string]
   TGCMode* = enum             # the selected GC
-    gcNone, gcBoehm, gcRegions, gcMarkAndSweep, gcDestructors,
-    gcRefc, gcV2, gcGo
-    # gcRefc and the GCs that follow it use a write barrier,
-    # as far as usesWriteBarrier() is concerned
+    gcNone, gcBoehm, gcGo, gcRegions, gcMarkAndSweep, gcDestructors,
+    gcRefc, gcV2
 
   IdeCmd* = enum
     ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideMod,
@@ -130,32 +122,21 @@ type
     forLoopMacros,
     caseStmtMacros,
     codeReordering,
-    compiletimeFFI,
-      ## This requires building nim with `-d:nimHasLibFFI`
-      ## which itself requires `nimble install libffi`, see #10150
-      ## Note: this feature can't be localized with {.push.}
-
-  LegacyFeature* = enum
-    allowSemcheckedAstModification,
-      ## Allows to modify a NimNode where the type has already been
-      ## flaged with nfSem. If you actually do this, it will cause
-      ## bugs.
 
   SymbolFilesOption* = enum
     disabledSf, writeOnlySf, readOnlySf, v2Sf
 
   TSystemCC* = enum
     ccNone, ccGcc, ccNintendoSwitch, ccLLVM_Gcc, ccCLang, ccLcc, ccBcc, ccDmc, ccWcc, ccVcc,
-    ccTcc, ccPcc, ccUcc, ccIcl, ccIcc, ccClangCl
+    ccTcc, ccPcc, ccUcc, ccIcl, ccIcc
 
   CfileFlag* {.pure.} = enum
     Cached,    ## no need to recompile this time
     External   ## file was introduced via .compile pragma
 
   Cfile* = object
-    nimname*: string
     cname*, obj*: AbsoluteFile
-    flags*: set[CfileFlag]
+    flags*: set[CFileFlag]
   CfileList* = seq[Cfile]
 
   Suggest* = ref object
@@ -186,7 +167,6 @@ type
     linesCompiled*: int  # all lines that have been compiled
     options*: TOptions    # (+)
     globalOptions*: TGlobalOptions # (+)
-    macrosToExpand*: StringTableRef
     m*: MsgConfig
     evalTemplateCounter*: int
     evalMacroCounter*: int
@@ -202,9 +182,9 @@ type
     cppDefines*: HashSet[string] # (*)
     headerFile*: string
     features*: set[Feature]
-    legacyFeatures*: set[LegacyFeature]
     arguments*: string ## the arguments to be passed to the program that
                        ## should be run
+    helpWritten*: bool
     ideCmd*: IdeCmd
     oldNewlines*: bool
     cCompiler*: TSystemCC
@@ -225,8 +205,7 @@ type
     packageCache*: StringTableRef
     searchPaths*: seq[AbsoluteDir]
     lazyPaths*: seq[AbsoluteDir]
-    outFile*: RelativeFile
-    outDir*: AbsoluteDir
+    outFile*: AbsoluteFile
     prefixDir*, libpath*, nimcacheDir*: AbsoluteDir
     dllOverrides, moduleOverrides*: StringTableRef
     projectName*: string # holds a name like 'nim'
@@ -253,19 +232,17 @@ type
     compileOptionsCmd*: seq[string]
     linkOptions*: string          # (*)
     compileOptions*: string       # (*)
-    cCompilerPath*: string
+    ccompilerpath*: string
     toCompile*: CfileList         # (*)
     suggestionResultHook*: proc (result: Suggest) {.closure.}
     suggestVersion*: int
     suggestMaxResults*: int
     lastLineInfo*: TLineInfo
-    writelnHook*: proc (output: string) {.closure.} # cannot make this gcsafe yet because of Nimble
+    writelnHook*: proc (output: string) {.closure.}
     structuredErrorHook*: proc (config: ConfigRef; info: TLineInfo; msg: string;
-                                severity: Severity) {.closure, gcsafe.}
+                                severity: Severity) {.closure.}
     cppCustomNamespace*: string
 
-proc hcrOn*(conf: ConfigRef): bool = return optHotCodeReloading in conf.globalOptions
-
 template depConfigFields*(fn) {.dirty.} =
   fn(target)
   fn(options)
@@ -277,14 +254,13 @@ const oldExperimentalFeatures* = {implicitDeref, dotOperators, callOperator, par
 const
   ChecksOptions* = {optObjCheck, optFieldCheck, optRangeCheck, optNilCheck,
     optOverflowCheck, optBoundsCheck, optAssert, optNaNCheck, optInfCheck,
-    optStyleCheck, optRefCheck}
+    optMoveCheck}
 
   DefaultOptions* = {optObjCheck, optFieldCheck, optRangeCheck,
-    optBoundsCheck, optOverflowCheck, optAssert, optWarns, optRefCheck,
+    optBoundsCheck, optOverflowCheck, optAssert, optWarns,
     optHints, optStackTrace, optLineTrace,
-    optTrMacros, optNilCheck, optStyleCheck}
-  DefaultGlobalOptions* = {optThreadAnalysis,
-    optExcessiveStackTrace, optListFullPaths}
+    optPatterns, optNilCheck, optMoveCheck}
+  DefaultGlobalOptions* = {optThreadAnalysis}
 
 proc getSrcTimestamp(): DateTime =
   try:
@@ -317,11 +293,10 @@ proc newConfigRef*(): ConfigRef =
     verbosity: 1,
     options: DefaultOptions,
     globalOptions: DefaultGlobalOptions,
-    macrosToExpand: newStringTable(modeStyleInsensitive),
     m: initMsgConfig(),
     evalExpr: "",
-    cppDefines: initHashSet[string](),
-    headerFile: "", features: {}, legacyFeatures: {}, foreignPackageNotes: {hintProcessing, warnUnknownMagic,
+    cppDefines: initSet[string](),
+    headerFile: "", features: {}, foreignPackageNotes: {hintProcessing, warnUnknownMagic,
     hintQuitCalled, hintExecuting},
     notes: NotesVerbosity[1], mainPackageNotes: NotesVerbosity[1],
     configVars: newStringTable(modeStyleInsensitive),
@@ -329,9 +304,7 @@ proc newConfigRef*(): ConfigRef =
     packageCache: newPackageCache(),
     searchPaths: @[],
     lazyPaths: @[],
-    outFile: RelativeFile"",
-    outDir: AbsoluteDir"",
-    prefixDir: AbsoluteDir"",
+    outFile: AbsoluteFile"", prefixDir: AbsoluteDir"",
     libpath: AbsoluteDir"", nimcacheDir: AbsoluteDir"",
     dllOverrides: newStringTable(modeCaseInsensitive),
     moduleOverrides: newStringTable(modeStyleInsensitive),
@@ -381,7 +354,7 @@ proc cppDefine*(c: ConfigRef; define: string) =
 
 proc isDefined*(conf: ConfigRef; symbol: string): bool =
   if conf.symbols.hasKey(symbol):
-    result = true
+    result = conf.symbols[symbol] != "false"
   elif cmpIgnoreStyle(symbol, CPU[conf.target.targetCPU].name) == 0:
     result = true
   elif cmpIgnoreStyle(symbol, platform.OS[conf.target.targetOS].name) == 0:
@@ -425,8 +398,7 @@ proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.cmd in {cmdDoc,
 proc usesWriteBarrier*(conf: ConfigRef): bool {.inline.} = conf.selectedGC >= gcRefc
 
 template compilationCachePresent*(conf: ConfigRef): untyped =
-  false
-#  conf.symbolFiles in {v2Sf, writeOnlySf}
+  conf.symbolFiles in {v2Sf, writeOnlySf}
 
 template optPreserveOrigSource*(conf: ConfigRef): untyped =
   optEmbedOrigSrc in conf.globalOptions
@@ -447,6 +419,9 @@ const
 
 const oKeepVariableNames* = true
 
+template compilingLib*(conf: ConfigRef): bool =
+  gGlobalOptions * {optGenGuiApp, optGenDynLib} != {}
+
 proc mainCommandArg*(conf: ConfigRef): string =
   ## This is intended for commands like check or parse
   ## which will work on the main project file unless
@@ -466,15 +441,8 @@ proc setConfigVar*(conf: ConfigRef; key, val: string) =
   conf.configVars[key] = val
 
 proc getOutFile*(conf: ConfigRef; filename: RelativeFile, ext: string): AbsoluteFile =
-  conf.outDir / changeFileExt(filename, ext)
-
-proc absOutFile*(conf: ConfigRef): AbsoluteFile =
-  conf.outDir / conf.outFile
-
-proc prepareToWriteOutput*(conf: ConfigRef): AbsoluteFile =
-  ## Create the output directory and returns a full path to the output file
-  createDir conf.outDir
-  return conf.outDir / conf.outFile
+  if not conf.outFile.isEmpty: result = conf.outFile
+  else: result = conf.projectPath / changeFileExt(filename, ext)
 
 proc getPrefixDir*(conf: ConfigRef): AbsoluteDir =
   ## Gets the prefix dir, usually the parent directory where the binary resides.
@@ -505,11 +473,20 @@ proc setDefaultLibpath*(conf: ConfigRef) =
     # Find out if $nim/../../lib/system.nim exists.
     let parentNimLibPath = realNimPath.parentDir.parentDir / "lib"
     if not fileExists(conf.libpath.string / "system.nim") and
-        fileExists(parentNimLibPath / "system.nim"):
+        fileExists(parentNimlibPath / "system.nim"):
       conf.libpath = AbsoluteDir parentNimLibPath
 
 proc canonicalizePath*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile =
-  result = AbsoluteFile path.string.expandFilename
+  # on Windows, 'expandFilename' calls getFullPathName which doesn't do
+  # case corrections, so we have to use this convoluted way of retrieving
+  # the true filename (see tests/modules and Nimble uses 'import Uri' instead
+  # of 'import uri'):
+  when defined(windows):
+    result = AbsoluteFile path.string.expandFilename
+    for x in walkFiles(result.string):
+      return AbsoluteFile x
+  else:
+    result = AbsoluteFile path.string.expandFilename
 
 proc shortenDir*(conf: ConfigRef; dir: string): string {.
     deprecated: "use 'relativeTo' instead".} =
@@ -548,7 +525,7 @@ proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir =
              conf.projectPath / genSubDir
            else:
             AbsoluteDir(getOsCacheDir() / splitFile(conf.projectName).name &
-               (if isDefined(conf, "release") or isDefined(conf, "danger"): "_r" else: "_d"))
+               (if isDefined(conf, "release"): "_r" else: "_d"))
 
 proc pathSubs*(conf: ConfigRef; p, config: string): string =
   let home = removeTrailingDirSep(os.getHomeDir())
@@ -597,7 +574,7 @@ proc rawFindFile2(conf: ConfigRef; f: RelativeFile): AbsoluteFile =
     result = it / f
     if fileExists(result):
       # bring to front
-      for j in countdown(i, 1):
+      for j in countDown(i,1):
         swap(conf.lazyPaths[j], conf.lazyPaths[j-1])
 
       return canonicalizePath(conf, result)

compiler/packagehandling.nim

@@ -29,6 +29,9 @@ proc getPackageName*(conf: ConfigRef; path: string): string =
       for file in walkFiles(d / "*.nimble"):
         result = file.splitFile.name
         break packageSearch
+      for file in walkFiles(d / "*.babel"):
+        result = file.splitFile.name
+        break packageSearch
   # we also store if we didn't find anything:
   when not defined(nimNoNilSeqs):
     if result.isNil: result = ""
@@ -38,23 +41,10 @@ proc getPackageName*(conf: ConfigRef; path: string): string =
     dec parents
     if parents <= 0: break
 
-proc fakePackageName*(conf: ConfigRef; path: AbsoluteFile): string =
-  # foo/../bar becomes foo7_7bar
-  result = relativeTo(path, conf.projectPath, '/').string.multiReplace(
-    {"/": "7", "..": "_", "7": "77", "_": "__", ":": "8", "8": "88"})
-
-proc demanglePackageName*(path: string): string =
-  result = path.multiReplace(
-    {"88": "8", "8": ":", "77": "7", "__": "_", "_7": "../", "7": "/"})
-
 proc withPackageName*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile =
   let x = getPackageName(conf, path.string)
   if x.len == 0:
     result = path
   else:
     let (p, file, ext) = path.splitFile
-    if x == "stdlib":
-      # Hot code reloading now relies on 'stdlib_system' names etc.
-      result = p / RelativeFile((x & '_' & file) & ext)
-    else:
-      result = p / RelativeFile(fakePackageName(conf, path))
+    result = p / RelativeFile((x & '_' & file) & ext)

compiler/parampatterns.nim

@@ -10,7 +10,7 @@
 ## This module implements the pattern matching features for term rewriting
 ## macro support.
 
-import strutils, ast, types, msgs, idents, renderer, wordrecg, trees,
+import strutils, ast, astalgo, types, msgs, idents, renderer, wordrecg, trees,
   options
 
 # we precompile the pattern here for efficiency into some internal
@@ -115,19 +115,20 @@ proc compileConstraints(p: PNode, result: var TPatternCode; conf: ConfigRef) =
   else:
     patternError(p, conf)
 
-proc semNodeKindConstraints*(n: PNode; conf: ConfigRef; start: Natural): PNode =
+proc semNodeKindConstraints*(p: PNode; conf: ConfigRef): PNode =
   ## does semantic checking for a node kind pattern and compiles it into an
   ## efficient internal format.
-  result = newNodeI(nkStrLit, n.info)
+  assert p.kind == nkCurlyExpr
+  result = newNodeI(nkStrLit, p.info)
   result.strVal = newStringOfCap(10)
   result.strVal.add(chr(aqNone.ord))
-  if n.len >= 2:
-    for i in start..<n.len:
-      compileConstraints(n[i], result.strVal, conf)
+  if p.len >= 2:
+    for i in 1..<p.len:
+      compileConstraints(p.sons[i], result.strVal, conf)
     if result.strVal.len > MaxStackSize-1:
-      internalError(conf, n.info, "parameter pattern too complex")
+      internalError(conf, p.info, "parameter pattern too complex")
   else:
-    patternError(n, conf)
+    patternError(p, conf)
   result.strVal.add(ppEof)
 
 type
@@ -230,10 +231,8 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
       let t = n.sym.typ.skipTypes({tyTypeDesc})
       if t.kind == tyVar: result = arStrange
   of nkDotExpr:
-    let t = skipTypes(n.sons[0].typ, abstractInst-{tyTypeDesc})
-    if t.kind in {tyVar, tyPtr, tyRef}:
-      result = arLValue
-    elif isUnsafeAddr and t.kind == tyLent:
+    if skipTypes(n.sons[0].typ, abstractInst-{tyTypeDesc}).kind in
+        {tyVar, tyPtr, tyRef}:
       result = arLValue
     else:
       result = isAssignable(owner, n.sons[0], isUnsafeAddr)
@@ -241,10 +240,8 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
         sfDiscriminant in n[1].sym.flags:
       result = arDiscriminant
   of nkBracketExpr:
-    let t = skipTypes(n.sons[0].typ, abstractInst-{tyTypeDesc})
-    if t.kind in {tyVar, tyPtr, tyRef}:
-      result = arLValue
-    elif isUnsafeAddr and t.kind == tyLent:
+    if skipTypes(n.sons[0].typ, abstractInst-{tyTypeDesc}).kind in
+        {tyVar, tyPtr, tyRef}:
       result = arLValue
     else:
       result = isAssignable(owner, n.sons[0], isUnsafeAddr)
@@ -258,8 +255,7 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
       # types that are equal modulo distinction preserve l-value:
       result = isAssignable(owner, n.sons[1], isUnsafeAddr)
   of nkHiddenDeref:
-    if isUnsafeAddr and n[0].typ.kind == tyLent: result = arLValue
-    elif n[0].typ.kind == tyLent: result = arDiscriminant
+    if n[0].typ.kind == tyLent: result = arDiscriminant
     else: result = arLValue
   of nkDerefExpr, nkHiddenAddr:
     result = arLValue
@@ -271,8 +267,6 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
       result = isAssignable(owner, n.sons[1], isUnsafeAddr)
     elif n.typ != nil and n.typ.kind == tyVar:
       result = arLValue
-    elif isUnsafeAddr and n.typ != nil and n.typ.kind == tyLent:
-      result = arLValue
   of nkStmtList, nkStmtListExpr:
     if n.typ != nil:
       result = isAssignable(owner, n.lastSon, isUnsafeAddr)

compiler/parser.nim

@@ -27,24 +27,25 @@ when isMainModule:
   outp.close
 
 import
-  llstream, lexer, idents, strutils, ast, msgs, options, lineinfos,
+  llstream, lexer, idents, strutils, ast, astalgo, msgs, options, lineinfos,
   pathutils
 
-when defined(nimpretty):
+when defined(nimpretty2):
   import layouter
 
 type
   TParser* = object            # A TParser object represents a file that
                                # is being parsed
     currInd: int               # current indentation level
-    firstTok: bool             # Has the first token been read?
+    firstTok, strongSpaces: bool # Has the first token been read?
+                                 # Is strongSpaces on?
     hasProgress: bool          # some while loop requires progress ensurance
     lex*: TLexer               # The lexer that is used for parsing
     tok*: TToken               # The current token
     inPragma*: int             # Pragma level
     inSemiStmtList*: int
     emptyNode: PNode
-    when defined(nimpretty):
+    when defined(nimpretty2):
       em*: Emitter
 
   SymbolMode = enum
@@ -89,44 +90,40 @@ proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode
 
 # implementation
 
-template prettySection(body) =
-  when defined(nimpretty): beginSection(p.em)
-  body
-  when defined(nimpretty): endSection(p.em)
-
 proc getTok(p: var TParser) =
   ## Get the next token from the parser's lexer, and store it in the parser's
   ## `tok` member.
   rawGetTok(p.lex, p.tok)
   p.hasProgress = true
-  when defined(nimpretty):
+  when defined(nimpretty2):
     emitTok(p.em, p.lex, p.tok)
-    # skip the additional tokens that nimpretty needs but the parser has no
-    # interest in:
     while p.tok.tokType == tkComment:
       rawGetTok(p.lex, p.tok)
       emitTok(p.em, p.lex, p.tok)
 
 proc openParser*(p: var TParser, fileIdx: FileIndex, inputStream: PLLStream,
-                 cache: IdentCache; config: ConfigRef) =
+                 cache: IdentCache; config: ConfigRef;
+                 strongSpaces=false) =
   ## Open a parser, using the given arguments to set up its internal state.
   ##
   initToken(p.tok)
   openLexer(p.lex, fileIdx, inputStream, cache, config)
-  when defined(nimpretty):
+  when defined(nimpretty2):
     openEmitter(p.em, cache, config, fileIdx)
   getTok(p)                   # read the first token
   p.firstTok = true
+  p.strongSpaces = strongSpaces
   p.emptyNode = newNode(nkEmpty)
 
 proc openParser*(p: var TParser, filename: AbsoluteFile, inputStream: PLLStream,
-                 cache: IdentCache; config: ConfigRef) =
-  openParser(p, fileInfoIdx(config, filename), inputStream, cache, config)
+                 cache: IdentCache; config: ConfigRef;
+                 strongSpaces=false) =
+  openParser(p, fileInfoIdx(config, filename), inputStream, cache, config, strongSpaces)
 
 proc closeParser(p: var TParser) =
   ## Close a parser, freeing up its resources.
   closeLexer(p.lex)
-  when defined(nimpretty):
+  when defined(nimpretty2):
     closeEmitter(p.em)
 
 proc parMessage(p: TParser, msg: TMsgKind, arg = "") =
@@ -147,17 +144,10 @@ template withInd(p, body: untyped) =
   body
   p.currInd = oldInd
 
-template newlineWasSplitting(p: var TParser) =
-  when defined(nimpretty):
-    layouter.newlineWasSplitting(p.em)
-
 template realInd(p): bool = p.tok.indent > p.currInd
 template sameInd(p): bool = p.tok.indent == p.currInd
 template sameOrNoInd(p): bool = p.tok.indent == p.currInd or p.tok.indent < 0
 
-proc validInd(p: var TParser): bool {.inline.} =
-  result = p.tok.indent < 0 or p.tok.indent > p.currInd
-
 proc rawSkipComment(p: var TParser, node: PNode) =
   if p.tok.tokType == tkComment:
     if node != nil:
@@ -279,7 +269,7 @@ proc checkBinary(p: TParser) {.inline.} =
   ## Check if the current parser token is a binary operator.
   # we don't check '..' here as that's too annoying
   if p.tok.tokType == tkOpr:
-    if p.tok.strongSpaceB > 0 and p.tok.strongSpaceA == 0:
+    if p.tok.strongSpaceB > 0 and p.tok.strongSpaceA != p.tok.strongSpaceB:
       parMessage(p, warnInconsistentSpacing, prettyTok(p.tok))
 
 #| module = stmt ^* (';' / IND{=})
@@ -346,17 +336,14 @@ proc parseSymbol(p: var TParser, mode = smNormal): PNode =
           parMessage(p, errIdentifierExpected, p.tok)
         break
       of tkOpr, tkDot, tkDotDot, tkEquals, tkParLe..tkParDotRi:
-        let lineinfo = parLineInfo(p)
         var accm = ""
         while p.tok.tokType in {tkOpr, tkDot, tkDotDot, tkEquals,
                                 tkParLe..tkParDotRi}:
-          accm.add($p.tok)
+          accm.add(tokToStr(p.tok))
           getTok(p)
-        let node = newNodeI(nkIdent, lineinfo)
-        node.ident = p.lex.cache.getIdent(accm)
-        result.add(node)
+        result.add(newIdentNodeP(p.lex.cache.getIdent(accm), p))
       of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCharLit:
-        result.add(newIdentNodeP(p.lex.cache.getIdent($p.tok), p))
+        result.add(newIdentNodeP(p.lex.cache.getIdent(tokToStr(p.tok)), p))
         getTok(p)
       else:
         parMessage(p, errIdentifierExpected, p.tok)
@@ -374,7 +361,6 @@ proc colonOrEquals(p: var TParser, a: PNode): PNode =
   if p.tok.tokType == tkColon:
     result = newNodeP(nkExprColonExpr, p)
     getTok(p)
-    newlineWasSplitting(p)
     #optInd(p, result)
     addSon(result, a)
     addSon(result, parseExpr(p))
@@ -397,7 +383,7 @@ proc exprColonEqExpr(p: var TParser): PNode =
 
 proc exprList(p: var TParser, endTok: TTokType, result: PNode) =
   #| exprList = expr ^+ comma
-  when defined(nimpretty):
+  when defined(nimpretty2):
     inc p.em.doIndentMore
   getTok(p)
   optInd(p, result)
@@ -408,14 +394,13 @@ proc exprList(p: var TParser, endTok: TTokType, result: PNode) =
     if p.tok.tokType != tkComma: break
     getTok(p)
     optInd(p, a)
-  when defined(nimpretty):
+  when defined(nimpretty2):
     dec p.em.doIndentMore
 
 proc exprColonEqExprListAux(p: var TParser, endTok: TTokType, result: PNode) =
   assert(endTok in {tkCurlyRi, tkCurlyDotRi, tkBracketRi, tkParRi})
   getTok(p)
-  flexComment(p, result)
-  optPar(p)
+  optInd(p, result)
   # progress guaranteed
   while p.tok.tokType != endTok and p.tok.tokType != tkEof:
     var a = exprColonEqExpr(p)
@@ -617,7 +602,7 @@ proc identOrLiteral(p: var TParser, mode: TPrimaryMode): PNode =
   #| tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')'
   #| arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']'
   case p.tok.tokType
-  of tkSymbol, tkBuiltInMagics, tkOut:
+  of tkSymbol, tkBuiltInMagics:
     result = newIdentNodeP(p.tok.ident, p)
     getTok(p)
     result = parseGStrLit(p, result)
@@ -721,11 +706,8 @@ proc namedParams(p: var TParser, callee: PNode,
   # progress guaranteed
   exprColonEqExprListAux(p, endTok, result)
 
-proc commandParam(p: var TParser, isFirstParam: var bool; mode: TPrimaryMode): PNode =
-  if mode == pmTypeDesc:
-    result = simpleExpr(p, mode)
-  else:
-    result = parseExpr(p)
+proc commandParam(p: var TParser, isFirstParam: var bool): PNode =
+  result = parseExpr(p)
   if p.tok.tokType == tkDo:
     result = postExprBlocks(p, result)
   elif p.tok.tokType == tkEquals and not isFirstParam:
@@ -740,14 +722,6 @@ const
   tkTypeClasses = {tkRef, tkPtr, tkVar, tkStatic, tkType,
                    tkEnum, tkTuple, tkObject, tkProc}
 
-proc commandExpr(p: var TParser; r: PNode; mode: TPrimaryMode): PNode =
-  result = newNodeP(nkCommand, p)
-  addSon(result, r)
-  var isFirstParam = true
-  # progress NOT guaranteed
-  p.hasProgress = false
-  addSon result, commandParam(p, isFirstParam, mode)
-
 proc primarySuffix(p: var TParser, r: PNode,
                    baseIndent: int, mode: TPrimaryMode): PNode =
   #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks?
@@ -758,6 +732,8 @@ proc primarySuffix(p: var TParser, r: PNode,
   #|       | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax
   result = r
 
+  template somePar() =
+    if p.tok.strongSpaceA > 0: break
   # progress guaranteed
   while p.tok.indent < 0 or
        (p.tok.tokType == tkDot and p.tok.indent >= baseIndent):
@@ -771,8 +747,6 @@ proc primarySuffix(p: var TParser, r: PNode,
           result = newNodeP(nkCommand, p)
           result.addSon r
           result.addSon primary(p, pmNormal)
-        else:
-          result = commandExpr(p, result, mode)
         break
       result = namedParams(p, result, nkCall, tkParRi)
       if result.len > 1 and result.sons[1].kind == nkExprColonExpr:
@@ -783,27 +757,39 @@ proc primarySuffix(p: var TParser, r: PNode,
       result = parseGStrLit(p, result)
     of tkBracketLe:
       # progress guaranteed
-      if p.tok.strongSpaceA > 0:
-        result = commandExpr(p, result, mode)
-        break
+      somePar()
       result = namedParams(p, result, nkBracketExpr, tkBracketRi)
     of tkCurlyLe:
       # progress guaranteed
-      if p.tok.strongSpaceA > 0:
-        result = commandExpr(p, result, mode)
-        break
+      somePar()
       result = namedParams(p, result, nkCurlyExpr, tkCurlyRi)
     of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast,
        tkOpr, tkDotDot, tkTypeClasses - {tkRef, tkPtr}:
-      # XXX: In type sections we allow the free application of the
-      # command syntax, with the exception of expressions such as
-      # `foo ref` or `foo ptr`. Unfortunately, these two are also
-      # used as infix operators for the memory regions feature and
-      # the current parsing rules don't play well here.
+        # XXX: In type sections we allow the free application of the
+        # command syntax, with the exception of expressions such as
+        # `foo ref` or `foo ptr`. Unfortunately, these two are also
+        # used as infix operators for the memory regions feature and
+        # the current parsing rules don't play well here.
       if p.inPragma == 0 and (isUnary(p) or p.tok.tokType notin {tkOpr, tkDotDot}):
         # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet
         # solution, but pragmas.nim can't handle that
-        result = commandExpr(p, result, mode)
+        let a = result
+        result = newNodeP(nkCommand, p)
+        addSon(result, a)
+        var isFirstParam = true
+        when true:
+          # progress NOT guaranteed
+          p.hasProgress = false
+          addSon result, commandParam(p, isFirstParam)
+          if not p.hasProgress: break
+        else:
+          while p.tok.tokType != tkEof:
+            let x = parseExpr(p)
+            addSon(result, x)
+            if p.tok.tokType != tkComma: break
+            getTok(p)
+            optInd(p, x)
+          result = postExprBlocks(p, result)
       break
     else:
       break
@@ -812,7 +798,7 @@ proc parseOperators(p: var TParser, headNode: PNode,
                     limit: int, mode: TPrimaryMode): PNode =
   result = headNode
   # expand while operators have priorities higher than 'limit'
-  var opPrec = getPrecedence(p.tok, false)
+  var opPrec = getPrecedence(p.tok, p.strongSpaces)
   let modeB = if mode == pmTypeDef: pmTypeDesc else: mode
   # the operator itself must not start on a new line:
   # progress guaranteed
@@ -822,15 +808,14 @@ proc parseOperators(p: var TParser, headNode: PNode,
     var a = newNodeP(nkInfix, p)
     var opNode = newIdentNodeP(p.tok.ident, p) # skip operator:
     getTok(p)
-    flexComment(p, a)
-    optPar(p)
+    optInd(p, a)
     # read sub-expression with higher priority:
     var b = simpleExprAux(p, opPrec + leftAssoc, modeB)
     addSon(a, opNode)
     addSon(a, result)
     addSon(a, b)
     result = a
-    opPrec = getPrecedence(p.tok, false)
+    opPrec = getPrecedence(p.tok, p.strongSpaces)
 
 proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode =
   result = primary(p, mode)
@@ -843,10 +828,10 @@ proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode =
   result = parseOperators(p, result, limit, mode)
 
 proc simpleExpr(p: var TParser, mode = pmNormal): PNode =
-  when defined(nimpretty):
+  when defined(nimpretty2):
     inc p.em.doIndentMore
   result = simpleExprAux(p, -1, mode)
-  when defined(nimpretty):
+  when defined(nimpretty2):
     dec p.em.doIndentMore
 
 proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode =
@@ -911,9 +896,6 @@ proc parsePragma(p: var TParser): PNode =
   #| pragma = '{.' optInd (exprColonExpr comma?)* optPar ('.}' | '}')
   result = newNodeP(nkPragma, p)
   inc p.inPragma
-  when defined(nimpretty):
-    inc p.em.doIndentMore
-    inc p.em.keepIndents
   getTok(p)
   optInd(p, result)
   while p.tok.tokType notin {tkCurlyDotRi, tkCurlyRi, tkEof}:
@@ -926,22 +908,19 @@ proc parsePragma(p: var TParser): PNode =
       skipComment(p, a)
   optPar(p)
   if p.tok.tokType in {tkCurlyDotRi, tkCurlyRi}:
-    when defined(nimpretty):
+    when defined(nimpretty2):
       if p.tok.tokType == tkCurlyRi: curlyRiWasPragma(p.em)
     getTok(p)
   else:
     parMessage(p, "expected '.}'")
   dec p.inPragma
-  when defined(nimpretty):
-    dec p.em.doIndentMore
-    dec p.em.keepIndents
 
 proc identVis(p: var TParser; allowDot=false): PNode =
   #| identVis = symbol opr?  # postfix position
   #| identVisDot = symbol '.' optInd symbol opr?
   var a = parseSymbol(p)
   if p.tok.tokType == tkOpr:
-    when defined(nimpretty):
+    when defined(nimpretty2):
       starWasExportMarker(p.em)
     result = newNodeP(nkPostfix, p)
     addSon(result, newIdentNodeP(p.tok.ident, p))
@@ -981,7 +960,7 @@ proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode =
   while true:
     case p.tok.tokType
     of tkSymbol, tkAccent:
-      if withPragma in flags: a = identWithPragma(p, allowDot=withDot in flags)
+      if withPragma in flags: a = identWithPragma(p, allowDot=withdot in flags)
       else: a = parseSymbol(p)
       if a.kind == nkEmpty: return
     else: break
@@ -1020,7 +999,7 @@ proc parseTuple(p: var TParser, indentAllowed = false): PNode =
       var a = parseIdentColonEquals(p, {})
       addSon(result, a)
       if p.tok.tokType notin {tkComma, tkSemiColon}: break
-      when defined(nimpretty):
+      when defined(nimpretty2):
         commaWasSemicolon(p.em)
       getTok(p)
       skipComment(p, a)
@@ -1056,7 +1035,7 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
   var a: PNode
   result = newNodeP(nkFormalParams, p)
   addSon(result, p.emptyNode) # return type
-  when defined(nimpretty):
+  when defined(nimpretty2):
     inc p.em.doIndentMore
     inc p.em.keepIndents
   let hasParLe = p.tok.tokType == tkParLe and p.tok.indent < 0
@@ -1078,7 +1057,7 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
         break
       addSon(result, a)
       if p.tok.tokType notin {tkComma, tkSemiColon}: break
-      when defined(nimpretty):
+      when defined(nimpretty2):
         commaWasSemicolon(p.em)
       getTok(p)
       skipComment(p, a)
@@ -1090,10 +1069,10 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
     getTok(p)
     optInd(p, result)
     result.sons[0] = parseTypeDesc(p)
-  elif not retColon and not hasParLe:
+  elif not retColon and not hasParle:
     # Mark as "not there" in order to mark for deprecation in the semantic pass:
     result = p.emptyNode
-  when defined(nimpretty):
+  when defined(nimpretty2):
     dec p.em.doIndentMore
     dec p.em.keepIndents
 
@@ -1132,8 +1111,6 @@ proc parseProcExpr(p: var TParser; isExpr: bool; kind: TNodeKind): PNode =
     result = newNodeI(nkProcTy, info)
     if hasSignature:
       addSon(result, params)
-      if kind == nkFuncDef:
-        parMessage(p, "func keyword is not allowed in type descriptions, use proc with {.noSideEffect.} pragma instead")
       addSon(result, pragmas)
 
 proc isExprStart(p: TParser): bool =
@@ -1178,38 +1155,23 @@ proc parseTypeDescKAux(p: var TParser, kind: TNodeKind,
     result.addSon list
     parseSymbolList(p, list)
 
-proc parseVarTuple(p: var TParser): PNode
-
 proc parseFor(p: var TParser): PNode =
   #| forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt
   #| forExpr = forStmt
-  getTokNoInd(p)
   result = newNodeP(nkForStmt, p)
-  if p.tok.tokType == tkParLe:
-    addSon(result, parseVarTuple(p))
-  else:
-    var a = identWithPragma(p)
+  getTokNoInd(p)
+  var a = identWithPragma(p)
+  addSon(result, a)
+  while p.tok.tokType == tkComma:
+    getTok(p)
+    optInd(p, a)
+    a = identWithPragma(p)
     addSon(result, a)
-    while p.tok.tokType == tkComma:
-      getTok(p)
-      optInd(p, a)
-      if p.tok.tokType == tkParLe:
-        addSon(result, parseVarTuple(p))
-        break
-      a = identWithPragma(p)
-      addSon(result, a)
   eat(p, tkIn)
   addSon(result, parseExpr(p))
   colcom(p, result)
   addSon(result, parseStmt(p))
 
-template nimprettyDontTouch(body) =
-  when defined(nimpretty):
-    inc p.em.keepIndents
-  body
-  when defined(nimpretty):
-    dec p.em.keepIndents
-
 proc parseExpr(p: var TParser): PNode =
   #| expr = (blockExpr
   #|       | ifExpr
@@ -1219,26 +1181,12 @@ proc parseExpr(p: var TParser): PNode =
   #|       | tryExpr)
   #|       / simpleExpr
   case p.tok.tokType:
-  of tkBlock:
-    nimprettyDontTouch:
-      result = parseBlock(p)
-  of tkIf:
-    nimprettyDontTouch:
-      result = parseIfExpr(p, nkIfExpr)
-  of tkFor:
-    nimprettyDontTouch:
-      result = parseFor(p)
-  of tkWhen:
-    nimprettyDontTouch:
-      result = parseIfExpr(p, nkWhenExpr)
-  of tkCase:
-    # Currently we think nimpretty is good enough with case expressions,
-    # so it is allowed to touch them:
-    #nimprettyDontTouch:
-    result = parseCase(p)
-  of tkTry:
-    nimprettyDontTouch:
-      result = parseTry(p, isExpr=true)
+  of tkBlock: result = parseBlock(p)
+  of tkIf: result = parseIfExpr(p, nkIfExpr)
+  of tkFor: result = parseFor(p)
+  of tkWhen: result = parseIfExpr(p, nkWhenExpr)
+  of tkCase: result = parseCase(p)
+  of tkTry: result = parseTry(p, isExpr=true)
   else: result = simpleExpr(p)
 
 proc parseEnum(p: var TParser): PNode
@@ -1277,15 +1225,13 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
     else: result.kind = nkIteratorTy
   of tkEnum:
     if mode == pmTypeDef:
-      prettySection:
-        result = parseEnum(p)
+      result = parseEnum(p)
     else:
       result = newNodeP(nkEnumTy, p)
       getTok(p)
   of tkObject:
     if mode == pmTypeDef:
-      prettySection:
-        result = parseObject(p)
+      result = parseObject(p)
     else:
       result = newNodeP(nkObjectTy, p)
       getTok(p)
@@ -1300,6 +1246,7 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
     optInd(p, result)
     addSon(result, primary(p, pmNormal))
   of tkVar: result = parseTypeDescKAux(p, nkVarTy, mode)
+  of tkOut: result = parseTypeDescKAux(p, nkVarTy, mode)
   of tkRef: result = parseTypeDescKAux(p, nkRefTy, mode)
   of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, mode)
   of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, mode)
@@ -1309,30 +1256,14 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
     if mode != pmSkipSuffix:
       result = primarySuffix(p, result, baseInd, mode)
 
-proc binaryNot(p: var TParser; a: PNode): PNode =
-  if p.tok.tokType == tkNot:
-    let notOpr = newIdentNodeP(p.tok.ident, p)
-    getTok(p)
-    optInd(p, notOpr)
-    let b = parseExpr(p)
-    result = newNodeP(nkInfix, p)
-    result.add notOpr
-    result.add a
-    result.add b
-  else:
-    result = a
-
 proc parseTypeDesc(p: var TParser): PNode =
-  #| typeDesc = simpleExpr ('not' expr)?
-  newlineWasSplitting(p)
+  #| typeDesc = simpleExpr
   result = simpleExpr(p, pmTypeDesc)
-  result = binaryNot(p, result)
 
 proc parseTypeDefAux(p: var TParser): PNode =
-  #| typeDefAux = simpleExpr ('not' expr)?
+  #| typeDefAux = simpleExpr
   #|            | 'concept' typeClass
   result = simpleExpr(p, pmTypeDef)
-  result = binaryNot(p, result)
 
 proc makeCall(n: PNode): PNode =
   ## Creates a call if the given node isn't already a call.
@@ -1440,12 +1371,12 @@ proc parseExprStmt(p: var TParser): PNode =
       while true:
         getTok(p)
         optInd(p, result)
-        addSon(result, commandParam(p, isFirstParam, pmNormal))
+        addSon(result, commandParam(p, isFirstParam))
         if p.tok.tokType != tkComma: break
     elif p.tok.indent < 0 and isExprStart(p):
       result = newNode(nkCommand, a.info, @[a])
       while true:
-        addSon(result, commandParam(p, isFirstParam, pmNormal))
+        addSon(result, commandParam(p, isFirstParam))
         if p.tok.tokType != tkComma: break
         getTok(p)
         optInd(p, result)
@@ -1540,7 +1471,6 @@ proc parseReturnOrRaise(p: var TParser, kind: TNodeKind): PNode =
   elif p.tok.indent >= 0 and p.tok.indent <= p.currInd or not isExprStart(p):
     # NL terminates:
     addSon(result, p.emptyNode)
-    # nimpretty here!
   else:
     var e = parseExpr(p)
     e = postExprBlocks(p, e)
@@ -1740,7 +1670,7 @@ proc parseGenericParamList(p: var TParser): PNode =
     var a = parseGenericParam(p)
     addSon(result, a)
     if p.tok.tokType notin {tkComma, tkSemiColon}: break
-    when defined(nimpretty):
+    when defined(nimpretty2):
       commaWasSemicolon(p.em)
     getTok(p)
     skipComment(p, a)
@@ -1753,6 +1683,9 @@ proc parsePattern(p: var TParser): PNode =
   result = parseStmt(p)
   eat(p, tkCurlyRi)
 
+proc validInd(p: var TParser): bool =
+  result = p.tok.indent < 0 or p.tok.indent > p.currInd
+
 proc parseRoutine(p: var TParser, kind: TNodeKind): PNode =
   #| indAndComment = (IND{>} COMMENT)? | COMMENT?
   #| routine = optInd identVis pattern? genericParamList?
@@ -1818,8 +1751,23 @@ proc parseSection(p: var TParser, kind: TNodeKind,
   else:
     parMessage(p, errIdentifierExpected, p.tok)
 
+proc parseConstant(p: var TParser): PNode =
+  #| constant = identWithPragma (colon typeDesc)? '=' optInd expr indAndComment
+  result = newNodeP(nkConstDef, p)
+  addSon(result, identWithPragma(p))
+  if p.tok.tokType == tkColon:
+    getTok(p)
+    optInd(p, result)
+    addSon(result, parseTypeDesc(p))
+  else:
+    addSon(result, p.emptyNode)
+  eat(p, tkEquals)
+  optInd(p, result)
+  addSon(result, parseExpr(p))
+  indAndComment(p, result)
+
 proc parseEnum(p: var TParser): PNode =
-  #| enum = 'enum' optInd (symbol optPragmas optInd ('=' optInd expr COMMENT?)? comma?)+
+  #| enum = 'enum' optInd (symbol optInd ('=' optInd expr COMMENT?)? comma?)+
   result = newNodeP(nkEnumTy, p)
   getTok(p)
   addSon(result, p.emptyNode)
@@ -1829,35 +1777,25 @@ proc parseEnum(p: var TParser): PNode =
   while true:
     var a = parseSymbol(p)
     if a.kind == nkEmpty: return
-
-    var symPragma = a
-    var pragma: PNode
-    if p.tok.tokType == tkCurlyDotLe:
-      pragma = optPragmas(p)
-      symPragma = newNodeP(nkPragmaExpr, p)
-      addSon(symPragma, a)
-      addSon(symPragma, pragma)
-    # nimpretty support here
     if p.tok.indent >= 0 and p.tok.indent <= p.currInd:
-      add(result, symPragma)
+      add(result, a)
       break
-
     if p.tok.tokType == tkEquals and p.tok.indent < 0:
       getTok(p)
-      optInd(p, symPragma)
-      var b = symPragma
-      symPragma = newNodeP(nkEnumFieldDef, p)
-      addSon(symPragma, b)
-      addSon(symPragma, parseExpr(p))
+      optInd(p, a)
+      var b = a
+      a = newNodeP(nkEnumFieldDef, p)
+      addSon(a, b)
+      addSon(a, parseExpr(p))
       if p.tok.indent < 0 or p.tok.indent >= p.currInd:
-        rawSkipComment(p, symPragma)
+        rawSkipComment(p, a)
     if p.tok.tokType == tkComma and p.tok.indent < 0:
       getTok(p)
-      rawSkipComment(p, symPragma)
+      rawSkipComment(p, a)
     else:
       if p.tok.indent < 0 or p.tok.indent >= p.currInd:
-        rawSkipComment(p, symPragma)
-    addSon(result, symPragma)
+        rawSkipComment(p, a)
+    addSon(result, a)
     if p.tok.indent >= 0 and p.tok.indent <= p.currInd or
         p.tok.tokType == tkEof:
       break
@@ -1969,8 +1907,6 @@ proc parseObject(p: var TParser): PNode =
   result = newNodeP(nkObjectTy, p)
   getTok(p)
   if p.tok.tokType == tkCurlyDotLe and p.validInd:
-    # Deprecated since v0.20.0
-    parMessage(p, warnDeprecated, "type pragmas follow the type name; this form of writing pragmas is deprecated")
     addSon(result, parsePragma(p))
   else:
     addSon(result, p.emptyNode)
@@ -2043,42 +1979,13 @@ proc parseTypeClass(p: var TParser): PNode =
 proc parseTypeDef(p: var TParser): PNode =
   #|
   #| typeDef = identWithPragmaDot genericParamList? '=' optInd typeDefAux
-  #|             indAndComment? / identVisDot genericParamList? pragma '=' optInd typeDefAux
   #|             indAndComment?
   result = newNodeP(nkTypeDef, p)
-  var identifier = identVis(p, allowDot=true)
-  var identPragma = identifier
-  var pragma: PNode
-  var genericParam: PNode
-  var noPragmaYet = true
-
-  if p.tok.tokType == tkCurlyDotLe:
-    pragma = optPragmas(p)
-    identPragma = newNodeP(nkPragmaExpr, p)
-    addSon(identPragma, identifier)
-    addSon(identPragma, pragma)
-    noPragmaYet = false
-
+  addSon(result, identWithPragma(p, allowDot=true))
   if p.tok.tokType == tkBracketLe and p.validInd:
-    if not noPragmaYet:
-      # Deprecated since v0.20.0
-      parMessage(p, warnDeprecated, "pragma before generic parameter list is deprecated")
-    genericParam = parseGenericParamList(p)
+    addSon(result, parseGenericParamList(p))
   else:
-    genericParam = p.emptyNode
-
-  if noPragmaYet:
-    pragma = optPragmas(p)
-    if pragma.kind != nkEmpty:
-      identPragma = newNodeP(nkPragmaExpr, p)
-      addSon(identPragma, identifier)
-      addSon(identPragma, pragma)
-  elif p.tok.tokType == tkCurlyDotLe:
-    parMessage(p, errGenerated, "pragma already present")
-
-  addSon(result, identPragma)
-  addSon(result, genericParam)
-
+    addSon(result, p.emptyNode)
   if p.tok.tokType == tkEquals:
     result.info = parLineInfo(p)
     getTok(p)
@@ -2103,35 +2010,15 @@ proc parseVarTuple(p: var TParser): PNode =
   addSon(result, p.emptyNode)         # no type desc
   optPar(p)
   eat(p, tkParRi)
+  eat(p, tkEquals)
+  optInd(p, result)
+  addSon(result, parseExpr(p))
 
 proc parseVariable(p: var TParser): PNode =
   #| colonBody = colcom stmt doBlocks?
   #| variable = (varTuple / identColonEquals) colonBody? indAndComment
-  if p.tok.tokType == tkParLe:
-    result = parseVarTuple(p)
-    eat(p, tkEquals)
-    optInd(p, result)
-    addSon(result, parseExpr(p))
-  else: result = parseIdentColonEquals(p, {withPragma, withDot})
-  result[^1] = postExprBlocks(p, result[^1])
-  indAndComment(p, result)
-
-proc parseConstant(p: var TParser): PNode =
-  #| constant = (parseVarTuple / identWithPragma) (colon typeDesc)? '=' optInd expr indAndComment
   if p.tok.tokType == tkParLe: result = parseVarTuple(p)
-  else:
-    result = newNodeP(nkConstDef, p)
-    addSon(result, identWithPragma(p))
-    if p.tok.tokType == tkColon:
-      getTok(p)
-      optInd(p, result)
-      addSon(result, parseTypeDesc(p))
-    else:
-      addSon(result, p.emptyNode)
-  eat(p, tkEquals)
-  optInd(p, result)
-  #addSon(result, parseStmtListExpr(p))
-  addSon(result, parseExpr(p))
+  else: result = parseIdentColonEquals(p, {withPragma, withDot})
   result[^1] = postExprBlocks(p, result[^1])
   indAndComment(p, result)
 
@@ -2228,16 +2115,10 @@ proc complexOrSimpleStmt(p: var TParser): PNode =
       result = parseOperators(p, result, -1, pmNormal)
     else:
       result = parseSection(p, nkTypeSection, parseTypeDef)
-  of tkConst:
-    prettySection:
-      result = parseSection(p, nkConstSection, parseConstant)
-  of tkLet:
-    prettySection:
-      result = parseSection(p, nkLetSection, parseVariable)
-  of tkVar:
-    prettySection:
-      result = parseSection(p, nkVarSection, parseVariable)
+  of tkConst: result = parseSection(p, nkConstSection, parseConstant)
+  of tkLet: result = parseSection(p, nkLetSection, parseVariable)
   of tkWhen: result = parseIfOrWhen(p, nkWhenStmt)
+  of tkVar: result = parseSection(p, nkVarSection, parseVariable)
   of tkBind: result = parseBind(p, nkBindStmt)
   of tkMixin: result = parseBind(p, nkMixinStmt)
   of tkUsing: result = parseSection(p, nkUsingStmt, parseVariable)
@@ -2247,7 +2128,6 @@ proc parseStmt(p: var TParser): PNode =
   #| stmt = (IND{>} complexOrSimpleStmt^+(IND{=} / ';') DED)
   #|      / simpleStmt ^+ ';'
   if p.tok.indent > p.currInd:
-    # nimpretty support here
     result = newNodeP(nkStmtList, p)
     withInd(p):
       while true:
@@ -2324,7 +2204,6 @@ proc parseTopLevelStmt(p: var TParser): PNode =
   result = p.emptyNode
   # progress guaranteed
   while true:
-    # nimpretty support here
     if p.tok.indent != 0:
       if p.firstTok and p.tok.indent < 0: discard
       elif p.tok.tokType != tkSemiColon:
@@ -2358,8 +2237,10 @@ proc parseString*(s: string; cache: IdentCache; config: ConfigRef;
   stream.lineOffset = line
 
   var parser: TParser
+  # XXX for now the builtin 'parseStmt/Expr' functions do not know about strong
+  # spaces...
   parser.lex.errorHandler = errorHandler
-  openParser(parser, AbsoluteFile filename, stream, cache, config)
+  openParser(parser, AbsoluteFile filename, stream, cache, config, false)
 
   result = parser.parseAll
   closeParser(parser)

compiler/passaux.nim

@@ -10,7 +10,7 @@
 ## implements some little helper passes
 
 import
-  ast, passes, idents, msgs, options, idgen, lineinfos
+  strutils, ast, astalgo, passes, idents, msgs, options, idgen, lineinfos
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -30,6 +30,6 @@ proc verboseProcess(context: PPassContext, n: PNode): PNode =
     # system.nim deactivates all hints, for verbosity:3 we want the processing
     # messages nonetheless, so we activate them again unconditionally:
     incl(v.config.notes, hintProcessing)
-    message(v.config, n.info, hintProcessing, $idgen.gFrontEndId)
+    message(v.config, n.info, hintProcessing, $idgen.gFrontendId)
 
 const verbosePass* = makePass(open = verboseOpen, process = verboseProcess)

compiler/passes.nim

@@ -11,9 +11,9 @@
 ## `TPass` interface.
 
 import
-  options, ast, llstream, msgs,
-  idents,
-  syntaxes, idgen, modulegraphs, reorder, rod,
+  strutils, options, ast, astalgo, llstream, msgs, platform, os,
+  condsyms, idents, renderer, types, extccomp, math, magicsys, nversion,
+  nimsets, syntaxes, times, idgen, modulegraphs, reorder, rod,
   lineinfos, pathutils
 
 type
@@ -67,21 +67,21 @@ proc carryPasses*(g: ModuleGraph; nodes: PNode, module: PSym;
 
 proc openPasses(g: ModuleGraph; a: var TPassContextArray;
                 module: PSym) =
-  for i in 0 ..< g.passes.len:
+  for i in countup(0, g.passes.len - 1):
     if not isNil(g.passes[i].open):
       a[i] = g.passes[i].open(g, module)
     else: a[i] = nil
 
 proc closePasses(graph: ModuleGraph; a: var TPassContextArray) =
   var m: PNode = nil
-  for i in 0 ..< graph.passes.len:
+  for i in countup(0, graph.passes.len - 1):
     if not isNil(graph.passes[i].close): m = graph.passes[i].close(graph, a[i], m)
     a[i] = nil                # free the memory here
 
 proc processTopLevelStmt(graph: ModuleGraph, n: PNode, a: var TPassContextArray): bool =
   # this implements the code transformation pipeline
   var m = n
-  for i in 0 ..< graph.passes.len:
+  for i in countup(0, graph.passes.len - 1):
     if not isNil(graph.passes[i].process):
       m = graph.passes[i].process(a[i], m)
       if isNil(m): return false
@@ -90,7 +90,7 @@ proc processTopLevelStmt(graph: ModuleGraph, n: PNode, a: var TPassContextArray)
 proc resolveMod(conf: ConfigRef; module, relativeTo: string): FileIndex =
   let fullPath = findModule(conf, module, relativeTo)
   if fullPath.isEmpty:
-    result = InvalidFileIdx
+    result = InvalidFileIDX
   else:
     result = fileInfoIdx(conf, fullPath)
 
@@ -102,9 +102,9 @@ proc processImplicits(graph: ModuleGraph; implicits: seq[string], nodeKind: TNod
   for module in items(implicits):
     # implicit imports should not lead to a module importing itself
     if m.position != resolveMod(graph.config, module, relativeTo).int32:
-      var importStmt = newNodeI(nodeKind, m.info)
+      var importStmt = newNodeI(nodeKind, gCmdLineInfo)
       var str = newStrNode(nkStrLit, module)
-      str.info = m.info
+      str.info = gCmdLineInfo
       importStmt.addSon str
       if not processTopLevelStmt(graph, importStmt, a): break
 
@@ -113,19 +113,6 @@ const
     nkMacroDef, nkConverterDef, nkIteratorDef, nkFuncDef, nkPragma,
     nkExportStmt, nkExportExceptStmt, nkFromStmt, nkImportStmt, nkImportExceptStmt}
 
-proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
-  if sfMainModule in module.flags:
-    graph.config.mainPackageId = module.owner.id
-  # don't be verbose unless the module belongs to the main package:
-  if module.owner.id == graph.config.mainPackageId:
-    graph.config.notes = graph.config.mainPackageNotes
-  else:
-    if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes
-    graph.config.notes = graph.config.foreignPackageNotes
-
-proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
-  result = module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")
-
 proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {.discardable.} =
   if graph.stopCompile(): return true
   var
@@ -133,7 +120,6 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
     a: TPassContextArray
     s: PLLStream
     fileIdx = module.fileIdx
-  prepareConfigNotes(graph, module)
   if module.id < 0:
     # new module caching mechanism:
     for i in 0 ..< graph.passes.len:
@@ -169,22 +155,19 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
     while true:
       openParsers(p, fileIdx, s, graph.cache, graph.config)
 
-      if module.owner == nil or module.owner.name.s != "stdlib" or module.name.s == "distros":
+      if sfSystemModule notin module.flags:
         # XXX what about caching? no processing then? what if I change the
         # modules to include between compilation runs? we'd need to track that
         # in ROD files. I think we should enable this feature only
         # for the interactive mode.
-        if module.name.s != "nimscriptapi":
-          processImplicits graph, graph.config.implicitImports, nkImportStmt, a, module
-          processImplicits graph, graph.config.implicitIncludes, nkIncludeStmt, a, module
+        processImplicits graph, graph.config.implicitImports, nkImportStmt, a, module
+        processImplicits graph, graph.config.implicitIncludes, nkIncludeStmt, a, module
 
       while true:
         if graph.stopCompile(): break
         var n = parseTopLevelStmt(p)
         if n.kind == nkEmpty: break
-        if (sfSystemModule notin module.flags and
-            ({sfNoForward, sfReorder} * module.flags != {} or
-            codeReordering in graph.config.features)):
+        if {sfNoForward, sfReorder} * module.flags != {}:
           # read everything, no streaming possible
           var sl = newNodeI(nkStmtList, n.info)
           sl.add n
@@ -192,7 +175,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
             var n = parseTopLevelStmt(p)
             if n.kind == nkEmpty: break
             sl.add n
-          if sfReorder in module.flags or codeReordering in graph.config.features:
+          if sfReorder in module.flags:
             sl = reorder(graph, sl, module)
           discard processTopLevelStmt(graph, sl, a)
           break

compiler/pathutils.nim

@@ -9,17 +9,20 @@
 
 ## Path handling utilities for Nim. Strictly typed code in order
 ## to avoid the never ending time sink in getting path handling right.
+## Might be a candidate for the stdlib later.
 
-import os, pathnorm
+import os, strutils
 
 type
   AbsoluteFile* = distinct string
   AbsoluteDir* = distinct string
   RelativeFile* = distinct string
   RelativeDir* = distinct string
-  AnyPath* = AbsoluteFile|AbsoluteDir|RelativeFile|RelativeDir
 
-proc isEmpty*(x: AnyPath): bool {.inline.} = x.string.len == 0
+proc isEmpty*(x: AbsoluteFile): bool {.inline.} = x.string.len == 0
+proc isEmpty*(x: AbsoluteDir): bool {.inline.} = x.string.len == 0
+proc isEmpty*(x: RelativeFile): bool {.inline.} = x.string.len == 0
+proc isEmpty*(x: RelativeDir): bool {.inline.} = x.string.len == 0
 
 proc copyFile*(source, dest: AbsoluteFile) =
   os.copyFile(source.string, dest.string)
@@ -42,37 +45,172 @@ proc cmpPaths*(x, y: AbsoluteDir): int {.borrow.}
 
 proc createDir*(x: AbsoluteDir) {.borrow.}
 
-proc toAbsoluteDir*(path: string): AbsoluteDir =
-  result = if path.isAbsolute: AbsoluteDir(path)
-           else: AbsoluteDir(getCurrentDir() / path)
+type
+  PathIter = object
+    i, prev: int
+    notFirst: bool
 
-proc `$`*(x: AnyPath): string = x.string
+proc hasNext(it: PathIter; x: string): bool =
+  it.i < x.len
+
+proc next(it: var PathIter; x: string): (int, int) =
+  it.prev = it.i
+  if not it.notFirst and x[it.i] in {DirSep, AltSep}:
+    # absolute path:
+    inc it.i
+  else:
+    while it.i < x.len and x[it.i] notin {DirSep, AltSep}: inc it.i
+  if it.i > it.prev:
+    result = (it.prev, it.i-1)
+  elif hasNext(it, x):
+    result = next(it, x)
+
+  # skip all separators:
+  while it.i < x.len and x[it.i] in {DirSep, AltSep}: inc it.i
+  it.notFirst = true
+
+iterator dirs(x: string): (int, int) =
+  var it: PathIter
+  while hasNext(it, x): yield next(it, x)
+
+when false:
+  iterator dirs(x: string): (int, int) =
+    var i = 0
+    var first = true
+    while i < x.len:
+      let prev = i
+      if first and x[i] in {DirSep, AltSep}:
+        # absolute path:
+        inc i
+      else:
+        while i < x.len and x[i] notin {DirSep, AltSep}: inc i
+      if i > prev:
+        yield (prev, i-1)
+      first = false
+      # skip all separators:
+      while i < x.len and x[i] in {DirSep, AltSep}: inc i
+
+proc isDot(x: string; bounds: (int, int)): bool =
+  bounds[1] == bounds[0] and x[bounds[0]] == '.'
+
+proc isDotDot(x: string; bounds: (int, int)): bool =
+  bounds[1] == bounds[0] + 1 and x[bounds[0]] == '.' and x[bounds[0]+1] == '.'
+
+proc isSlash(x: string; bounds: (int, int)): bool =
+  bounds[1] == bounds[0] and x[bounds[0]] in {DirSep, AltSep}
+
+const canonDirSep = when isMainModule: '/' else: DirSep
+
+proc canon(x: string; result: var string; state: var int) =
+  # state: 0th bit set if isAbsolute path. Other bits count
+  # the number of path components.
+  for b in dirs(x):
+    if (state shr 1 == 0) and isSlash(x, b):
+      result.add canonDirSep
+      state = state or 1
+    elif result.len > (state and 1) and isDotDot(x, b):
+      var d = result.len
+      # f/..
+      while (d-1) > (state and 1) and result[d-1] notin {DirSep, AltSep}:
+        dec d
+      if d > 0: setLen(result, d-1)
+    elif isDot(x, b):
+      discard "discard the dot"
+    elif b[1] >= b[0]:
+      if result.len > 0 and result[^1] notin {DirSep, AltSep}:
+        result.add canonDirSep
+      result.add substr(x, b[0], b[1])
+    inc state, 2
+
+proc canon(x: string): string =
+  # - Turn multiple slashes into single slashes.
+  # - Resolve '/foo/../bar' to '/bar'.
+  # - Remove './' from the path.
+  result = newStringOfCap(x.len)
+  var state = 0
+  canon(x, result, state)
+
+when FileSystemCaseSensitive:
+  template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b)
+else:
+  template `!=?`(a, b: char): bool = a != b
+
+proc relativeTo(full, base: string; sep = canonDirSep): string =
+  if full.len == 0: return ""
+  var f, b: PathIter
+  var ff = (0, -1)
+  var bb = (0, -1) # (int, int)
+  result = newStringOfCap(full.len)
+  # skip the common prefix:
+  while f.hasNext(full) and b.hasNext(base):
+    ff = next(f, full)
+    bb = next(b, base)
+    let diff = ff[1] - ff[0]
+    if diff != bb[1] - bb[0]: break
+    var same = true
+    for i in 0..diff:
+      if full[i + ff[0]] !=? base[i + bb[0]]:
+        same = false
+        break
+    if not same: break
+    ff = (0, -1)
+    bb = (0, -1)
+  #  for i in 0..diff:
+  #    result.add base[i + bb[0]]
+
+  # /foo/bar/xxx/ -- base
+  # /foo/bar/baz  -- full path
+  #   ../baz
+  # every directory that is in 'base', needs to add '..'
+  while true:
+    if bb[1] >= bb[0]:
+      if result.len > 0 and result[^1] != sep:
+        result.add sep
+      result.add ".."
+    if not b.hasNext(base): break
+    bb = b.next(base)
+
+  # add the rest of 'full':
+  while true:
+    if ff[1] >= ff[0]:
+      if result.len > 0 and result[^1] != sep:
+        result.add sep
+      for i in 0..ff[1] - ff[0]:
+        result.add full[i + ff[0]]
+    if not f.hasNext(full): break
+    ff = f.next(full)
 
 when true:
-  proc eqImpl(x, y: string): bool {.inline.} =
-    result = cmpPaths(x, y) == 0
+  proc eqImpl(x, y: string): bool =
+    when FileSystemCaseSensitive:
+      result = cmpIgnoreCase(canon x, canon y) == 0
+    else:
+      result = canon(x) == canon(y)
 
-  proc `==`*[T: AnyPath](x, y: T): bool = eqImpl(x.string, y.string)
+  proc `==`*(x, y: AbsoluteFile): bool = eqImpl(x.string, y.string)
+  proc `==`*(x, y: AbsoluteDir): bool = eqImpl(x.string, y.string)
+  proc `==`*(x, y: RelativeFile): bool = eqImpl(x.string, y.string)
+  proc `==`*(x, y: RelativeDir): bool = eqImpl(x.string, y.string)
 
   proc `/`*(base: AbsoluteDir; f: RelativeFile): AbsoluteFile =
     #assert isAbsolute(base.string)
     assert(not isAbsolute(f.string))
     result = AbsoluteFile newStringOfCap(base.string.len + f.string.len)
     var state = 0
-    addNormalizePath(base.string, result.string, state)
-    addNormalizePath(f.string, result.string, state)
+    canon(base.string, result.string, state)
+    canon(f.string, result.string, state)
 
   proc `/`*(base: AbsoluteDir; f: RelativeDir): AbsoluteDir =
     #assert isAbsolute(base.string)
     assert(not isAbsolute(f.string))
     result = AbsoluteDir newStringOfCap(base.string.len + f.string.len)
     var state = 0
-    addNormalizePath(base.string, result.string, state)
-    addNormalizePath(f.string, result.string, state)
+    canon(base.string, result.string, state)
+    canon(f.string, result.string, state)
 
   proc relativeTo*(fullPath: AbsoluteFile, baseFilename: AbsoluteDir;
-                   sep = DirSep): RelativeFile =
-    RelativeFile(relativePath(fullPath.string, baseFilename.string, sep))
+                   sep = canonDirSep): RelativeFile =
+    RelativeFile(relativeTo(fullPath.string, baseFilename.string, sep))
 
   proc toAbsolute*(file: string; base: AbsoluteDir): AbsoluteFile =
     if isAbsolute(file): result = AbsoluteFile(file)
@@ -87,14 +225,37 @@ when true:
   proc writeFile*(x: AbsoluteFile; content: string) {.borrow.}
 
 when isMainModule:
+  doAssert canon"/foo/../bar" == "/bar"
+  doAssert canon"foo/../bar" == "bar"
+
+  doAssert canon"/f/../bar///" == "/bar"
+  doAssert canon"f/..////bar" == "bar"
+
+  doAssert canon"../bar" == "../bar"
+  doAssert canon"/../bar" == "/../bar"
+
+  doAssert canon("foo/../../bar/") == "../bar"
+  doAssert canon("./bla/blob/") == "bla/blob"
+  doAssert canon(".hiddenFile") == ".hiddenFile"
+  doAssert canon("./bla/../../blob/./zoo.nim") == "../blob/zoo.nim"
+
+  doAssert canon("C:/file/to/this/long") == "C:/file/to/this/long"
+  doAssert canon("") == ""
+  doAssert canon("foobar") == "foobar"
+  doAssert canon("f/////////") == "f"
+
+  doAssert relativeTo("/foo/bar//baz.nim", "/foo") == "bar/baz.nim"
+
+  doAssert relativeTo("/Users/me/bar/z.nim", "/Users/other/bad") == "../../me/bar/z.nim"
+
+  doAssert relativeTo("/Users/me/bar/z.nim", "/Users/other") == "../me/bar/z.nim"
+  doAssert relativeTo("/Users///me/bar//z.nim", "//Users/") == "me/bar/z.nim"
+  doAssert relativeTo("/Users/me/bar/z.nim", "/Users/me") == "bar/z.nim"
+  doAssert relativeTo("", "/users/moo") == ""
+  doAssert relativeTo("foo", "") == "foo"
+
   doAssert AbsoluteDir"/Users/me///" / RelativeFile"z.nim" == AbsoluteFile"/Users/me/z.nim"
-  doAssert relativePath("/foo/bar.nim", "/foo/", '/') == "bar.nim"
-  doAssert $RelativeDir"foo/bar" == "foo/bar"
-  doAssert RelativeDir"foo/bar" == RelativeDir"foo/bar"
-  doAssert RelativeFile"foo/bar".changeFileExt(".txt") == RelativeFile"foo/bar.txt"
-  doAssert RelativeFile"foo/bar".addFileExt(".txt") == RelativeFile"foo/bar.txt"
-  doAssert not RelativeDir"foo/bar".isEmpty
-  doAssert RelativeDir"".isEmpty
+  doAssert relativeTo("/foo/bar.nim", "/foo/") == "bar.nim"
 
 when isMainModule and defined(windows):
   let nasty = string(AbsoluteDir(r"C:\Users\rumpf\projects\nim\tests\nimble\nimbleDir\linkedPkgs\pkgB-#head\../../simplePkgs/pkgB-#head/") / RelativeFile"pkgA/module.nim")

compiler/patterns.nim

@@ -11,7 +11,8 @@
 ## macro support.
 
 import
-  ast, types, semdata, sigmatch, idents, aliases, parampatterns, trees
+  ast, astalgo, types, semdata, sigmatch, msgs, idents, aliases, parampatterns,
+  trees
 
 type
   TPatternContext = object
@@ -47,7 +48,7 @@ proc canonKind(n: PNode): TNodeKind =
 proc sameKinds(a, b: PNode): bool {.inline.} =
   result = a.kind == b.kind or a.canonKind == b.canonKind
 
-proc sameTrees*(a, b: PNode): bool =
+proc sameTrees(a, b: PNode): bool =
   if sameKinds(a, b):
     case a.kind
     of nkSym: result = a.sym == b.sym
@@ -59,7 +60,7 @@ proc sameTrees*(a, b: PNode): bool =
     of nkType: result = sameTypeOrNil(a.typ, b.typ)
     else:
       if sonsLen(a) == sonsLen(b):
-        for i in 0 ..< sonsLen(a):
+        for i in countup(0, sonsLen(a) - 1):
           if not sameTrees(a.sons[i], b.sons[i]): return
         result = true
 
@@ -77,7 +78,7 @@ proc checkTypes(c: PPatternContext, p: PSym, n: PNode): bool =
     result = matchNodeKinds(p.constraint, n)
     if not result: return
   if isNil(n.typ):
-    result = p.typ.kind in {tyVoid, tyTyped}
+    result = p.typ.kind in {tyVoid, tyStmt}
   else:
     result = sigmatch.argtypeMatches(c.c, p.typ, n.typ, fromHlo = true)
 
@@ -181,7 +182,7 @@ proc matches(c: PPatternContext, p, n: PNode): bool =
       if isPatternParam(c, v) and v.sym.typ.kind == tyVarargs:
         var arglist: PNode
         if plen <= sonsLen(n):
-          for i in 0 .. plen - 2:
+          for i in countup(0, plen - 2):
             if not matches(c, p.sons[i], n.sons[i]): return
           if plen == sonsLen(n) and lastSon(n).kind == nkHiddenStdConv and
               lastSon(n).sons[1].kind == nkBracket:
@@ -193,16 +194,16 @@ proc matches(c: PPatternContext, p, n: PNode): bool =
             arglist = newNodeI(nkArgList, n.info, sonsLen(n) - plen + 1)
             # f(1, 2, 3)
             # p(X)
-            for i in 0 .. sonsLen(n) - plen:
+            for i in countup(0, sonsLen(n) - plen):
               arglist.sons[i] = n.sons[i + plen - 1]
           return bindOrCheck(c, v.sym, arglist)
         elif plen-1 == sonsLen(n):
-          for i in 0 .. plen - 2:
+          for i in countup(0, plen - 2):
             if not matches(c, p.sons[i], n.sons[i]): return
           arglist = newNodeI(nkArgList, n.info)
           return bindOrCheck(c, v.sym, arglist)
       if plen == sonsLen(n):
-        for i in 0 ..< sonsLen(p):
+        for i in countup(0, sonsLen(p) - 1):
           if not matches(c, p.sons[i], n.sons[i]): return
         result = true
 
@@ -240,7 +241,7 @@ proc aliasAnalysisRequested(params: PNode): bool =
       if whichAlias(param) != aqNone: return true
 
 proc addToArgList(result, n: PNode) =
-  if n.typ != nil and n.typ.kind != tyTyped:
+  if n.typ != nil and n.typ.kind != tyStmt:
     if n.kind != nkArgList: result.add(n)
     else:
       for i in 0 ..< n.len: result.add(n.sons[i])

compiler/platform.nim

@@ -21,7 +21,7 @@ type
                     # conditionals to condsyms (end of module).
     osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris,
     osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osAix, osPalmos, osQnx,
-    osAmiga, osAtari, osNetware, osMacos, osMacosx, osHaiku, osAndroid, osVxWorks
+    osAmiga, osAtari, osNetware, osMacos, osMacosx, osHaiku, osAndroid, osVxworks
     osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch
 
 type
@@ -181,7 +181,7 @@ type
     cpuNone, cpuI386, cpuM68k, cpuAlpha, cpuPowerpc, cpuPowerpc64,
     cpuPowerpc64el, cpuSparc, cpuVm, cpuIa64, cpuAmd64, cpuMips, cpuMipsel,
     cpuArm, cpuArm64, cpuJS, cpuNimVM, cpuAVR, cpuMSP430, cpuSparc64,
-    cpuMips64, cpuMips64el, cpuRiscV64, cpuWasm32
+    cpuMips64, cpuMips64el, cpuRiscV64
 
 type
   TEndian* = enum
@@ -213,8 +213,7 @@ const
     (name: "sparc64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64),
     (name: "mips64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64),
     (name: "mips64el", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
-    (name: "riscv64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
-    (name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32)]
+    (name: "riscv64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)]
 
 type
   Target* = object
@@ -237,25 +236,17 @@ proc setTarget*(t: var Target; o: TSystemOS, c: TSystemCPU) =
   t.tnl = OS[o].newLine
 
 proc nameToOS*(name: string): TSystemOS =
-  for i in succ(osNone) .. high(TSystemOS):
+  for i in countup(succ(osNone), high(TSystemOS)):
     if cmpIgnoreStyle(name, OS[i].name) == 0:
       return i
   result = osNone
 
-proc listOSnames*(): seq[string] =
-  for i in succ(osNone) .. high(TSystemOS):
-    result.add OS[i].name
-
 proc nameToCPU*(name: string): TSystemCPU =
-  for i in succ(cpuNone) .. high(TSystemCPU):
+  for i in countup(succ(cpuNone), high(TSystemCPU)):
     if cmpIgnoreStyle(name, CPU[i].name) == 0:
       return i
   result = cpuNone
 
-proc listCPUnames*(): seq[string] =
-  for i in succ(cpuNone) .. high(TSystemCPU):
-    result.add CPU[i].name
-
 proc setTargetFromSystem*(t: var Target) =
   t.hostOS = nameToOS(system.hostOS)
   t.hostCPU = nameToCPU(system.hostCPU)

compiler/plugins/itersgen.nim

@@ -9,7 +9,9 @@
 
 ## Plugin to transform an inline iterator into a data structure.
 
-import ".." / [ast, lookups, semdata, lambdalifting, msgs]
+import ".." / [ast, astalgo,
+  magicsys, lookups, semdata,
+  lambdalifting, msgs]
 
 proc iterToProcImpl*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkStmtList, n.info)

compiler/plugins/locals.nim

@@ -9,7 +9,7 @@
 
 ## The builtin 'system.locals' implemented as a plugin.
 
-import ".." / [ast, astalgo,
+import ".." / [pluginsupport, ast, astalgo,
   magicsys, lookups, semdata, lowerings]
 
 proc semLocals*(c: PContext, n: PNode): PNode =
@@ -26,7 +26,7 @@ proc semLocals*(c: PContext, n: PNode): PNode =
       #if it.owner != c.p.owner: return result
       if it.kind in skLocalVars and
           it.typ.skipTypes({tyGenericInst, tyVar}).kind notin
-            {tyVarargs, tyOpenArray, tyTypeDesc, tyStatic, tyUntyped, tyTyped, tyEmpty}:
+            {tyVarargs, tyOpenArray, tyTypeDesc, tyStatic, tyExpr, tyStmt, tyEmpty}:
 
         var field = newSym(skField, it.name, getCurrOwner(c), n.info)
         field.typ = it.typ.skipTypes({tyVar})