bump version to 0.18.0

* Adds description of the difference between ``fmt`` and ``&``.
* Moves runnable examples to bottom of file.
* Separates examples at the top of module better.

.gitignore

@@ -57,3 +57,7 @@ dist/
 # Private directories and files (IDEs)
 .*/
 ~*
+
+# testament cruft
+testresults/
+test.txt

.gitlab-ci.yml

@@ -8,7 +8,7 @@ stages:
 
 .linux_set_path: &linux_set_path_def
   before_script:
-    - export PATH=$(pwd)/bin:$PATH
+    - export PATH=$(pwd)/bin${PATH:+:$PATH}
   tags:
     - linux
 

.travis.yml

@@ -16,10 +16,12 @@ addons:
     - libcurl4-openssl-dev
     - libsdl1.2-dev
     - libgc-dev
+    - libsfml-dev
 
 before_install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install boehmgc; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install sfml; fi
 
 before_script:
   - set -e
@@ -28,7 +30,7 @@ before_script:
   - sh build.sh
   - cd ..
   - sed -i -e 's,cc = gcc,cc = clang,' config/nim.cfg
-  - export PATH=$(pwd)/bin:$PATH
+  - export PATH=$(pwd)/bin${PATH:+:$PATH}
 script:
   - nim c koch
   - ./koch boot
@@ -44,3 +46,7 @@ script:
   - tests/testament/tester --pedantic all -d:nimCoroutines
   - ./koch web
   - ./koch csource
+  - ./koch nimsuggest
+#  - nim c -r nimsuggest/tester
+  - ( ! grep -F '.. code-block' -l -r --include '*.html' --exclude contributing.html --exclude docgen.html --exclude tut2.html )
+  - ( ! grep -F '..code-block' -l -r --include '*.html' --exclude contributing.html --exclude docgen.html --exclude tut2.html )

appveyor.yml

@@ -24,6 +24,7 @@ environment:
     #   platform: x86
 
 install:
+  - ps: Install-Product node 8 # node 8 or later is required to test js async stuff
   - MKDIR %CD%\DIST
   - MKDIR %CD%\DIST\PCRE
   - nuget install pcre -Verbosity quiet -Version 8.33.0.1 -OutputDirectory %CD%\DIST\PCRE

changelog.md

@@ -0,0 +1,663 @@
+## v0.18.0 - 01/03/2018
+
+### Changes affecting backwards compatibility
+
+#### Breaking changes in the standard library
+
+- The ``[]`` proc for strings now raises an ``IndexError`` exception when
+  the specified slice is out of bounds. See issue
+  [#6223](https://github.com/nim-lang/Nim/issues/6223) for more details.
+  You can use ``substr(str, start, finish)`` to get the old behaviour back,
+  see [this commit](https://github.com/nim-lang/nimbot/commit/98cc031a27ea89947daa7f0bb536bcf86462941f) for an example.
+
+- ``strutils.split`` and ``strutils.rsplit`` with an empty string and a
+  separator now returns that empty string.
+  See issue [#4377](https://github.com/nim-lang/Nim/issues/4377).
+
+- Arrays of char cannot be converted to ``cstring`` anymore, pointers to
+  arrays of char can! This means ``$`` for arrays can finally exist
+  in ``system.nim`` and do the right thing. This means ``$myArrayOfChar`` changed
+  its behaviour! Compile with ``-d:nimNoArrayToString`` to see where to fix your
+  code.
+
+- `reExtended` is no longer default for the `re` constructor in the `re`
+  module.
+
+- The behavior of ``$`` has been changed for all standard library collections. The
+  collection-to-string implementations now perform proper quoting and escaping of
+  strings and chars.
+
+- `newAsyncSocket` taking an `AsyncFD` now runs `setBlocking(false)` on the
+  fd.
+
+- ``mod`` and bitwise ``and`` do not produce ``range`` subtypes anymore. This
+  turned out to be more harmful than helpful and the language is simpler
+  without this special typing rule.
+
+- ``formatFloat``/``formatBiggestFloat`` now support formatting floats with zero
+  precision digits. The previous ``precision = 0`` behavior (default formatting)
+  is now available via ``precision = -1``.
+
+- Moved from stdlib into Nimble packages:
+  - [``basic2d``](https://github.com/nim-lang/basic2d)
+    _deprecated: use ``glm``, ``arraymancer``, ``neo``, or another package instead_
+  - [``basic3d``](https://github.com/nim-lang/basic3d)
+    _deprecated: use ``glm``, ``arraymancer``, ``neo``, or another package instead_
+  - [``gentabs``](https://github.com/lcrees/gentabs)
+  - [``libuv``](https://github.com/lcrees/libuv)
+  - [``numeric``](https://github.com/lcrees/polynumeric)
+  - [``poly``](https://github.com/lcrees/polynumeric)
+  - [``pdcurses``](https://github.com/lcrees/pdcurses)
+  - [``romans``](https://github.com/lcrees/romans)
+  - [``libsvm``](https://github.com/nim-lang/libsvm_legacy)
+  - [``joyent_http_parser``](https://github.com/nim-lang/joyent_http_parser)
+
+- Proc [toCountTable](https://nim-lang.org/docs/tables.html#toCountTable,openArray[A])
+  now produces a `CountTable` with values correspoding to the number of occurrences
+  of the key in the input. It used to produce a table with all values set to `1`.
+
+  Counting occurrences in a sequence used to be:
+
+  ```nim
+  let mySeq = @[1, 2, 1, 3, 1, 4]
+  var myCounter = initCountTable[int]()
+
+  for item in mySeq:
+    myCounter.inc item
+  ```
+
+  Now, you can simply do:
+
+  ```nim
+  let
+    mySeq = @[1, 2, 1, 3, 1, 4]
+    myCounter = mySeq.toCountTable()
+  ```
+
+- If you use ``--dynlibOverride:ssl`` with OpenSSL 1.0.x, you now have to
+  define ``openssl10`` symbol (``-d:openssl10``). By default OpenSSL 1.1.x is
+  assumed.
+
+- ``newNativeSocket`` is now named ``createNativeSocket``.
+
+- ``newAsyncNativeSocket`` is now named ``createAsyncNativeSocket``
+  and it no longer raises an OS error but returns an ``osInvalidSocket`` when
+  creation fails.
+
+- The ``securehash`` module is now deprecated. Instead import ``std / sha1``.
+
+- The ``readPasswordFromStdin`` proc has been moved from the ``rdstdin``
+  to the ``terminal`` module, thus it does not depend on linenoise anymore.
+
+#### Breaking changes in the compiler
+
+- ``\n`` is now only the single line feed character like in most
+  other programming languages. The new platform specific newline escape sequence is
+  written as ``\p``. This change only affects the Windows platform.
+
+- The overloading rules changed slightly so that constrained generics are
+  preferred over unconstrained generics. (Bug #6526)
+
+- We changed how array accesses "from backwards" like ``a[^1]`` or ``a[0..^1]`` are
+  implemented. These are now implemented purely in ``system.nim`` without compiler
+  support. There is a new "heterogenous" slice type ``system.HSlice`` that takes 2
+  generic parameters which can be ``BackwardsIndex`` indices. ``BackwardsIndex`` is
+  produced by ``system.^``.
+  This means if you overload ``[]`` or ``[]=`` you need to ensure they also work
+  with ``system.BackwardsIndex`` (if applicable for the accessors).
+
+- The parsing rules of ``if`` expressions were changed so that multiple
+  statements are allowed in the branches. We found few code examples that
+  now fail because of this change, but here is one:
+
+```nim
+t[ti] = if exp_negative: '-' else: '+'; inc(ti)
+```
+
+This now needs to be written as:
+
+```nim
+t[ti] = (if exp_negative: '-' else: '+'); inc(ti)
+```
+
+- The experimental overloading of the dot ``.`` operators now take
+  an ``untyped``` parameter as the field name, it used to be
+  a ``static[string]``. You can use ``when defined(nimNewDot)`` to make
+  your code work with both old and new Nim versions.
+  See [special-operators](https://nim-lang.org/docs/manual.html#special-operators)
+  for more information.
+
+- ``yield`` (or ``await`` which is mapped to ``yield``) never worked reliably
+  in an array, seq or object constructor and is now prevented at compile-time.
+
+### Library additions
+
+- **Added ``sequtils.mapLiterals`` for easier construction of array and tuple literals.**
+
+- Added ``system.runnableExamples`` to make examples in Nim's documentation easier
+  to write and test. The examples are tested as the last step of
+  ``nim doc``.
+
+- Implemented ``getIoHandler`` proc in the ``asyncdispatch`` module that allows
+  you to retrieve the underlying IO Completion Port or ``Selector[AsyncData]``
+  object in the specified dispatcher.
+
+- For string formatting / interpolation a new module
+  called [strformat](https://nim-lang.org/docs/strformat.html) has been added
+  to the stdlib.
+
+- The `ReadyKey` type in the selectors module now contains an ``errorCode``
+  field to help distinguish between ``Event.Error`` events.
+
+- Implemented an `accept` proc that works on a `SocketHandle` in
+  ``nativesockets``.
+
+- Added ``algorithm.rotateLeft``.
+
+- Added ``typetraits.$`` as an alias for ``typetraits.name``.
+
+- Added ``system.getStackTraceEntries`` that allows you to access the stack
+  trace in a structured manner without string parsing.
+
+- Added ``parseutils.parseSaturatedNatural``.
+
+- Added ``macros.unpackVarargs``.
+
+- Added support for asynchronous programming for the JavaScript backend using
+  the `asyncjs` module.
+
+- Added true color support for some terminals. Example:
+```nim
+import colors, terminal
+
+const Nim = "Efficient and expressive programming."
+
+var
+  fg = colYellow
+  bg = colBlue
+  int = 1.0
+
+enableTrueColors()
+
+for i in 1..15:
+  styledEcho bgColor, bg, fgColor, fg, Nim, resetStyle
+  int -= 0.01
+  fg = intensity(fg, int)
+
+setForegroundColor colRed
+setBackgroundColor colGreen
+styledEcho "Red on Green.", resetStyle
+```
+
+### Library changes
+
+- ``echo`` now works with strings that contain ``\0`` (the binary zero is not
+  shown) and ``nil`` strings are equal to empty strings.
+
+- JSON: Deprecated `getBVal`, `getFNum`, and `getNum` in favour of
+  `getBool`, `getFloat`, `getBiggestInt`. A new `getInt` procedure was also
+  added.
+
+- ``rationals.toRational`` now uses an algorithm based on continued fractions.
+  This means its results are more precise and it can't run into an infinite loop
+  anymore.
+
+- ``os.getEnv`` now takes an optional ``default`` parameter that tells ``getEnv``
+  what to return if the environment variable does not exist.
+
+- The ``random`` procs in ``random.nim`` have all been deprecated. Instead use
+  the new ``rand`` procs. The module now exports the state of the random
+  number generator as type ``Rand`` so multiple threads can easily use their
+  own random number generators that do not require locking. For more information
+  about this rename see issue [#6934](https://github.com/nim-lang/Nim/issues/6934)
+
+- ``writeStackTrace`` is now proclaimed to have no IO effect (even though it does)
+  so that it is more useful for debugging purposes.
+
+- ``db_mysql`` module: ``DbConn`` is now a ``distinct`` type that doesn't expose the
+  details of the underlying ``PMySQL`` type.
+
+- ``parseopt2`` is now deprecated, use ``parseopt`` instead.
+
+### Language additions
+
+- It is now possible to forward declare object types so that mutually
+  recursive types can be created across module boundaries. See
+  [package level objects](https://nim-lang.org/docs/manual.html#package-level-objects)
+  for more information.
+
+- Added support for casting between integers of same bitsize in VM (compile time and nimscript).
+  This allows to, among other things, reinterpret signed integers as unsigned.
+
+- Custom pragmas are now supported using pragma ``pragma``, please see language
+  manual for details.
+
+- Standard library modules can now also be imported via the ``std`` pseudo-directory.
+  This is useful in order to distinguish between standard library and nimble package
+  imports:
+
+  ```nim
+  import std / [strutils, os, osproc]
+  import someNimblePackage / [strutils, os]
+  ```
+
+### Language changes
+
+- The **unary** ``<`` is now deprecated, for ``.. <`` use ``..<`` for other usages
+  use the ``pred`` proc.
+
+- Bodies of ``for`` loops now get their own scope:
+
+```nim
+# now compiles:
+for i in 0..4:
+  let i = i + 1
+  echo i
+```
+
+- To make Nim even more robust the system iterators ``..`` and ``countup``
+  now only accept a single generic type ``T``. This means the following code
+  doesn't die with an "out of range" error anymore:
+
+```nim
+var b = 5.Natural
+var a = -5
+for i in a..b:
+  echo i
+```
+
+- ``atomic`` and ``generic`` are no longer keywords in Nim. ``generic`` used to be
+  an alias for ``concept``, ``atomic`` was not used for anything.
+
+- The memory manager now uses a variant of the TLSF algorithm that has much
+  better memory fragmentation behaviour. According
+  to [http://www.gii.upv.es/tlsf/](http://www.gii.upv.es/tlsf/) the maximum
+  fragmentation measured is lower than 25%. As a nice bonus ``alloc`` and
+  ``dealloc`` became O(1) operations.
+
+- The compiler is now more consistent in its treatment of ambiguous symbols:
+  Types that shadow procs and vice versa are marked as ambiguous (bug #6693).
+
+- codegenDecl pragma now works for the JavaScript backend. It returns an empty
+  string for function return type placeholders.
+
+- Extra semantic checks for procs with noreturn pragma: return type is not allowed,
+  statements after call to noreturn procs are no longer allowed.
+
+- Noreturn proc calls and raising exceptions branches are now skipped during common type
+  deduction in ``if`` and ``case`` expressions. The following code snippets now compile:
+  ```nim
+  import strutils
+  let str = "Y"
+  let a = case str:
+    of "Y": true
+    of "N": false
+    else: raise newException(ValueError, "Invalid boolean")
+  let b = case str:
+    of nil, "": raise newException(ValueError, "Invalid boolean")
+    elif str.startsWith("Y"): true
+    elif str.startsWith("N"): false
+    else: false
+  let c = if str == "Y": true
+    elif str == "N": false
+    else:
+      echo "invalid bool"
+      quit("this is the end")
+  ```
+
+- Pragmas now support call syntax, for example: ``{.exportc"myname".}`` and
+  ``{.exportc("myname").}``
+
+- The ``deprecated`` pragma now supports a user-definable warning message for procs.
+
+  ```nim
+  proc bar {.deprecated: "use foo instead".} =
+    return
+
+  bar()
+  ```
+
+### Tool changes
+
+- The ``nim doc`` command is now an alias for ``nim doc2``, the second version of
+  the documentation generator. The old version 1 can still be accessed
+  via the new ``nim doc0`` command.
+
+- Nim's ``rst2html`` command now supports the testing of code snippets via an RST
+  extension that we called ``:test:``::
+
+  ```rst
+  .. code-block:: nim
+      :test:
+    # shows how the 'if' statement works
+    if true: echo "yes"
+  ```
+
+### Compiler changes
+
+### Bugfixes
+
+- Fixed "ReraiseError when using try/except within finally block"
+  ([#5871](https://github.com/nim-lang/Nim/issues/5871))
+- Fixed "Range type inference leads to counter-intuitive behvaiour"
+  ([#5854](https://github.com/nim-lang/Nim/issues/5854))
+- Fixed "JSON % operator can fail in extern procs with dynamic types"
+  ([#6385](https://github.com/nim-lang/Nim/issues/6385))
+- Fixed ""intVal is not accessible" in VM"
+  ([#6083](https://github.com/nim-lang/Nim/issues/6083))
+- Fixed "Add excl for OrderedSet"
+  ([#2467](https://github.com/nim-lang/Nim/issues/2467))
+- Fixed "newSeqOfCap actually doesn't reserve memory"
+  ([#6403](https://github.com/nim-lang/Nim/issues/6403))
+- Fixed "[Regression] Nim segfaults"
+  ([#6435](https://github.com/nim-lang/Nim/issues/6435))
+- Fixed "Seq assignment is slower than expected"
+  ([#6433](https://github.com/nim-lang/Nim/issues/6433))
+- Fixed "json module issues with empty dicts and lists"
+  ([#6438](https://github.com/nim-lang/Nim/issues/6438))
+- Fixed "mingw installed via finish.exe fails to link if Nim located in path with whitespace"
+  ([#6452](https://github.com/nim-lang/Nim/issues/6452))
+- Fixed "unittest.check does not perform short-circuit evaluation"
+  ([#5784](https://github.com/nim-lang/Nim/issues/5784))
+- Fixed "Error while concatenating an array of chars."
+  ([#5861](https://github.com/nim-lang/Nim/issues/5861))
+- Fixed "range initialization: [ProveInit] hint: Cannot prove that"
+  ([#6474](https://github.com/nim-lang/Nim/issues/6474))
+- Fixed "scanf can call procs with side-effects multiple times"
+  ([#6487](https://github.com/nim-lang/Nim/issues/6487))
+- Fixed "gcsafe detection problem"
+  ([#5620](https://github.com/nim-lang/Nim/issues/5620))
+- Fixed "C++ codegen: `mitems` generates invalid code."
+  ([#4910](https://github.com/nim-lang/Nim/issues/4910))
+- Fixed "strange runtime behavior on macOS"
+  ([#6496](https://github.com/nim-lang/Nim/issues/6496))
+- Fixed "stdtmpl: invalid indentation after a line ending in question mark"
+  ([#5070](https://github.com/nim-lang/Nim/issues/5070))
+- Fixed "Windows: NAN troubles on c backend"
+  ([#6511](https://github.com/nim-lang/Nim/issues/6511))
+- Fixed "lib/nim/system/cellsets.nim(33, 31) Error: type mismatch while attempting to compile for 16bit CPUs"
+  ([#3558](https://github.com/nim-lang/Nim/issues/3558))
+- Fixed "Can't compile dynlib with ``-d:useNimRtl`` and ``--threads:on``"
+  ([#5143](https://github.com/nim-lang/Nim/issues/5143))
+- Fixed "var s = @[0,1,2,...] can generate thousand of single assignments in C code"
+  ([#5007](https://github.com/nim-lang/Nim/issues/5007))
+- Fixed "`echo` discards everything after a null character"
+  ([#1137](https://github.com/nim-lang/Nim/issues/1137))
+- Fixed "Turn off reExtended by default"
+  ([#5627](https://github.com/nim-lang/Nim/issues/5627))
+- Fixed "Bad Links in docs/backends.html"
+  ([#5914](https://github.com/nim-lang/Nim/issues/5914))
+- Fixed "Index out of bounds error in db_postgres when executing non parameter-substituted queries containing "?""
+  ([#6571](https://github.com/nim-lang/Nim/issues/6571))
+- Fixed "Please add pipe2 support to posix stdlib"
+  ([#6553](https://github.com/nim-lang/Nim/issues/6553))
+- Fixed "Return semantics vary depending on return style"
+  ([#6422](https://github.com/nim-lang/Nim/issues/6422))
+- Fixed "parsecsv.open reports SIGSEGV when calling 'open' on missing file"
+  ([#6148](https://github.com/nim-lang/Nim/issues/6148))
+- Fixed "VCC: Nim generates non-compilable code for system.nim"
+  ([#6606](https://github.com/nim-lang/Nim/issues/6606))
+- Fixed "Generic subtype matches worse than a generic"
+  ([#6526](https://github.com/nim-lang/Nim/issues/6526))
+- Fixed "formatFloat inconsistent scientific notation"
+  ([#6589](https://github.com/nim-lang/Nim/issues/6589))
+- Fixed "Generated c code calls function twice"
+  ([#6292](https://github.com/nim-lang/Nim/issues/6292))
+- Fixed "Range type inference leads to counter-intuitive behvaiour"
+  ([#5854](https://github.com/nim-lang/Nim/issues/5854))
+- Fixed "New backward indexing is too limited"
+  ([#6631](https://github.com/nim-lang/Nim/issues/6631))
+- Fixed "Table usage in a macro (SIGSEGV: Illegal storage access.)"
+  ([#1860](https://github.com/nim-lang/Nim/issues/1860))
+- Fixed "Incorrect deprecation error"
+  ([#6634](https://github.com/nim-lang/Nim/issues/6634))
+- Fixed "Wrong indices in arrays not starting with 0"
+  ([#6675](https://github.com/nim-lang/Nim/issues/6675))
+- Fixed "if expressions"
+  ([#6609](https://github.com/nim-lang/Nim/issues/6609))
+- Fixed "BackwardsIndex: converter + `[]` + unrelated type[^1]: lib/system.nim(3536, 3) Error"
+  ([#6692](https://github.com/nim-lang/Nim/issues/6692))
+- Fixed "BackwardsIndex: converter + `[]` + unrelated type[^1]: lib/system.nim(3536, 3) Error"
+  ([#6692](https://github.com/nim-lang/Nim/issues/6692))
+- Fixed "js backend 0.17.3: array bounds check for non zero based arrays is buggy"
+  ([#6532](https://github.com/nim-lang/Nim/issues/6532))
+- Fixed "HttpClient's new API doesn't work through a proxy for https URLs"
+  ([#6685](https://github.com/nim-lang/Nim/issues/6685))
+- Fixed "isServing isn't declared and isn't compiling"
+  ([#6707](https://github.com/nim-lang/Nim/issues/6707))
+- Fixed "[Regression] value out of range"
+  ([#6710](https://github.com/nim-lang/Nim/issues/6710))
+
+- Fixed "Error when using `multisync` macro"
+  ([#6708](https://github.com/nim-lang/Nim/issues/6708))
+
+- Fixed "formatFloat inconsistent scientific notation"
+  ([#6589](https://github.com/nim-lang/Nim/issues/6589))
+- Fixed "Using : (constructor arguments) for passing values to functions with default arguments causes a compiler crash."
+  ([#6765](https://github.com/nim-lang/Nim/issues/6765))
+- Fixed "In-place object initialization leads to vcc incompatible code"
+  ([#6757](https://github.com/nim-lang/Nim/issues/6757))
+- Fixed "Improve parseCookies doc"
+  ([#5721](https://github.com/nim-lang/Nim/issues/5721))
+- Fixed "Parser regression with nested do notation inside conditional"
+  ([#6166](https://github.com/nim-lang/Nim/issues/6166))
+- Fixed "Request for better error message"
+  ([#6776](https://github.com/nim-lang/Nim/issues/6776))
+- Fixed "Testament tester does not execute test with `exitcode` only"
+  ([#6775](https://github.com/nim-lang/Nim/issues/6775))
+- Fixed "JS integer division off by one"
+  ([#6753](https://github.com/nim-lang/Nim/issues/6753))
+- Fixed "Regression: cannot prove not nil"
+  ([#5781](https://github.com/nim-lang/Nim/issues/5781))
+- Fixed "SIGSEGV: Illegal storage access. (Attempt to read from nil?) in generic proc"
+  ([#6073](https://github.com/nim-lang/Nim/issues/6073))
+- Fixed "Request for better error message"
+  ([#6776](https://github.com/nim-lang/Nim/issues/6776))
+- Fixed "Nim #head: sorting via reference hangs compiler"
+  ([#6724](https://github.com/nim-lang/Nim/issues/6724))
+- Fixed "Cannot cast pointer to char in cpp"
+  ([#5979](https://github.com/nim-lang/Nim/issues/5979))
+- Fixed "asynchttpserver replies with several errors on single request"
+  ([#6386](https://github.com/nim-lang/Nim/issues/6386))
+- Fixed "object variants superclass trigger bad codegen"
+  ([#5521](https://github.com/nim-lang/Nim/issues/5521))
+- Fixed "JS integer division off by one"
+  ([#6753](https://github.com/nim-lang/Nim/issues/6753))
+- Fixed "js backend compiler crash with tables indexed by certain types"
+  ([#6568](https://github.com/nim-lang/Nim/issues/6568))
+- Fixed "Jsgen bug with is"
+  ([#6445](https://github.com/nim-lang/Nim/issues/6445))
+- Fixed "Subrange definition with ..<"
+  ([#6788](https://github.com/nim-lang/Nim/issues/6788))
+- Fixed "fields not initialized: array with enum index type as object field."
+  ([#6682](https://github.com/nim-lang/Nim/issues/6682))
+- Fixed "Can not delete data in table when table's data type is kind of "not nil""
+  ([#6555](https://github.com/nim-lang/Nim/issues/6555))
+- Fixed "tables.nim: Cannot prove that 'n' is initialized"
+  ([#6121](https://github.com/nim-lang/Nim/issues/6121))
+- Fixed "issues with 'not nil' applied to a closure proc"
+  ([#6489](https://github.com/nim-lang/Nim/issues/6489))
+- Fixed "`not nil` not working in some cases"
+  ([#4686](https://github.com/nim-lang/Nim/issues/4686))
+- Fixed "Cannot prove '@[v]' is not nil"
+  ([#3993](https://github.com/nim-lang/Nim/issues/3993))
+
+- Fixed "Feature: support TCP_NODELAY in net.sockets"
+  ([#6795](https://github.com/nim-lang/Nim/issues/6795))
+- Fixed "Code that makes the compiler throw an error message and then hangs"
+  ([#6820](https://github.com/nim-lang/Nim/issues/6820))
+- Fixed "Code that makes the compiler throw an error message and then hangs"
+  ([#6820](https://github.com/nim-lang/Nim/issues/6820))
+- Fixed "Inconsistent behavior with sequence and string slicing"
+  ([#6223](https://github.com/nim-lang/Nim/issues/6223))
+- Fixed "Wrong behavior of "split" (proc and iterator)"
+  ([#4377](https://github.com/nim-lang/Nim/issues/4377))
+- Fixed "[Documentation] Invalid module name: [foo, bar]"
+  ([#6831](https://github.com/nim-lang/Nim/issues/6831))
+- Fixed "The destructor is not called for temporary objects"
+  ([#4214](https://github.com/nim-lang/Nim/issues/4214))
+- Fixed "Destructors does not work with implicit items iterator in for loop"
+  ([#985](https://github.com/nim-lang/Nim/issues/985))
+- Fixed "Error in template when using the type of the parameter inside it"
+  ([#6756](https://github.com/nim-lang/Nim/issues/6756))
+- Fixed "should json.to() respect parent attributes?"
+  ([#5856](https://github.com/nim-lang/Nim/issues/5856))
+- Fixed "json 'to' macro can not marshalize into tuples"
+  ([#6095](https://github.com/nim-lang/Nim/issues/6095))
+- Fixed "json.to fails with seq[T]"
+  ([#6604](https://github.com/nim-lang/Nim/issues/6604))
+- Fixed "json.to() is not worth using compared to marshal.to[T]"
+  ([#5848](https://github.com/nim-lang/Nim/issues/5848))
+- Fixed "Memory not being released in time, running out of memory"
+  ([#6031](https://github.com/nim-lang/Nim/issues/6031))
+- Fixed "[Regression] Bad C codegen for generic code"
+  ([#6889](https://github.com/nim-lang/Nim/issues/6889))
+- Fixed "rollingFileLogger deletes file on every start."
+  ([#6264](https://github.com/nim-lang/Nim/issues/6264))
+- Fixed "Remove/deprecate securehash module."
+  ([#6033](https://github.com/nim-lang/Nim/issues/6033))
+- Fixed "[bug or not] object construction for seq[T] failed without space after colon"
+  ([#5999](https://github.com/nim-lang/Nim/issues/5999))
+- Fixed "issues with the random module"
+  ([#4726](https://github.com/nim-lang/Nim/issues/4726))
+- Fixed "Reassigning local var to seq of objects results in nil element in Object's seq field"
+  ([#668](https://github.com/nim-lang/Nim/issues/668))
+- Fixed "Compilation error with "newseq[string]""
+  ([#6726](https://github.com/nim-lang/Nim/issues/6726))
+- Fixed "await inside array/dict literal produces invalid code - Part 2"
+  ([#6626](https://github.com/nim-lang/Nim/issues/6626))
+- Fixed "terminal.eraseline() gives OverflowError on Windows"
+  ([#6931](https://github.com/nim-lang/Nim/issues/6931))
+- Fixed "[Regression] `sequtils.any` conflicts with `system.any`"
+  ([#6932](https://github.com/nim-lang/Nim/issues/6932))
+- Fixed "C++ codegen: `mitems` generates invalid code."
+  ([#4910](https://github.com/nim-lang/Nim/issues/4910))
+- Fixed "seq.mitems produces invalid cpp codegen"
+  ([#6892](https://github.com/nim-lang/Nim/issues/6892))
+- Fixed "Concepts regression"
+  ([#6108](https://github.com/nim-lang/Nim/issues/6108))
+- Fixed "Generic iterable concept with array crashes compiler"
+  ([#6277](https://github.com/nim-lang/Nim/issues/6277))
+- Fixed "C code generation "‘a’ is a pointer; did you mean to use ‘->’?""
+  ([#6462](https://github.com/nim-lang/Nim/issues/6462))
+- Fixed "`--NimblePath` fails if a `-` in path which is not followed by a number"
+  ([#6949](https://github.com/nim-lang/Nim/issues/6949))
+- Fixed ""not registered in the selector" in asyncfile.close() for something that clearly was registered"
+  ([#6906](https://github.com/nim-lang/Nim/issues/6906))
+- Fixed "strange frexp behavior"
+  ([#6353](https://github.com/nim-lang/Nim/issues/6353))
+
+- Fixed "noreturn branches of case statements shouldn't contribute to type"
+  ([#6885](https://github.com/nim-lang/Nim/issues/6885))
+- Fixed "Type inference for 'if' statements changed"
+  ([#6980](https://github.com/nim-lang/Nim/issues/6980))
+- Fixed "newest asyncdispatch recursion"
+  ([#6100](https://github.com/nim-lang/Nim/issues/6100))
+- Fixed "Ambiguous identifier between set type and proc"
+  ([#6965](https://github.com/nim-lang/Nim/issues/6965))
+
+- Fixed "Inconsistent behavior with sequence and string slicing"
+  ([#6223](https://github.com/nim-lang/Nim/issues/6223))
+
+- Fixed "Unsupported OpenSSL library imported dynamically"
+  ([#5000](https://github.com/nim-lang/Nim/issues/5000))
+- Fixed "`nim check` segfaults"
+  ([#6972](https://github.com/nim-lang/Nim/issues/6972))
+- Fixed "GC deadlock"
+  ([#6988](https://github.com/nim-lang/Nim/issues/6988))
+- Fixed "Create a seq without memory initialization"
+  ([#6401](https://github.com/nim-lang/Nim/issues/6401))
+- Fixed "Fix bug for getch on Windows while using the arrow keys"
+  ([#6966](https://github.com/nim-lang/Nim/issues/6966))
+- Fixed "runnableExamples doesn't work in templates"
+  ([#7018](https://github.com/nim-lang/Nim/issues/7018))
+- Fixed "runnableExamples doesn't work with untyped statement blocks"
+  ([#7019](https://github.com/nim-lang/Nim/issues/7019))
+
+- Fixed "Critical bug in parseBiggestFloat"
+  ([#7060](https://github.com/nim-lang/Nim/issues/7060))
+- Fixed "[RFC] strformat.% should be gone"
+  ([#7078](https://github.com/nim-lang/Nim/issues/7078))
+- Fixed "compiler crash on simple macro"
+  ([#7093](https://github.com/nim-lang/Nim/issues/7093))
+- Fixed "Make newlines sane again"
+  ([#7089](https://github.com/nim-lang/Nim/issues/7089))
+- Fixed "JS - Unicode enum string representation issue"
+  ([#6741](https://github.com/nim-lang/Nim/issues/6741))
+- Fixed "Strange behaviour of 0.17.3 (working ok in 0.17.2)"
+  ([#6989](https://github.com/nim-lang/Nim/issues/6989))
+- Fixed "Strange behaviour of 0.17.3 (working ok in 0.17.2)"
+  ([#6989](https://github.com/nim-lang/Nim/issues/6989))
+- Fixed "Compiler crash: try expression with infix as"
+  ([#7116](https://github.com/nim-lang/Nim/issues/7116))
+- Fixed "nimsuggest crash"
+  ([#7140](https://github.com/nim-lang/Nim/issues/7140))
+- Fixed "[RFC] Reintroduce readChar"
+  ([#7072](https://github.com/nim-lang/Nim/issues/7072))
+- Fixed "Copyright line needs updating"
+  ([#7129](https://github.com/nim-lang/Nim/issues/7129))
+- Fixed "-0.0 doesn't result in negative zero in VM"
+  ([#7079](https://github.com/nim-lang/Nim/issues/7079))
+- Fixed "Windows large filesize"
+  ([#7121](https://github.com/nim-lang/Nim/issues/7121))
+- Fixed "Securehash is not parsimonious with MD5 and other hash modules"
+  ([#6961](https://github.com/nim-lang/Nim/issues/6961))
+- Fixed "os.findExe() shouldn't look in current directory on posix, unless exe has a /"
+  ([#6939](https://github.com/nim-lang/Nim/issues/6939))
+- Fixed "`compiles(...)` with `fatal` pragma causes compiler to exit early"
+  ([#7080](https://github.com/nim-lang/Nim/issues/7080))
+- Fixed "NPE when compile macro that returns concrete value"
+  ([#5450](https://github.com/nim-lang/Nim/issues/5450))
+- Fixed "Using a variable of type `int | float` causes internal compiler error"
+  ([#6946](https://github.com/nim-lang/Nim/issues/6946))
+- Fixed "Unsigned integers could not be used as array indexes."
+  ([#7153](https://github.com/nim-lang/Nim/issues/7153))
+- Fixed "countdown with uint causes underflow"
+  ([#4220](https://github.com/nim-lang/Nim/issues/4220))
+- Fixed "Inconsistent method call syntax"
+  ([#7200](https://github.com/nim-lang/Nim/issues/7200))
+- Fixed "Impossible to create an empty const array"
+  ([#6853](https://github.com/nim-lang/Nim/issues/6853))
+- Fixed "Strange UINT handling"
+  ([#3985](https://github.com/nim-lang/Nim/issues/3985))
+- Fixed "Bad codegen when passing arg that is part of return value destination"
+  ([#6960](https://github.com/nim-lang/Nim/issues/6960))
+- Fixed "No info about gcsafety in error message when global var is accessed in async proc"
+  ([#6186](https://github.com/nim-lang/Nim/issues/6186))
+- Fixed "Resolving package vs. local import ambiguities"
+  ([#2819](https://github.com/nim-lang/Nim/issues/2819))
+- Fixed "Internal error with type() operator"
+  ([#3711](https://github.com/nim-lang/Nim/issues/3711))
+- Fixed "newAsyncSocket should raise an OS error plus other inconsistencies"
+  ([#4995](https://github.com/nim-lang/Nim/issues/4995))
+- Fixed "mapLiterals fails with negative values"
+  ([#7215](https://github.com/nim-lang/Nim/issues/7215))
+- Fixed "fmWrite doesn't truncate file with openAsync, unlike open()"
+  ([#5531](https://github.com/nim-lang/Nim/issues/5531))
+- Fixed "Move libsvm to an external nimble module"
+  ([#5786](https://github.com/nim-lang/Nim/issues/5786))
+- Fixed "Prevent acceptAddr gotcha with newSocket"
+  ([#7227](https://github.com/nim-lang/Nim/issues/7227))
+- Fixed "strtabs.getOrDefault is inconsistent with tables.getOrDefault"
+  ([#4265](https://github.com/nim-lang/Nim/issues/4265))
+
+- Fixed "Code falling through into exception handler when no exception thrown."
+  ([#7232](https://github.com/nim-lang/Nim/issues/7232))
+- Fixed "the new generic inference rules are broken"
+  ([#7247](https://github.com/nim-lang/Nim/issues/7247))
+- Fixed "Odd `..<` regression"
+  ([#6992](https://github.com/nim-lang/Nim/issues/6992))
+- Fixed "Different proc type inferred from default parameter"
+  ([#4659](https://github.com/nim-lang/Nim/issues/4659))
+- Fixed "Different proc type inferred from default parameter"
+  ([#4659](https://github.com/nim-lang/Nim/issues/4659))
+- Fixed "Testament sometimes ignores test failures"
+  ([#7236](https://github.com/nim-lang/Nim/issues/7236))
+- Fixed "New Allocator Fails On >=4GB Requests"
+  ([#7120](https://github.com/nim-lang/Nim/issues/7120))
+- Fixed "User pragmas hide effect specifications from sempass2"
+  ([#7216](https://github.com/nim-lang/Nim/issues/7216))
+- Fixed "C++: SIGABRT instead of IndexError for out-of-bounds"
+  ([#6512](https://github.com/nim-lang/Nim/issues/6512))
+- Fixed "An uncaught exception in cpp mode doesn't show the exception name/msg"
+  ([#6431](https://github.com/nim-lang/Nim/issues/6431))

ci/build.sh

@@ -6,7 +6,7 @@ cd csources
 sh build.sh
 cd ..
 # Add Nim to the PATH
-export PATH=$(pwd)/bin:$PATH
+export PATH=$(pwd)/bin${PATH:+:$PATH}
 # Bootstrap.
 nim -v
 nim c koch

ci/deps.sh

@@ -7,7 +7,7 @@ apt-get install -y -qq build-essential git libcurl4-openssl-dev libsdl1.2-dev li
 
 gcc -v
 
-export PATH=$(pwd)/bin:$PATH
+export PATH=$(pwd)/bin${PATH:+:$PATH}
 
 # Nimble deps
 nim e install_nimble.nims

compiler/aliases.nim

@@ -49,7 +49,7 @@ proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult =
     if a.sons[0] != nil:
       result = isPartOfAux(a.sons[0].skipTypes(skipPtrs), b, marker)
     if result == arNo: result = isPartOfAux(a.n, b, marker)
-  of tyGenericInst, tyDistinct, tyAlias:
+  of tyGenericInst, tyDistinct, tyAlias, tySink:
     result = isPartOfAux(lastSon(a), b, marker)
   of tyArray, tySet, tyTuple:
     for i in countup(0, sonsLen(a) - 1):
@@ -95,7 +95,7 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
   if a.kind == b.kind:
     case a.kind
     of nkSym:
-      const varKinds = {skVar, skTemp, skProc}
+      const varKinds = {skVar, skTemp, skProc, skFunc}
       # same symbol: aliasing:
       if a.sym.id == b.sym.id: result = arYes
       elif a.sym.kind in varKinds or b.sym.kind in varKinds:
@@ -179,5 +179,11 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
           result = isPartOf(a[0], b)
           if result == arNo: result = arMaybe
       else: discard
+    of nkObjConstr:
+      result = arNo
+      for i in 1..<b.len:
+        let res = isPartOf(a, b[i][1])
+        if res != arNo:
+          result = res
+          if res == arYes: break
     else: discard
-

compiler/ast.nim

@@ -10,7 +10,7 @@
 # abstract syntax tree + symbol table
 
 import
-  msgs, hashes, nversion, options, strutils, securehash, ropes, idents,
+  msgs, hashes, nversion, options, strutils, std / sha1, ropes, idents,
   intsets, idgen
 
 type
@@ -62,8 +62,8 @@ type
     nkTripleStrLit,       # a triple string literal """
     nkNilLit,             # the nil literal
                           # end of atoms
-    nkMetaNode_Obsolete,  # difficult to explain; represents itself
-                          # (used for macros)
+    nkComesFrom,          # "comes from" template/macro information for
+                          # better stack trace generation
     nkDotCall,            # used to temporarily flag a nkCall node;
                           # this is used
                           # for transforming ``s.len`` to ``len(s)``
@@ -221,6 +221,8 @@ type
     nkGotoState,          # used for the state machine (for iterators)
     nkState,              # give a label to a code section (for iterators)
     nkBreakState,         # special break statement for easier code generation
+    nkFuncDef             # a func
+
   TNodeKinds* = set[TNodeKind]
 
 type
@@ -303,6 +305,7 @@ const
   sfEscapes* = sfProcvar              # param escapes
   sfBase* = sfDiscriminant
   sfIsSelf* = sfOverriden             # param is 'self'
+  sfCustomPragma* = sfRegister        # symbol is custom pragma template
 
 const
   # getting ready for the future expr/stmt merge
@@ -352,7 +355,7 @@ type
     tyInt, tyInt8, tyInt16, tyInt32, tyInt64, # signed integers
     tyFloat, tyFloat32, tyFloat64, tyFloat128,
     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64,
-    tyUnused0, tyUnused1, tyUnused2,
+    tyOptAsRef, tySink, tyLent,
     tyVarargs,
     tyUnused,
     tyProxy # used as errornous type (for idetools)
@@ -402,14 +405,8 @@ type
       # instantiation and prior to this it has the potential to
       # be any type.
 
-    tyFieldAccessor
-      # Expressions such as Type.field (valid in contexts such
-      # as the `is` operator and magics like `high` and `low`).
-      # Could be lifted to a single argument proc returning the
-      # field value.
-      # sons[0]: type of containing object or tuple
-      # sons[1]: field type
-      # .n: nkDotExpr storing the field name
+    tyOpt
+      # Builtin optional type
 
     tyVoid
       # now different from tyEmpty, hurray!
@@ -534,6 +531,7 @@ type
     skConst,              # a constant
     skResult,             # special 'result' variable
     skProc,               # a proc
+    skFunc,               # a func
     skMethod,             # a method
     skIterator,           # an iterator
     skConverter,          # a type converter
@@ -552,7 +550,7 @@ type
   TSymKinds* = set[TSymKind]
 
 const
-  routineKinds* = {skProc, skMethod, skIterator,
+  routineKinds* = {skProc, skFunc, skMethod, skIterator,
                    skConverter, skMacro, skTemplate}
   tfIncompleteStruct* = tfVarargs
   tfUncheckedArray* = tfVarargs
@@ -621,7 +619,7 @@ type
     mSwap, mIsNil, mArrToSeq, mCopyStr, mCopyStrLast,
     mNewString, mNewStringOfCap, mParseBiggestFloat,
     mReset,
-    mArray, mOpenArray, mRange, mSet, mSeq, mVarargs,
+    mArray, mOpenArray, mRange, mSet, mSeq, mOpt, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
     mOrdinal,
     mInt, mInt8, mInt16, mInt32, mInt64,
@@ -631,7 +629,7 @@ type
     mPointer, mEmptySet, mIntSetBaseType, mNil, mExpr, mStmt, mTypeDesc,
     mVoidType, mPNimrodNode, mShared, mGuarded, mLock, mSpawn, mDeepCopy,
     mIsMainModule, mCompileDate, mCompileTime, mProcCall,
-    mCpuEndian, mHostOS, mHostCPU, mAppType,
+    mCpuEndian, mHostOS, mHostCPU, mBuildOS, mBuildCPU, mAppType,
     mNaN, mInf, mNegInf,
     mCompileOption, mCompileOptionArg,
     mNLen, mNChild, mNSetChild, mNAdd, mNAddMultiple, mNDel, mNKind,
@@ -642,7 +640,8 @@ type
     mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl,
     mNHint, mNWarning, mNError,
     mInstantiationInfo, mGetTypeInfo, mNGenSym,
-    mNimvm, mIntDefine, mStrDefine
+    mNimvm, mIntDefine, mStrDefine, mRunnableExamples,
+    mException, mBuiltinType
 
 # things that we can evaluate safely at compile time, even if not asked for it:
 const
@@ -747,15 +746,18 @@ type
     OnUnknown,                # location is unknown (stack, heap or static)
     OnStatic,                 # in a static section
     OnStack,                  # location is on hardware stack
+    OnStackShadowDup,         # location is on the stack but also replicated
+                              # on the shadow stack
     OnHeap                    # location is on heap or global
                               # (reference counting needed)
   TLocFlags* = set[TLocFlag]
   TLoc* = object
     k*: TLocKind              # kind of location
-    s*: TStorageLoc
+    storage*: TStorageLoc
     flags*: TLocFlags         # location's flags
-    t*: PType                 # type of location
+    lode*: PNode              # Node where the location came from; can be faked
     r*: Rope                  # rope value of location (code generators)
+    dup*: Rope                # duplicated location for precise stack scans
 
   # ---------------- end of backend information ------------------------------
 
@@ -874,7 +876,8 @@ type
                               # mean that there is no destructor.
                               # see instantiateDestructor in semdestruct.nim
     deepCopy*: PSym           # overriden 'deepCopy' operation
-    assignment*: PSym         # overriden '=' operator
+    assignment*: PSym         # overriden '=' operation
+    sink*: PSym               # overriden '=sink' operation
     methods*: seq[(int,PSym)] # attached methods
     size*: BiggestInt         # the size of the type in bytes
                               # -1 means that the size is unkwown
@@ -931,20 +934,20 @@ type
 # the poor naming choices in the standard library.
 
 const
-  OverloadableSyms* = {skProc, skMethod, skIterator,
+  OverloadableSyms* = {skProc, skFunc, skMethod, skIterator,
     skConverter, skModule, skTemplate, skMacro}
 
   GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody,
     tyGenericParam}
 
   StructuralEquivTypes*: TTypeKinds = {tyNil, tyTuple, tyArray,
-    tySet, tyRange, tyPtr, tyRef, tyVar, tySequence, tyProc, tyOpenArray,
+    tySet, tyRange, tyPtr, tyRef, tyVar, tyLent, tySequence, tyProc, tyOpenArray,
     tyVarargs}
 
   ConcreteTypes*: TTypeKinds = { # types of the expr that may occur in::
                                  # var x = expr
     tyBool, tyChar, tyEnum, tyArray, tyObject,
-    tySet, tyTuple, tyRange, tyPtr, tyRef, tyVar, tySequence, tyProc,
+    tySet, tyTuple, tyRange, tyPtr, tyRef, tyVar, tyLent, tySequence, tyProc,
     tyPointer,
     tyOpenArray, tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128,
     tyUInt..tyUInt64}
@@ -954,7 +957,7 @@ const
                                     tyTuple, tySequence}
   NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, tySequence,
     tyProc, tyString, tyError}
-  ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType,
+  ExportableSymKinds* = {skVar, skConst, skProc, skFunc, skMethod, skType,
     skIterator,
     skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
@@ -976,16 +979,17 @@ const
   nkIdentKinds* = {nkIdent, nkSym, nkAccQuoted, nkOpenSymChoice,
                    nkClosedSymChoice}
 
+  nkPragmaCallKinds* = {nkExprColonExpr, nkCall, nkCallStrLit}
   nkLiterals* = {nkCharLit..nkTripleStrLit}
   nkLambdaKinds* = {nkLambda, nkDo}
-  declarativeDefs* = {nkProcDef, nkMethodDef, nkIteratorDef, nkConverterDef}
+  declarativeDefs* = {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef}
   procDefs* = nkLambdaKinds + declarativeDefs
 
   nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice}
   nkStrKinds* = {nkStrLit..nkTripleStrLit}
 
   skLocalVars* = {skVar, skLet, skForVar, skParam, skResult}
-  skProcKinds* = {skProc, skTemplate, skMacro, skIterator,
+  skProcKinds* = {skProc, skFunc, skTemplate, skMacro, skIterator,
                   skMethod, skConverter}
 
 var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things
@@ -1006,6 +1010,12 @@ proc safeLen*(n: PNode): int {.inline.} =
   if n.kind in {nkNone..nkNilLit} or isNil(n.sons): result = 0
   else: result = len(n.sons)
 
+proc safeArrLen*(n: PNode): int {.inline.} =
+  ## works for array-like objects (strings passed as openArray in VM).
+  if n.kind in {nkStrLit..nkTripleStrLit}:result = len(n.strVal)
+  elif n.kind in {nkNone..nkFloat128Lit}: result = 0
+  else: result = len(n)
+
 proc add*(father, son: PNode) =
   assert son != nil
   if isNil(father.sons): father.sons = @[]
@@ -1013,16 +1023,11 @@ proc add*(father, son: PNode) =
 
 type Indexable = PNode | PType
 
-template `[]`*(n: Indexable, i: int): Indexable =
-  n.sons[i]
+template `[]`*(n: Indexable, i: int): Indexable = n.sons[i]
+template `[]=`*(n: Indexable, i: int; x: Indexable) = n.sons[i] = x
 
-template `-|`*(b, s: untyped): untyped =
-  (if b >= 0: b else: s.len + b)
-
-# son access operators with support for negative indices
-template `{}`*(n: Indexable, i: int): untyped = n[i -| n]
-template `{}=`*(n: Indexable, i: int, s: Indexable) =
-  n.sons[i -| n] = s
+template `[]`*(n: Indexable, i: BackwardsIndex): Indexable = n[n.len - i.int]
+template `[]=`*(n: Indexable, i: BackwardsIndex; x: Indexable) = n[n.len - i.int] = x
 
 when defined(useNodeIds):
   const nodeIdToDebug* = -1 # 299750 # 300761 #300863 # 300879
@@ -1032,9 +1037,9 @@ proc newNode*(kind: TNodeKind): PNode =
   new(result)
   result.kind = kind
   #result.info = UnknownLineInfo() inlined:
-  result.info.fileIndex = int32(- 1)
-  result.info.col = int16(- 1)
-  result.info.line = int16(- 1)
+  result.info.fileIndex = int32(-1)
+  result.info.col = int16(-1)
+  result.info.line = int16(-1)
   when defined(useNodeIds):
     result.id = gNodeId
     if result.id == nodeIdToDebug:
@@ -1044,6 +1049,8 @@ proc newNode*(kind: TNodeKind): PNode =
 
 proc newTree*(kind: TNodeKind; children: varargs[PNode]): PNode =
   result = newNode(kind)
+  if children.len > 0:
+    result.info = children[0].info
   result.sons = @children
 
 proc newIntNode*(kind: TNodeKind, intVal: BiggestInt): PNode =
@@ -1075,7 +1082,7 @@ proc newSym*(symKind: TSymKind, name: PIdent, owner: PSym,
   result.info = info
   result.options = gOptions
   result.owner = owner
-  result.offset = - 1
+  result.offset = -1
   result.id = getID()
   when debugIds:
     registerId(result)
@@ -1255,11 +1262,10 @@ proc newType*(kind: TTypeKind, owner: PSym): PType =
 
 proc mergeLoc(a: var TLoc, b: TLoc) =
   if a.k == low(a.k): a.k = b.k
-  if a.s == low(a.s): a.s = b.s
+  if a.storage == low(a.storage): a.storage = b.storage
   a.flags = a.flags + b.flags
-  if a.t == nil: a.t = b.t
+  if a.lode == nil: a.lode = b.lode
   if a.r == nil: a.r = b.r
-  #if a.a == 0: a.a = b.a
 
 proc newSons*(father: PNode, length: int) =
   if isNil(father.sons):
@@ -1288,6 +1294,7 @@ proc assignType*(dest, src: PType) =
   dest.align = src.align
   dest.destructor = src.destructor
   dest.deepCopy = src.deepCopy
+  dest.sink = src.sink
   dest.assignment = src.assignment
   dest.lockLevel = src.lockLevel
   # this fixes 'type TLock = TSysLock':
@@ -1386,6 +1393,14 @@ proc skipTypes*(t: PType, kinds: TTypeKinds): PType =
   result = t
   while result.kind in kinds: result = lastSon(result)
 
+proc skipTypes*(t: PType, kinds: TTypeKinds; maxIters: int): PType =
+  result = t
+  var i = maxIters
+  while result.kind in kinds:
+    result = lastSon(result)
+    dec i
+    if i == 0: return nil
+
 proc skipTypesOrNil*(t: PType, kinds: TTypeKinds): PType =
   ## same as skipTypes but handles 'nil'
   result = t
@@ -1398,8 +1413,8 @@ proc isGCedMem*(t: PType): bool {.inline.} =
            t.kind == tyProc and t.callConv == ccClosure
 
 proc propagateToOwner*(owner, elem: PType) =
-  const HaveTheirOwnEmpty = {tySequence, tySet, tyPtr, tyRef, tyProc}
-  owner.flags = owner.flags + (elem.flags * {tfHasMeta})
+  const HaveTheirOwnEmpty = {tySequence, tyOpt, tySet, tyPtr, tyRef, tyProc}
+  owner.flags = owner.flags + (elem.flags * {tfHasMeta, tfTriggersCompileTime})
   if tfNotNil in elem.flags:
     if owner.kind in {tyGenericInst, tyGenericBody, tyGenericInvocation}:
       owner.flags.incl tfNotNil
@@ -1414,18 +1429,15 @@ proc propagateToOwner*(owner, elem: PType) =
     owner.flags.incl tfHasMeta
 
   if tfHasAsgn in elem.flags:
-    let o2 = elem.skipTypes({tyGenericInst, tyAlias})
+    let o2 = owner.skipTypes({tyGenericInst, tyAlias, tySink})
     if o2.kind in {tyTuple, tyObject, tyArray,
-                   tySequence, tySet, tyDistinct}:
+                   tySequence, tyOpt, tySet, tyDistinct}:
       o2.flags.incl tfHasAsgn
       owner.flags.incl tfHasAsgn
 
-  if tfTriggersCompileTime in elem.flags:
-    owner.flags.incl tfTriggersCompileTime
-
   if owner.kind notin {tyProc, tyGenericInst, tyGenericBody,
                        tyGenericInvocation, tyPtr}:
-    let elemB = elem.skipTypes({tyGenericInst, tyAlias})
+    let elemB = elem.skipTypes({tyGenericInst, tyAlias, tySink})
     if elemB.isGCedMem or tfHasGCedMem in elemB.flags:
       # for simplicity, we propagate this flag even to generics. We then
       # ensure this doesn't bite us in sempass2.
@@ -1436,6 +1448,10 @@ proc rawAddSon*(father, son: PType) =
   add(father.sons, son)
   if not son.isNil: propagateToOwner(father, son)
 
+proc rawAddSonNoPropagationOfTypeFlags*(father, son: PType) =
+  if isNil(father.sons): father.sons = @[]
+  add(father.sons, son)
+
 proc addSonNilAllowed*(father, son: PNode) =
   if isNil(father.sons): father.sons = @[]
   add(father.sons, son)
@@ -1598,10 +1614,10 @@ proc hasPattern*(s: PSym): bool {.inline.} =
   result = isRoutine(s) and s.ast.sons[patternPos].kind != nkEmpty
 
 iterator items*(n: PNode): PNode =
-  for i in 0.. <n.safeLen: yield n.sons[i]
+  for i in 0..<n.safeLen: yield n.sons[i]
 
 iterator pairs*(n: PNode): tuple[i: int, n: PNode] =
-  for i in 0.. <n.len: yield (i, n.sons[i])
+  for i in 0..<n.len: yield (i, n.sons[i])
 
 proc isAtom*(n: PNode): bool {.inline.} =
   result = n.kind >= nkNone and n.kind <= nkNilLit
@@ -1641,6 +1657,20 @@ proc toObject*(typ: PType): PType =
   if result.kind == tyRef:
     result = result.lastSon
 
+proc isException*(t: PType): bool =
+  # check if `y` is object type and it inherits from Exception
+  assert(t != nil)
+
+  if t.kind != tyObject:
+    return false
+
+  var base = t
+  while base != nil:
+    if base.sym.magic == mException:
+      return true
+    base = base.lastSon
+  return false
+
 proc findUnresolvedStatic*(n: PNode): PNode =
   if n.kind == nkSym and n.typ.kind == tyStatic and n.typ.n == nil:
     return n
@@ -1657,3 +1687,10 @@ when false:
     if n.isNil: return true
     for i in 0 ..< n.safeLen:
       if n[i].containsNil: return true
+
+template hasDestructor*(t: PType): bool = tfHasAsgn in t.flags
+template incompleteType*(t: PType): bool =
+  t.sym != nil and {sfForward, sfNoForward} * t.sym.flags == {sfForward}
+
+template typeCompleted*(s: PSym) =
+  incl s.flags, sfNoForward

compiler/canonicalizer.nim

@@ -102,7 +102,7 @@ proc hashTree(c: var MD5Context, n: PNode) =
   of nkStrLit..nkTripleStrLit:
     c &= n.strVal
   else:
-    for i in 0.. <n.len: hashTree(c, n.sons[i])
+    for i in 0..<n.len: hashTree(c, n.sons[i])
 
 proc hashType(c: var MD5Context, t: PType) =
   # modelled after 'typeToString'
@@ -130,7 +130,7 @@ proc hashType(c: var MD5Context, t: PType) =
     c.hashSym body.sym
     for i in countup(1, sonsLen(t) - 2):
       c.hashType t.sons[i]
-  of tyFromExpr, tyFieldAccessor:
+  of tyFromExpr:
     c.hashTree(t.n)
   of tyArray:
     c.hashTree(t.sons[0].n)
@@ -151,13 +151,13 @@ proc hashType(c: var MD5Context, t: PType) =
     c.hashType(t.sons[0])
   of tyProc:
     c &= (if tfIterator in t.flags: "iterator " else: "proc ")
-    for i in 0.. <t.len: c.hashType(t.sons[i])
+    for i in 0..<t.len: c.hashType(t.sons[i])
     md5Update(c, cast[cstring](addr(t.callConv)), 1)
 
     if tfNoSideEffect in t.flags: c &= ".noSideEffect"
     if tfThread in t.flags: c &= ".thread"
   else:
-    for i in 0.. <t.len: c.hashType(t.sons[i])
+    for i in 0..<t.len: c.hashType(t.sons[i])
   if tfNotNil in t.flags: c &= "not nil"
 
 proc canonConst(n: PNode): TUid =

compiler/ccgcalls.nim

@@ -11,7 +11,7 @@
 
 proc leftAppearsOnRightSide(le, ri: PNode): bool =
   if le != nil:
-    for i in 1 .. <ri.len:
+    for i in 1 ..< ri.len:
       let r = ri[i]
       if isPartOf(le, r) != arNo: return true
 
@@ -32,7 +32,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
         if d.k == locNone: getTemp(p, typ.sons[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
-          resetLoc(p, d)
+          discard "resetLoc(p, d)"
         add(pl, addrLoc(d))
         add(pl, ~");$n")
         line(p, cpsStmts, pl)
@@ -56,7 +56,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
         if d.k == locNone: getTemp(p, typ.sons[0], d)
         assert(d.t != nil)        # generate an assignment to d:
         var list: TLoc
-        initLoc(list, locCall, d.t, OnUnknown)
+        initLoc(list, locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
   else:
@@ -71,7 +71,7 @@ proc isInCurrentFrame(p: BProc, n: PNode): bool =
     if n.sym.kind in {skVar, skResult, skTemp, skLet} and p.prc != nil:
       result = p.prc.id == n.sym.owner.id
   of nkDotExpr, nkBracketExpr:
-    if skipTypes(n.sons[0].typ, abstractInst).kind notin {tyVar,tyPtr,tyRef}:
+    if skipTypes(n.sons[0].typ, abstractInst).kind notin {tyVar,tyLent,tyPtr,tyRef}:
       result = isInCurrentFrame(p, n.sons[0])
   of nkHiddenStdConv, nkHiddenSubConv, nkConv:
     result = isInCurrentFrame(p, n.sons[1])
@@ -228,7 +228,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
           getTemp(p, typ.sons[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
-          resetLoc(p, d)
+          discard "resetLoc(p, d)"
         add(pl, addrLoc(d))
         genCallPattern()
       else:
@@ -241,7 +241,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
       if d.k == locNone: getTemp(p, typ.sons[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
-      initLoc(list, locCall, d.t, OnUnknown)
+      initLoc(list, locCall, d.lode, OnUnknown)
       list.r = callPattern % [op.r, pl, pl.addComma, rawProc]
       genAssignment(p, d, list, {}) # no need for deep copying
   else:
@@ -331,7 +331,7 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   # skip the deref:
   var ri = ri[i]
   while ri.kind == nkObjDownConv: ri = ri[0]
-  let t = typ.sons[i].skipTypes({tyGenericInst, tyAlias})
+  let t = typ.sons[i].skipTypes({tyGenericInst, tyAlias, tySink})
   if t.kind == tyVar:
     let x = if ri.kind == nkHiddenAddr: ri[0] else: ri
     if x.typ.kind == tyPtr:
@@ -364,7 +364,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
     of '@':
       if j < ri.len:
         result.add genOtherArg(p, ri, j, typ)
-        for k in j+1 .. < ri.len:
+        for k in j+1 ..< ri.len:
           result.add(~", ")
           result.add genOtherArg(p, ri, k, typ)
       inc i
@@ -377,7 +377,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
           result.add(~"(")
           if 1 < ri.len:
             result.add genOtherArg(p, ri, 1, typ)
-          for k in j+1 .. < ri.len:
+          for k in j+1 ..< ri.len:
             result.add(~", ")
             result.add genOtherArg(p, ri, k, typ)
           result.add(~")")
@@ -437,7 +437,7 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
         if d.k == locNone: getTemp(p, typ.sons[0], d)
         assert(d.t != nil)        # generate an assignment to d:
         var list: TLoc
-        initLoc(list, locCall, d.t, OnUnknown)
+        initLoc(list, locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
     else:
@@ -519,7 +519,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
       if d.k == locNone: getTemp(p, typ.sons[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
-      initLoc(list, locCall, nil, OnUnknown)
+      initLoc(list, locCall, ri, OnUnknown)
       list.r = pl
       genAssignment(p, d, list, {}) # no need for deep copying
   else:
@@ -527,7 +527,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
     line(p, cpsStmts, pl)
 
 proc genCall(p: BProc, e: PNode, d: var TLoc) =
-  if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias}).callConv == ccClosure:
+  if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure:
     genClosureCall(p, nil, e, d)
   elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags:
     genInfixCall(p, nil, e, d)
@@ -538,7 +538,7 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) =
   postStmtActions(p)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
-  if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias}).callConv == ccClosure:
+  if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
   elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags:
     genInfixCall(p, le, ri, d)

compiler/ccgmerge.nim

@@ -96,7 +96,7 @@ proc writeIntSet(a: IntSet, s: var string) =
   s.add('}')
 
 proc genMergeInfo*(m: BModule): Rope =
-  if optSymbolFiles notin gGlobalOptions: return nil
+  if not compilationCachePresent: return nil
   var s = "/*\tNIM_merge_INFO:"
   s.add(tnl)
   s.add("typeCache:{")

compiler/ccgstmts.nim

@@ -20,9 +20,13 @@ proc registerGcRoot(p: BProc, v: PSym) =
       containsGarbageCollectedRef(v.loc.t):
     # we register a specialized marked proc here; this has the advantage
     # that it works out of the box for thread local storage then :-)
-    let prc = genTraverseProcForGlobal(p.module, v)
-    appcg(p.module, p.module.initProc.procSec(cpsInit),
-      "#nimRegisterGlobalMarker($1);$n", [prc])
+    let prc = genTraverseProcForGlobal(p.module, v, v.info)
+    if sfThread in v.flags:
+      appcg(p.module, p.module.initProc.procSec(cpsInit),
+        "#nimRegisterThreadLocalMarker($1);$n", [prc])
+    else:
+      appcg(p.module, p.module.initProc.procSec(cpsInit),
+        "#nimRegisterGlobalMarker($1);$n", [prc])
 
 proc isAssignedImmediately(n: PNode): bool {.inline.} =
   if n.kind == nkEmpty: return false
@@ -48,16 +52,17 @@ proc genVarTuple(p: BProc, n: PNode) =
   initLocExpr(p, n.sons[L-1], tup)
   var t = tup.t.skipTypes(abstractInst)
   for i in countup(0, L-3):
-    var v = n.sons[i].sym
+    let vn = n.sons[i]
+    let v = vn.sym
     if sfCompileTime in v.flags: continue
     if sfGlobal in v.flags:
-      assignGlobalVar(p, v)
+      assignGlobalVar(p, vn)
       genObjectInit(p, cpsInit, v.typ, v.loc, true)
       registerGcRoot(p, v)
     else:
-      assignLocalVar(p, v)
+      assignLocalVar(p, vn)
       initLocalVar(p, v, immediateAsgn=isAssignedImmediately(n[L-1]))
-    initLoc(field, locExpr, t.sons[i], tup.s)
+    initLoc(field, locExpr, vn, tup.storage)
     if t.kind == tyTuple:
       field.r = "$1.Field$2" % [rdLoc(tup), rope(i)]
     else:
@@ -140,9 +145,13 @@ template preserveBreakIdx(body: untyped): untyped =
   p.breakIdx = oldBreakIdx
 
 proc genState(p: BProc, n: PNode) =
-  internalAssert n.len == 1 and n.sons[0].kind == nkIntLit
-  let idx = n.sons[0].intVal
-  linefmt(p, cpsStmts, "STATE$1: ;$n", idx.rope)
+  internalAssert n.len == 1
+  let n0 = n[0]
+  if n0.kind == nkIntLit:
+    let idx = n.sons[0].intVal
+    linefmt(p, cpsStmts, "STATE$1: ;$n", idx.rope)
+  elif n0.kind == nkStrLit:
+    linefmt(p, cpsStmts, "$1: ;$n", n0.strVal.rope)
 
 proc genGotoState(p: BProc, n: PNode) =
   # we resist the temptation to translate it into duff's device as it later
@@ -155,8 +164,13 @@ proc genGotoState(p: BProc, n: PNode) =
   lineF(p, cpsStmts, "switch ($1) {$n", [rdLoc(a)])
   p.beforeRetNeeded = true
   lineF(p, cpsStmts, "case -1: goto BeforeRet_;$n", [])
-  for i in 0 .. lastOrd(n.sons[0].typ):
-    lineF(p, cpsStmts, "case $1: goto STATE$1;$n", [rope(i)])
+  var statesCounter = lastOrd(n.sons[0].typ)
+  if n.len >= 2 and n[1].kind == nkIntLit:
+    statesCounter = n[1].intVal
+  let prefix = if n.len == 3 and n[2].kind == nkStrLit: n[2].strVal.rope
+               else: rope"STATE"
+  for i in 0i64 .. statesCounter:
+    lineF(p, cpsStmts, "case $2: goto $1$2;$n", [prefix, rope(i)])
   lineF(p, cpsStmts, "}$n", [])
 
 proc genBreakState(p: BProc, n: PNode) =
@@ -171,7 +185,7 @@ proc genBreakState(p: BProc, n: PNode) =
     lineF(p, cpsStmts, "if ((((NI*) $1.ClE_0)[1]) < 0) break;$n", [rdLoc(a)])
   #  lineF(p, cpsStmts, "if (($1) < 0) break;$n", [rdLoc(a)])
 
-proc genVarPrototypeAux(m: BModule, sym: PSym)
+proc genVarPrototypeAux(m: BModule, n: PNode)
 
 proc genGotoVar(p: BProc; value: PNode) =
   if value.kind notin {nkCharLit..nkUInt64Lit}:
@@ -180,7 +194,8 @@ proc genGotoVar(p: BProc; value: PNode) =
     lineF(p, cpsStmts, "goto NIMSTATE_$#;$n", [value.intVal.rope])
 
 proc genSingleVar(p: BProc, a: PNode) =
-  let v = a.sons[0].sym
+  let vn = a.sons[0]
+  let v = vn.sym
   if sfCompileTime in v.flags: return
   if sfGoto in v.flags:
     # translate 'var state {.goto.} = X' into 'goto LX':
@@ -195,7 +210,7 @@ proc genSingleVar(p: BProc, a: PNode) =
     if sfPure in v.flags:
       # v.owner.kind != skModule:
       targetProc = p.module.preInitProc
-    assignGlobalVar(targetProc, v)
+    assignGlobalVar(targetProc, vn)
     # XXX: be careful here.
     # Global variables should not be zeromem-ed within loops
     # (see bug #20).
@@ -206,7 +221,7 @@ proc genSingleVar(p: BProc, a: PNode) =
     # Alternative construction using default constructor (which may zeromem):
     # if sfImportc notin v.flags: constructLoc(p.module.preInitProc, v.loc)
     if sfExportc in v.flags and p.module.g.generatedHeader != nil:
-      genVarPrototypeAux(p.module.g.generatedHeader, v)
+      genVarPrototypeAux(p.module.g.generatedHeader, vn)
     registerGcRoot(p, v)
   else:
     let value = a.sons[2]
@@ -217,14 +232,14 @@ proc genSingleVar(p: BProc, a: PNode) =
       # parameterless constructor followed by an assignment operator. So we
       # generate better code here:
       genLineDir(p, a)
-      let decl = localVarDecl(p, v)
+      let decl = localVarDecl(p, vn)
       var tmp: TLoc
       if value.kind in nkCallKinds and value[0].kind == nkSym and
            sfConstructor in value[0].sym.flags:
         var params: Rope
         let typ = skipTypes(value.sons[0].typ, abstractInst)
         assert(typ.kind == tyProc)
-        for i in 1.. <value.len:
+        for i in 1..<value.len:
           if params != nil: params.add(~", ")
           assert(sonsLen(typ) == sonsLen(typ.n))
           add(params, genOtherArg(p, value, i, typ))
@@ -233,7 +248,7 @@ proc genSingleVar(p: BProc, a: PNode) =
         initLocExprSingleUse(p, value, tmp)
         lineF(p, cpsStmts, "$# = $#;$n", [decl, tmp.rdLoc])
       return
-    assignLocalVar(p, v)
+    assignLocalVar(p, vn)
     initLocalVar(p, v, imm)
 
   if a.sons[2].kind != nkEmpty:
@@ -375,7 +390,7 @@ proc genReturnStmt(p: BProc, t: PNode) =
   lineF(p, cpsStmts, "goto BeforeRet_;$n", [])
 
 proc genGotoForCase(p: BProc; caseStmt: PNode) =
-  for i in 1 .. <caseStmt.len:
+  for i in 1 ..< caseStmt.len:
     startBlock(p)
     let it = caseStmt.sons[i]
     for j in 0 .. it.len-2:
@@ -391,7 +406,7 @@ proc genComputedGoto(p: BProc; n: PNode) =
   # first pass: Generate array of computed labels:
   var casePos = -1
   var arraySize: int
-  for i in 0 .. <n.len:
+  for i in 0 ..< n.len:
     let it = n.sons[i]
     if it.kind == nkCaseStmt:
       if lastSon(it).kind != nkOfBranch:
@@ -421,7 +436,7 @@ proc genComputedGoto(p: BProc; n: PNode) =
   let oldBody = p.blocks[topBlock].sections[cpsStmts]
   p.blocks[topBlock].sections[cpsStmts] = nil
 
-  for j in casePos+1 .. <n.len: genStmts(p, n.sons[j])
+  for j in casePos+1 ..< n.len: genStmts(p, n.sons[j])
   let tailB = p.blocks[topBlock].sections[cpsStmts]
 
   p.blocks[topBlock].sections[cpsStmts] = nil
@@ -436,7 +451,7 @@ proc genComputedGoto(p: BProc; n: PNode) =
   # first goto:
   lineF(p, cpsStmts, "goto *$#[$#];$n", [tmp, a.rdLoc])
 
-  for i in 1 .. <caseStmt.len:
+  for i in 1 ..< caseStmt.len:
     startBlock(p)
     let it = caseStmt.sons[i]
     for j in 0 .. it.len-2:
@@ -446,7 +461,7 @@ proc genComputedGoto(p: BProc; n: PNode) =
       let val = getOrdValue(it.sons[j])
       lineF(p, cpsStmts, "TMP$#_:$n", [intLiteral(val+id+1)])
     genStmts(p, it.lastSon)
-    #for j in casePos+1 .. <n.len: genStmts(p, n.sons[j]) # tailB
+    #for j in casePos+1 ..< n.len: genStmts(p, n.sons[j]) # tailB
     #for j in 0 .. casePos-1: genStmts(p, n.sons[j])  # tailA
     add(p.s(cpsStmts), tailB)
     add(p.s(cpsStmts), tailA)
@@ -513,7 +528,7 @@ proc genParForStmt(p: BProc, t: PNode) =
   preserveBreakIdx:
     let forLoopVar = t.sons[0].sym
     var rangeA, rangeB: TLoc
-    assignLocalVar(p, forLoopVar)
+    assignLocalVar(p, t.sons[0])
     #initLoc(forLoopVar.loc, locLocalVar, forLoopVar.typ, onStack)
     #discard mangleName(forLoopVar)
     let call = t.sons[1]
@@ -553,11 +568,8 @@ proc genBreakStmt(p: BProc, t: PNode) =
   genLineDir(p, t)
   lineF(p, cpsStmts, "goto $1;$n", [label])
 
-proc getRaiseFrmt(p: BProc): string =
-  result = "#raiseException((#Exception*)$1, $2);$n"
-
 proc genRaiseStmt(p: BProc, t: PNode) =
-  if p.inExceptBlock > 0:
+  if p.inExceptBlock > 0 and p.inExceptBlock == p.nestedTryStmts.len:
     # if the current try stmt have a finally block,
     # we must execute it before reraising
     var finallyBlock = p.nestedTryStmts[p.nestedTryStmts.len - 1].lastSon
@@ -569,7 +581,8 @@ proc genRaiseStmt(p: BProc, t: PNode) =
     var e = rdLoc(a)
     var typ = skipTypes(t.sons[0].typ, abstractPtrs)
     genLineDir(p, t)
-    lineCg(p, cpsStmts, getRaiseFrmt(p), [e, makeCString(typ.sym.name.s)])
+    lineCg(p, cpsStmts, "#raiseException((#Exception*)$1, $2);$n",
+        [e, makeCString(typ.sym.name.s)])
   else:
     genLineDir(p, t)
     # reraise the last exception:
@@ -733,7 +746,7 @@ proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) =
   if splitPoint+1 < n.len:
     lineF(p, cpsStmts, "switch ($1) {$n", [rdCharLoc(a)])
     var hasDefault = false
-    for i in splitPoint+1 .. < n.len:
+    for i in splitPoint+1 ..< n.len:
       # bug #4230: avoid false sharing between branches:
       if d.k == locTemp and isEmptyType(n.typ): d.k = locNone
       var branch = n[i]
@@ -766,91 +779,75 @@ proc genCase(p: BProc, t: PNode, d: var TLoc) =
     else:
       genOrdinalCase(p, t, d)
 
+
 proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   # code to generate:
   #
-  # XXX: There should be a standard dispatch algorithm
-  # that's used both here and with multi-methods
-  #
   #   try
   #   {
   #      myDiv(4, 9);
-  #   } catch (NimException& exp) {
-  #      if (isObj(exp, EIO) {
-  #        ...
-  #      } else if (isObj(exp, ESystem) {
-  #        ...
-  #        finallyPart()
-  #        raise;
-  #      } else {
-  #        // general handler
-  #      }
-  #  }
-  #  finallyPart();
+  #   } catch (NimExceptionType1&) {
+  #      body
+  #   } catch (NimExceptionType2&) {
+  #      finallyPart()
+  #      raise;
+  #   }
+  #   catch(...) {
+  #     general_handler_body
+  #   }
+  #   finallyPart();
+ 
+  template genExceptBranchBody(body: PNode) {.dirty.} =
+    if optStackTrace in p.options:
+      linefmt(p, cpsStmts, "#setFrame((TFrame*)&FR_);$n") 
+    expr(p, body, d)
+    linefmt(p, cpsStmts, "#popCurrentException();$n")
+    
   if not isEmptyType(t.typ) and d.k == locNone:
     getTemp(p, t.typ, d)
   genLineDir(p, t)
-  let exc = getTempName(p.module)
-  if getCompilerProc("Exception") != nil:
-    discard cgsym(p.module, "Exception")
-  else:
-    discard cgsym(p.module, "E_Base")
+
+  let end_label = getLabel(p)
+  discard cgsym(p.module, "Exception")
   add(p.nestedTryStmts, t)
   startBlock(p, "try {$n")
-  expr(p, t.sons[0], d)
-  let length = sonsLen(t)
-  endBlock(p, ropecg(p.module, "} catch (NimException& $1) {$n", [exc]))
-  if optStackTrace in p.options:
-    linefmt(p, cpsStmts, "#setFrame((TFrame*)&FR_);$n")
-  inc p.inExceptBlock
-  var i = 1
+  expr(p, t[0], d)
+  endBlock(p)
+
   var catchAllPresent = false
-  while (i < length) and (t.sons[i].kind == nkExceptBranch):
+
+  inc p.inExceptBlock
+  for i in 1..<t.len:
+    if t[i].kind != nkExceptBranch: break
+
     # bug #4230: avoid false sharing between branches:
     if d.k == locTemp and isEmptyType(t.typ): d.k = locNone
-    let blen = sonsLen(t.sons[i])
-    if i > 1: addf(p.s(cpsStmts), "else ", [])
-    if blen == 1:
+
+    if t[i].len == 1:
       # general except section:
       catchAllPresent = true
-      startBlock(p)
-      expr(p, t.sons[i].sons[0], d)
-      linefmt(p, cpsStmts, "#popCurrentException();$n")
+      startBlock(p, "catch (...) {$n")
+      genExceptBranchBody(t[i][0])
       endBlock(p)
     else:
-      var orExpr: Rope = nil
-      for j in countup(0, blen - 2):
-        assert(t.sons[i].sons[j].kind == nkType)
-        if orExpr != nil: add(orExpr, "||")
-        appcg(p.module, orExpr,
-              "#isObj($1.exp->m_type, $2)",
-              [exc, genTypeInfo(p.module, t.sons[i].sons[j].typ)])
-      lineF(p, cpsStmts, "if ($1) ", [orExpr])
-      startBlock(p)
-      expr(p, t.sons[i].sons[blen-1], d)
-      linefmt(p, cpsStmts, "#popCurrentException();$n")
-      endBlock(p)
-    inc(i)
-
-  # reraise the exception if there was no catch all
-  # and none of the handlers matched
-  if not catchAllPresent:
-    if i > 1: lineF(p, cpsStmts, "else ", [])
-    startBlock(p)
-    var finallyBlock = t.lastSon
-    if finallyBlock.kind == nkFinally:
-      #expr(p, finallyBlock.sons[0], d)
-      genStmts(p, finallyBlock.sons[0])
+      for j in 0..t[i].len-2:
+        assert(t[i][j].kind == nkType)
+        startBlock(p, "catch ($1*) {$n", getTypeDesc(p.module, t[i][j].typ))
+        genExceptBranchBody(t[i][^1])  # exception handler body will duplicated for every type
+        endBlock(p)
 
+  if not catchAllPresent and t[^1].kind == nkFinally:
+    # finally requires catch all presence
+    startBlock(p, "catch (...) {$n")
+    genSimpleBlock(p, t[^1][0])
     line(p, cpsStmts, ~"throw;$n")
     endBlock(p)
 
-  lineF(p, cpsStmts, "}$n", []) # end of catch block
   dec p.inExceptBlock
-
   discard pop(p.nestedTryStmts)
-  if (i < length) and (t.sons[i].kind == nkFinally):
-    genSimpleBlock(p, t.sons[i].sons[0])
+
+  if t[^1].kind == nkFinally:
+    genSimpleBlock(p, t[^1][0])
 
 proc genTry(p: BProc, t: PNode, d: var TLoc) =
   # code to generate:
@@ -933,7 +930,7 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
           "#isObj(#getCurrentException()->Sup.m_type, $1)"
           else: "#isObj(#getCurrentException()->m_type, $1)"
         appcg(p.module, orExpr, isObjFormat,
-              [genTypeInfo(p.module, t.sons[i].sons[j].typ)])
+              [genTypeInfo(p.module, t[i][j].typ, t[i][j].info)])
       if i > 1: line(p, cpsStmts, "else ")
       startBlock(p, "if ($1) {$n", [orExpr])
       linefmt(p, cpsStmts, "$1.status = 0;$n", safePoint)
@@ -958,7 +955,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
       res.add(t.sons[i].strVal)
     of nkSym:
       var sym = t.sons[i].sym
-      if sym.kind in {skProc, skIterator, skMethod}:
+      if sym.kind in {skProc, skFunc, skIterator, skMethod}:
         var a: TLoc
         initLocExpr(p, t.sons[i], a)
         res.add($rdLoc(a))
@@ -1051,7 +1048,7 @@ proc genWatchpoint(p: BProc, n: PNode) =
   let typ = skipTypes(n.sons[1].typ, abstractVarRange)
   lineCg(p, cpsStmts, "#dbgRegisterWatchpoint($1, (NCSTRING)$2, $3);$n",
         [a.addrLoc, makeCString(renderTree(n.sons[1])),
-        genTypeInfo(p.module, typ)])
+        genTypeInfo(p.module, typ, n.info)])
 
 proc genPragma(p: BProc, n: PNode) =
   for i in countup(0, sonsLen(n) - 1):
@@ -1081,7 +1078,7 @@ proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
                           field: PSym) =
   var t = skipTypes(objtype, abstractVar)
   assert t.kind == tyObject
-  discard genTypeInfo(p.module, t)
+  discard genTypeInfo(p.module, t, a.lode.info)
   var L = lengthOrd(field.typ)
   if not containsOrIncl(p.module.declaredThings, field.id):
     appcg(p.module, cfsVars, "extern $1",
@@ -1101,19 +1098,46 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   genDiscriminantCheck(p, a, tmp, dotExpr.sons[0].typ, dotExpr.sons[1].sym)
   genAssignment(p, a, tmp, {})
 
+proc patchAsgnStmtListExpr(father, orig, n: PNode) =
+  case n.kind
+  of nkDerefExpr, nkHiddenDeref:
+    let asgn = copyNode(orig)
+    asgn.add orig[0]
+    asgn.add n
+    father.add asgn
+  of nkStmtList, nkStmtListExpr:
+    for x in n:
+      patchAsgnStmtListExpr(father, orig, x)
+  else:
+    father.add n
+
 proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
   if e.sons[0].kind == nkSym and sfGoto in e.sons[0].sym.flags:
     genLineDir(p, e)
     genGotoVar(p, e.sons[1])
   elif not fieldDiscriminantCheckNeeded(p, e):
+    # this fixes bug #6422 but we really need to change the representation of
+    # arrays in the backend...
+    let le = e[0]
+    let ri = e[1]
+    var needsRepair = false
+    var it = ri
+    while it.kind in {nkStmtList, nkStmtListExpr}:
+      it = it.lastSon
+      needsRepair = true
+    if it.kind in {nkDerefExpr, nkHiddenDeref} and needsRepair:
+      var patchedTree = newNodeI(nkStmtList, e.info)
+      patchAsgnStmtListExpr(patchedTree, e, ri)
+      genStmts(p, patchedTree)
+      return
+
     var a: TLoc
-    if e[0].kind in {nkDerefExpr, nkHiddenDeref}:
-      genDeref(p, e[0], a, enforceDeref=true)
+    if le.kind in {nkDerefExpr, nkHiddenDeref}:
+      genDeref(p, le, a, enforceDeref=true)
     else:
-      initLocExpr(p, e.sons[0], a)
+      initLocExpr(p, le, a)
     if fastAsgn: incl(a.flags, lfNoDeepCopy)
     assert(a.t != nil)
-    let ri = e.sons[1]
     genLineDir(p, ri)
     loadInto(p, e.sons[0], ri, a)
   else:

compiler/ccgtrav.nim

@@ -66,7 +66,8 @@ proc genTraverseProc(c: var TTraversalClosure, accessor: Rope, typ: PType) =
 
   var p = c.p
   case typ.kind
-  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct:
+  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred,
+     tySink:
     genTraverseProc(c, accessor, lastSon(typ))
   of tyArray:
     let arraySize = lengthOrd(typ.sons[0])
@@ -121,7 +122,8 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash;
   var c: TTraversalClosure
   var p = newProc(nil, m)
   result = "Marker_" & getTypeName(m, origTyp, sig)
-  let typ = origTyp.skipTypes(abstractInst)
+  var typ = origTyp.skipTypes(abstractInst)
+  if typ.kind == tyOpt: typ = optLowering(typ)
 
   case reason
   of tiNew: c.visitorFrmt = "#nimGCvisit((void*)$1, op);$n"
@@ -150,8 +152,8 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash;
   m.s[cfsProcHeaders].addf("$1;$n", [header])
   m.s[cfsProcs].add(generatedProc)
 
-proc genTraverseProcForGlobal(m: BModule, s: PSym): Rope =
-  discard genTypeInfo(m, s.loc.t)
+proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope =
+  discard genTypeInfo(m, s.loc.t, info)
 
   var c: TTraversalClosure
   var p = newProc(nil, m)

compiler/ccgtypes.nim

@@ -92,7 +92,7 @@ proc mangleParamName(m: BModule; s: PSym): Rope =
 
 proc mangleLocalName(p: BProc; s: PSym): Rope =
   assert s.kind in skLocalVars+{skTemp}
-  assert sfGlobal notin s.flags
+  #assert sfGlobal notin s.flags
   result = s.loc.r
   if result == nil:
     var key = s.name.s.mangle
@@ -119,7 +119,7 @@ proc scopeMangledParam(p: BProc; param: PSym) =
 
 const
   irrelevantForBackend = {tyGenericBody, tyGenericInst, tyGenericInvocation,
-                          tyDistinct, tyRange, tyStatic, tyAlias}
+                          tyDistinct, tyRange, tyStatic, tyAlias, tySink, tyInferred}
 
 proc typeName(typ: PType): Rope =
   let typ = typ.skipTypes(irrelevantForBackend)
@@ -139,7 +139,7 @@ proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope =
       t = t.lastSon
     else:
       break
-  let typ = if typ.kind == tyAlias: typ.lastSon else: typ
+  let typ = if typ.kind in {tyAlias, tySink}: typ.lastSon else: typ
   if typ.loc.r == nil:
     typ.loc.r = typ.typeName & $sig
   else:
@@ -170,7 +170,7 @@ proc mapType(typ: PType): TCTypeKind =
     internalAssert typ.isResolvedUserTypeClass
     return mapType(typ.lastSon)
   of tyGenericBody, tyGenericInst, tyGenericParam, tyDistinct, tyOrdinal,
-     tyTypeDesc, tyAlias, tyInferred:
+     tyTypeDesc, tyAlias, tySink, tyInferred:
     result = mapType(lastSon(typ))
   of tyEnum:
     if firstOrd(typ) < 0:
@@ -183,7 +183,7 @@ proc mapType(typ: PType): TCTypeKind =
       of 8: result = ctInt64
       else: internalError("mapType")
   of tyRange: result = mapType(typ.sons[0])
-  of tyPtr, tyVar, tyRef:
+  of tyPtr, tyVar, tyLent, tyRef, tyOptAsRef:
     var base = skipTypes(typ.lastSon, typedescInst)
     case base.kind
     of tyOpenArray, tyArray, tyVarargs: result = ctPtrToArray
@@ -194,6 +194,13 @@ proc mapType(typ: PType): TCTypeKind =
     else: result = ctPtr
   of tyPointer: result = ctPtr
   of tySequence: result = ctNimSeq
+  of tyOpt:
+    case optKind(typ)
+    of oBool: result = ctStruct
+    of oNil, oPtr: result = ctPtr
+    of oEnum:
+      # The 'nil' value is always negative, so we always use a signed integer
+      result = if getSize(typ.sons[0]) == 8: ctInt64 else: ctInt32
   of tyProc: result = if typ.callConv != ccClosure: ctProc else: ctStruct
   of tyString: result = ctNimStr
   of tyCString: result = ctCString
@@ -235,7 +242,7 @@ proc isInvalidReturnType(rettype: PType): bool =
     case mapType(rettype)
     of ctArray:
       result = not (skipTypes(rettype, typedescInst).kind in
-          {tyVar, tyRef, tyPtr})
+          {tyVar, tyLent, tyRef, tyPtr})
     of ctStruct:
       let t = skipTypes(rettype, typedescInst)
       if rettype.isImportedCppType or t.isImportedCppType: return false
@@ -271,7 +278,10 @@ proc ccgIntroducedPtr(s: PSym): bool =
   elif tfByCopy in pt.flags: return false
   case pt.kind
   of tyObject:
-    if (optByRef in s.options) or (getSize(pt) > platform.floatSize * 2):
+    if s.typ.sym != nil and sfForward in s.typ.sym.flags:
+      # forwarded objects are *always* passed by pointers for consistency!
+      result = true
+    elif (optByRef in s.options) or (getSize(pt) > platform.floatSize * 3):
       result = true           # requested anyway
     elif (tfFinal in pt.flags) and (pt.sons[0] == nil):
       result = false          # no need, because no subtyping possible
@@ -279,15 +289,16 @@ proc ccgIntroducedPtr(s: PSym): bool =
       result = true           # ordinary objects are always passed by reference,
                               # otherwise casting doesn't work
   of tyTuple:
-    result = (getSize(pt) > platform.floatSize*2) or (optByRef in s.options)
+    result = (getSize(pt) > platform.floatSize*3) or (optByRef in s.options)
   else: result = false
 
-proc fillResult(param: PSym) =
-  fillLoc(param.loc, locParam, param.typ, ~"Result",
+proc fillResult(param: PNode) =
+  fillLoc(param.sym.loc, locParam, param, ~"Result",
           OnStack)
-  if mapReturnType(param.typ) != ctArray and isInvalidReturnType(param.typ):
-    incl(param.loc.flags, lfIndirect)
-    param.loc.s = OnUnknown
+  let t = param.sym.typ
+  if mapReturnType(t) != ctArray and isInvalidReturnType(t):
+    incl(param.sym.loc.flags, lfIndirect)
+    param.sym.loc.storage = OnUnknown
 
 proc typeNameOrLiteral(m: BModule; t: PType, literal: string): Rope =
   if t.sym != nil and sfImportc in t.sym.flags and t.sym.magic == mNone:
@@ -317,7 +328,7 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
   of tyStatic:
     if typ.n != nil: result = getSimpleTypeDesc(m, lastSon typ)
     else: internalError("tyStatic for getSimpleTypeDesc")
-  of tyGenericInst, tyAlias:
+  of tyGenericInst, tyAlias, tySink:
     result = getSimpleTypeDesc(m, lastSon typ)
   else: result = nil
 
@@ -337,7 +348,7 @@ proc getTypePre(m: BModule, typ: PType; sig: SigHash): Rope =
     if result == nil: result = cacheGetType(m.typeCache, sig)
 
 proc structOrUnion(t: PType): Rope =
-  let t = t.skipTypes({tyAlias})
+  let t = t.skipTypes({tyAlias, tySink})
   (if tfUnion in t.flags: rope("union") else: rope("struct"))
 
 proc getForwardStructFormat(m: BModule): string =
@@ -349,7 +360,7 @@ proc getTypeForward(m: BModule, typ: PType; sig: SigHash): Rope =
   if result != nil: return
   result = getTypePre(m, typ, sig)
   if result != nil: return
-  let concrete = typ.skipTypes(abstractInst)
+  let concrete = typ.skipTypes(abstractInst + {tyOpt})
   case concrete.kind
   of tySequence, tyTuple, tyObject:
     result = getTypeName(m, typ, sig)
@@ -375,11 +386,17 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet): Rope =
   of tySequence:
     result = getTypeForward(m, t, hashType(t)) & "*"
     pushType(m, t)
+  of tyOpt:
+    if optKind(etB) == oPtr:
+      result = getTypeForward(m, t, hashType(t)) & "*"
+      pushType(m, t)
+    else:
+      result = getTypeDescAux(m, t, check)
   else:
     result = getTypeDescAux(m, t, check)
 
 proc paramStorageLoc(param: PSym): TStorageLoc =
-  if param.typ.skipTypes({tyVar, tyTypeDesc}).kind notin {
+  if param.typ.skipTypes({tyVar, tyLent, tyTypeDesc}).kind notin {
           tyArray, tyOpenArray, tyVarargs}:
     result = OnStack
   else:
@@ -398,13 +415,13 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
     var param = t.n.sons[i].sym
     if isCompileTimeOnly(param.typ): continue
     if params != nil: add(params, ~", ")
-    fillLoc(param.loc, locParam, param.typ, mangleParamName(m, param),
+    fillLoc(param.loc, locParam, t.n.sons[i], mangleParamName(m, param),
             param.paramStorageLoc)
     if ccgIntroducedPtr(param):
       add(params, getTypeDescWeak(m, param.typ, check))
       add(params, ~"*")
       incl(param.loc.flags, lfIndirect)
-      param.loc.s = OnUnknown
+      param.loc.storage = OnUnknown
     elif weakDep:
       add(params, getTypeDescWeak(m, param.typ, check))
     else:
@@ -413,11 +430,11 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
     add(params, param.loc.r)
     # declare the len field for open arrays:
     var arr = param.typ
-    if arr.kind == tyVar: arr = arr.sons[0]
+    if arr.kind in {tyVar, tyLent}: arr = arr.lastSon
     var j = 0
     while arr.kind in {tyOpenArray, tyVarargs}:
       # this fixes the 'sort' bug:
-      if param.typ.kind == tyVar: param.loc.s = OnUnknown
+      if param.typ.kind in {tyVar, tyLent}: param.loc.storage = OnUnknown
       # need to pass hidden parameter:
       addf(params, ", NI $1Len_$2", [param.loc.r, j.rope])
       inc(j)
@@ -479,7 +496,7 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
               if hasAttribute in CC[cCompiler].props:
                 add(unionBody, "struct __attribute__((__packed__)){" )
               else:
-                addf(unionBody, "#pragma pack(1)$nstruct{", [])
+                addf(unionBody, "#pragma pack(push, 1)$nstruct{", [])
             add(unionBody, a)
             addf(unionBody, "} $1;$n", [sname])
             if tfPacked in rectype.flags and hasAttribute notin CC[cCompiler].props:
@@ -496,16 +513,16 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
     let sname = mangleRecFieldName(m, field, rectype)
     let ae = if accessExpr != nil: "$1.$2" % [accessExpr, sname]
              else: sname
-    fillLoc(field.loc, locField, field.typ, ae, OnUnknown)
+    fillLoc(field.loc, locField, n, ae, OnUnknown)
     # for importcpp'ed objects, we only need to set field.loc, but don't
     # have to recurse via 'getTypeDescAux'. And not doing so prevents problems
     # with heavily templatized C++ code:
     if not isImportedCppType(rectype):
-      let fieldType = field.loc.t.skipTypes(abstractInst)
+      let fieldType = field.loc.lode.typ.skipTypes(abstractInst)
       if fieldType.kind == tyArray and tfUncheckedArray in fieldType.flags:
         addf(result, "$1 $2[SEQ_DECL_SIZE];$n",
             [getTypeDescAux(m, fieldType.elemType, check), sname])
-      elif fieldType.kind == tySequence:
+      elif fieldType.kind in {tySequence, tyOpt}:
         # we need to use a weak dependency here for trecursive_table.
         addf(result, "$1 $2;$n", [getTypeDescWeak(m, field.loc.t, check), sname])
       elif field.bitsize != 0:
@@ -534,7 +551,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
     if hasAttribute in CC[cCompiler].props:
       result = structOrUnion(typ) & " __attribute__((__packed__))"
     else:
-      result = "#pragma pack(1)" & tnl & structOrUnion(typ)
+      result = "#pragma pack(push, 1)" & tnl & structOrUnion(typ)
   else:
     result = structOrUnion(typ)
 
@@ -552,6 +569,8 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
     elif m.compileToCpp:
       appcg(m, result, " : public $1 {$n",
                       [getTypeDescAux(m, typ.sons[0].skipTypes(skipPtrs), check)])
+      if typ.isException:
+        appcg(m, result, "virtual void raise() {throw this;}$n") # required for polymorphic exceptions
       hasField = true
     else:
       appcg(m, result, " {$n  $1 Sup;$n",
@@ -624,7 +643,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
     excl(check, t.id)
     return
   case t.kind
-  of tyRef, tyPtr, tyVar:
+  of tyRef, tyOptAsRef, tyPtr, tyVar, tyLent:
     var star = if t.kind == tyVar and tfVarIsPtr notin origTyp.flags and
                     compileToCpp(m): "&" else: "*"
     var et = origTyp.skipTypes(abstractInst).lastSon
@@ -651,6 +670,21 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
       result = name & "*" & star
       m.typeCache[sig] = result
       pushType(m, et)
+    of tyOpt:
+      if etB.sons[0].kind in {tyObject, tyTuple}:
+        let name = getTypeForward(m, et, hashType et)
+        result = name & "*" & star
+        m.typeCache[sig] = result
+        pushType(m, et)
+      elif optKind(etB) == oBool:
+        let name = getTypeForward(m, et, hashType et)
+        result = name & "*"
+        m.typeCache[sig] = result
+        pushType(m, et)
+      else:
+        # else we have a strong dependency  :-(
+        result = getTypeDescAux(m, et, check) & star
+        m.typeCache[sig] = result
     else:
       # else we have a strong dependency  :-(
       result = getTypeDescAux(m, et, check) & star
@@ -726,6 +760,38 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
       else:
         result = rope("TGenericSeq")
     add(result, "*")
+  of tyOpt:
+    result = cacheGetType(m.typeCache, sig)
+    if result == nil:
+      case optKind(t)
+      of oBool:
+        result = cacheGetType(m.forwTypeCache, sig)
+        if result == nil:
+          result = getTypeName(m, origTyp, sig)
+          addf(m.s[cfsForwardTypes], getForwardStructFormat(m),
+              [structOrUnion(t), result])
+          m.forwTypeCache[sig] = result
+        appcg(m, m.s[cfsSeqTypes], "struct $2 {$n" &
+           "  NIM_BOOL Field0;$n" &
+           "  $1 Field1;$n" &
+           "};$n", [getTypeDescAux(m, t.sons[0], check), result])
+      of oPtr:
+        let et = t.sons[0]
+        if et.kind in {tyTuple, tyObject}:
+          let name = getTypeForward(m, et, hashType et)
+          result = name & "*"
+          pushType(m, et)
+        else:
+          result = getTypeDescAux(m, t.sons[0], check) & "*"
+      of oNil:
+        result = getTypeDescAux(m, t.sons[0], check)
+      of oEnum:
+        result = getTypeName(m, origTyp, sig)
+        if getSize(t.sons[0]) == 8:
+          addf(m.s[cfsTypes], "typedef NI64 $1;$n", [result])
+        else:
+          addf(m.s[cfsTypes], "typedef NI32 $1;$n", [result])
+      m.typeCache[sig] = result
   of tyArray:
     var n: BiggestInt = lengthOrd(t)
     if n <= 0: n = 1   # make an array of at least one element
@@ -745,7 +811,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
       var chunkStart = 0
       while i < cppName.data.len:
         if cppName.data[i] == '\'':
-          var chunkEnd = <i
+          var chunkEnd = i-1
           var idx, stars: int
           if scanCppGenericSlot(cppName.data, i, idx, stars):
             result.add cppName.data.substr(chunkStart, chunkEnd)
@@ -793,11 +859,12 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
              [structOrUnion(t), result])
         assert m.forwTypeCache[sig] == result
       m.typeCache[sig] = result # always call for sideeffects:
-      let recdesc = if t.kind != tyTuple: getRecordDesc(m, t, result, check)
-                    else: getTupleDesc(m, t, result, check)
-      if not isImportedType(t):
-        add(m.s[cfsTypes], recdesc)
-      elif tfIncompleteStruct notin t.flags: addAbiCheck(m, t, result)
+      if not incompleteType(t):
+        let recdesc = if t.kind != tyTuple: getRecordDesc(m, t, result, check)
+                      else: getTupleDesc(m, t, result, check)
+        if not isImportedType(t):
+          add(m.s[cfsTypes], recdesc)
+        elif tfIncompleteStruct notin t.flags: addAbiCheck(m, t, result)
   of tySet:
     result = $t.kind & '_' & getTypeName(m, t.lastSon, hashType t.lastSon)
     m.typeCache[sig] = result
@@ -807,7 +874,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
       of 1, 2, 4, 8: addf(m.s[cfsTypes], "typedef NU$2 $1;$n", [result, rope(s*8)])
       else: addf(m.s[cfsTypes], "typedef NU8 $1[$2];$n",
              [result, rope(getSize(t))])
-  of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias,
+  of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tySink,
      tyUserTypeClass, tyUserTypeClassInst, tyInferred:
     result = getTypeDescAux(m, lastSon(t), check)
   else:
@@ -860,8 +927,10 @@ proc genProcHeader(m: BModule, prc: PSym): Rope =
       result.add "N_LIB_EXPORT "
   elif prc.typ.callConv == ccInline:
     result.add "static "
+  elif {sfImportc, sfExportc} * prc.flags == {}:
+    result.add "N_LIB_PRIVATE "
   var check = initIntSet()
-  fillLoc(prc.loc, locProc, prc.typ, mangleName(m, prc), OnUnknown)
+  fillLoc(prc.loc, locProc, prc.ast[namePos], mangleName(m, prc), OnUnknown)
   genProcParams(m, prc.typ, rettype, params, check)
   # careful here! don't access ``prc.ast`` as that could reload large parts of
   # the object graph!
@@ -874,12 +943,13 @@ proc genProcHeader(m: BModule, prc: PSym): Rope =
 
 # ------------------ type info generation -------------------------------------
 
-proc genTypeInfo(m: BModule, t: PType): Rope
+proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope
 proc getNimNode(m: BModule): Rope =
   result = "$1[$2]" % [m.typeNodesName, rope(m.typeNodes)]
   inc(m.typeNodes)
 
-proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope) =
+proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
+                        name, base: Rope; info: TLineInfo) =
   var nimtypeKind: int
   #allocMemTI(m, typ, name)
   if isObjLackingTypeField(typ):
@@ -902,22 +972,30 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope) =
     addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)])
   discard cgsym(m, "TNimType")
   if isDefined("nimTypeNames"):
+    var typename = typeToString(if origType.typeInst != nil: origType.typeInst
+                                else: origType, preferName)
+    if typename == "ref object" and origType.skipTypes(skipPtrs).sym != nil:
+      typename = "anon ref object from " & $origType.skipTypes(skipPtrs).sym.info
     addf(m.s[cfsTypeInit3], "$1.name = $2;$n",
-        [name, makeCstring typeToString(origType, preferName)])
+        [name, makeCstring typename])
     discard cgsym(m, "nimTypeRoot")
     addf(m.s[cfsTypeInit3], "$1.nextType = nimTypeRoot; nimTypeRoot=&$1;$n",
          [name])
   addf(m.s[cfsVars], "TNimType $1;$n", [name])
 
-proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope) =
+proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope;
+                    info: TLineInfo) =
   var base: Rope
   if sonsLen(typ) > 0 and typ.lastSon != nil:
     var x = typ.lastSon
     if typ.kind == tyObject: x = x.skipTypes(skipPtrs)
-    base = genTypeInfo(m, x)
+    if typ.kind == tyPtr and x.kind == tyObject and incompleteType(x):
+      base = rope("0")
+    else:
+      base = genTypeInfo(m, x, info)
   else:
     base = rope("0")
-  genTypeInfoAuxBase(m, typ, origType, name, base)
+  genTypeInfoAuxBase(m, typ, origType, name, base, info)
 
 proc discriminatorTableName(m: BModule, objtype: PType, d: PSym): Rope =
   # bugfix: we need to search the type that contains the discriminator:
@@ -933,19 +1011,20 @@ proc discriminatorTableDecl(m: BModule, objtype: PType, d: PSym): Rope =
   var tmp = discriminatorTableName(m, objtype, d)
   result = "TNimNode* $1[$2];$n" % [tmp, rope(lengthOrd(d.typ)+1)]
 
-proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope) =
+proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
+                     info: TLineInfo) =
   case n.kind
   of nkRecList:
     var L = sonsLen(n)
     if L == 1:
-      genObjectFields(m, typ, origType, n.sons[0], expr)
+      genObjectFields(m, typ, origType, n.sons[0], expr, info)
     elif L > 0:
       var tmp = getTempName(m)
       addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(L)])
       for i in countup(0, L-1):
         var tmp2 = getNimNode(m)
         addf(m.s[cfsTypeInit3], "$1[$2] = &$3;$n", [tmp, rope(i), tmp2])
-        genObjectFields(m, typ, origType, n.sons[i], tmp2)
+        genObjectFields(m, typ, origType, n.sons[i], tmp2, info)
       addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 2; $1.sons = &$3[0];$n",
            [expr, rope(L), tmp])
     else:
@@ -963,14 +1042,14 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope) =
         "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" &
         "$1.name = $5;$n" & "$1.sons = &$6[0];$n" &
         "$1.len = $7;$n", [expr, getTypeDesc(m, origType), field.loc.r,
-                           genTypeInfo(m, field.typ),
+                           genTypeInfo(m, field.typ, info),
                            makeCString(field.name.s),
                            tmp, rope(L)])
     addf(m.s[cfsData], "TNimNode* $1[$2];$n", [tmp, rope(L+1)])
     for i in countup(1, sonsLen(n)-1):
       var b = n.sons[i]           # branch
       var tmp2 = getNimNode(m)
-      genObjectFields(m, typ, origType, lastSon(b), tmp2)
+      genObjectFields(m, typ, origType, lastSon(b), tmp2, info)
       case b.kind
       of nkOfBranch:
         if sonsLen(b) < 2:
@@ -998,15 +1077,20 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope) =
       addf(m.s[cfsTypeInit3], "$1.kind = 1;$n" &
           "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" &
           "$1.name = $5;$n", [expr, getTypeDesc(m, origType),
-          field.loc.r, genTypeInfo(m, field.typ), makeCString(field.name.s)])
+          field.loc.r, genTypeInfo(m, field.typ, info), makeCString(field.name.s)])
   else: internalError(n.info, "genObjectFields")
 
-proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope) =
-  if typ.kind == tyObject: genTypeInfoAux(m, typ, origType, name)
-  else: genTypeInfoAuxBase(m, typ, origType, name, rope("0"))
+proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) =
+  if typ.kind == tyObject:
+    if incompleteType(typ):
+      localError(info, "request for RTTI generation for incomplete object: " &
+                        typeToString(typ))
+    genTypeInfoAux(m, typ, origType, name, info)
+  else:
+    genTypeInfoAuxBase(m, typ, origType, name, rope("0"), info)
   var tmp = getNimNode(m)
   if not isImportedType(typ):
-    genObjectFields(m, typ, origType, typ.n, tmp)
+    genObjectFields(m, typ, origType, typ.n, tmp, info)
   addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, tmp])
   var t = typ.sons[0]
   while t != nil:
@@ -1014,8 +1098,8 @@ proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope) =
     t.flags.incl tfObjHasKids
     t = t.sons[0]
 
-proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope) =
-  genTypeInfoAuxBase(m, typ, typ, name, rope("0"))
+proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) =
+  genTypeInfoAuxBase(m, typ, typ, name, rope("0"), info)
   var expr = getNimNode(m)
   var length = sonsLen(typ)
   if length > 0:
@@ -1029,7 +1113,7 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope) =
           "$1.offset = offsetof($2, Field$3);$n" &
           "$1.typ = $4;$n" &
           "$1.name = \"Field$3\";$n",
-           [tmp2, getTypeDesc(m, origType), rope(i), genTypeInfo(m, a)])
+           [tmp2, getTypeDesc(m, origType), rope(i), genTypeInfo(m, a, info)])
     addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 2; $1.sons = &$3[0];$n",
          [expr, rope(length), tmp])
   else:
@@ -1037,12 +1121,12 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope) =
          [expr, rope(length)])
   addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, expr])
 
-proc genEnumInfo(m: BModule, typ: PType, name: Rope) =
+proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
   # Type information for enumerations is quite heavy, so we do some
   # optimizations here: The ``typ`` field is never set, as it is redundant
   # anyway. We generate a cstring array and a loop over it. Exceptional
   # positions will be reset after the loop.
-  genTypeInfoAux(m, typ, typ, name)
+  genTypeInfoAux(m, typ, typ, name, info)
   var nodePtrs = getTempName(m)
   var length = sonsLen(typ.n)
   addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n",
@@ -1080,15 +1164,15 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope) =
     # 1 << 2 is {ntfEnumHole}
     addf(m.s[cfsTypeInit3], "$1.flags = 1<<2;$n", [name])
 
-proc genSetInfo(m: BModule, typ: PType, name: Rope) =
+proc genSetInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
   assert(typ.sons[0] != nil)
-  genTypeInfoAux(m, typ, typ, name)
+  genTypeInfoAux(m, typ, typ, name, info)
   var tmp = getNimNode(m)
   addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 0;$n" & "$3.node = &$1;$n",
        [tmp, rope(firstOrd(typ)), name])
 
-proc genArrayInfo(m: BModule, typ: PType, name: Rope) =
-  genTypeInfoAuxBase(m, typ, typ, name, genTypeInfo(m, typ.sons[1]))
+proc genArrayInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
+  genTypeInfoAuxBase(m, typ, typ, name, genTypeInfo(m, typ.sons[1], info), info)
 
 proc fakeClosureType(owner: PSym): PType =
   # we generate the same RTTI as for a tuple[pointer, ref tuple[]]
@@ -1110,9 +1194,11 @@ proc genDeepCopyProc(m: BModule; s: PSym; result: Rope) =
   addf(m.s[cfsTypeInit3], "$1.deepcopy =(void* (N_RAW_NIMCALL*)(void*))$2;$n",
      [result, s.loc.r])
 
-proc genTypeInfo(m: BModule, t: PType): Rope =
+proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope =
   let origType = t
   var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses)
+  if t.kind == tyOpt:
+    return genTypeInfo(m, optLowering(t), info)
 
   let sig = hashType(origType)
   result = m.typeInfoMarker.getOrDefault(sig)
@@ -1134,7 +1220,7 @@ proc genTypeInfo(m: BModule, t: PType): Rope =
   let owner = t.skipTypes(typedescPtrs).owner.getModule
   if owner != m.module:
     # make sure the type info is created in the owner module
-    discard genTypeInfo(m.g.modules[owner.position], origType)
+    discard genTypeInfo(m.g.modules[owner.position], origType, info)
     # reference the type info as extern here
     discard cgsym(m, "TNimType")
     discard cgsym(m, "TNimNode")
@@ -1144,36 +1230,36 @@ proc genTypeInfo(m: BModule, t: PType): Rope =
   m.g.typeInfoMarker[sig] = result
   case t.kind
   of tyEmpty, tyVoid: result = rope"0"
-  of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar:
-    genTypeInfoAuxBase(m, t, t, result, rope"0")
+  of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar, tyLent:
+    genTypeInfoAuxBase(m, t, t, result, rope"0", info)
   of tyStatic:
-    if t.n != nil: result = genTypeInfo(m, lastSon t)
+    if t.n != nil: result = genTypeInfo(m, lastSon t, info)
     else: internalError("genTypeInfo(" & $t.kind & ')')
   of tyUserTypeClasses:
     internalAssert t.isResolvedUserTypeClass
-    return genTypeInfo(m, t.lastSon)
+    return genTypeInfo(m, t.lastSon, info)
   of tyProc:
     if t.callConv != ccClosure:
-      genTypeInfoAuxBase(m, t, t, result, rope"0")
+      genTypeInfoAuxBase(m, t, t, result, rope"0", info)
     else:
       let x = fakeClosureType(t.owner)
-      genTupleInfo(m, x, x, result)
-  of tySequence, tyRef:
-    genTypeInfoAux(m, t, t, result)
+      genTupleInfo(m, x, x, result, info)
+  of tySequence, tyRef, tyOptAsRef:
+    genTypeInfoAux(m, t, t, result, info)
     if gSelectedGC >= gcMarkAndSweep:
       let markerProc = genTraverseProc(m, origType, sig, tiNew)
       addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [result, markerProc])
-  of tyPtr, tyRange: genTypeInfoAux(m, t, t, result)
-  of tyArray: genArrayInfo(m, t, result)
-  of tySet: genSetInfo(m, t, result)
-  of tyEnum: genEnumInfo(m, t, result)
-  of tyObject: genObjectInfo(m, t, origType, result)
+  of tyPtr, tyRange: genTypeInfoAux(m, t, t, result, info)
+  of tyArray: genArrayInfo(m, t, result, info)
+  of tySet: genSetInfo(m, t, result, info)
+  of tyEnum: genEnumInfo(m, t, result, info)
+  of tyObject: genObjectInfo(m, t, origType, result, info)
   of tyTuple:
     # if t.n != nil: genObjectInfo(m, t, result)
     # else:
     # BUGFIX: use consistently RTTI without proper field names; otherwise
     # results are not deterministic!
-    genTupleInfo(m, t, origType, result)
+    genTupleInfo(m, t, origType, result, info)
   else: internalError("genTypeInfo(" & $t.kind & ')')
   if t.deepCopy != nil:
     genDeepCopyProc(m, t.deepCopy, result)

compiler/ccgutils.nim

@@ -16,11 +16,11 @@ import
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
   case n.kind
   of nkStmtList:
-    for i in 0 .. < n.len:
+    for i in 0 ..< n.len:
       result = getPragmaStmt(n[i], w)
       if result != nil: break
   of nkPragma:
-    for i in 0 .. < n.len:
+    for i in 0 ..< n.len:
       if whichPragma(n[i]) == w: return n[i]
   else: discard
 
@@ -100,7 +100,7 @@ proc getUniqueType*(key: PType): PType =
       if result == nil:
         gCanonicalTypes[k] = key
         result = key
-    of tyTypeDesc, tyTypeClasses, tyGenericParam, tyFromExpr, tyFieldAccessor:
+    of tyTypeDesc, tyTypeClasses, tyGenericParam, tyFromExpr:
       if key.isResolvedUserTypeClass:
         return getUniqueType(lastSon(key))
       if key.sym != nil:
@@ -110,13 +110,13 @@ proc getUniqueType*(key: PType): PType =
     of tyDistinct:
       if key.deepCopy != nil: result = key
       else: result = getUniqueType(lastSon(key))
-    of tyGenericInst, tyOrdinal, tyStatic, tyAlias, tyInferred:
+    of tyGenericInst, tyOrdinal, tyStatic, tyAlias, tySink, tyInferred:
       result = getUniqueType(lastSon(key))
       #let obj = lastSon(key)
       #if obj.sym != nil and obj.sym.name.s == "TOption":
       #  echo "for ", typeToString(key), " I returned "
       #  debug result
-    of tyPtr, tyRef, tyVar:
+    of tyPtr, tyRef, tyVar, tyLent:
       let elemType = lastSon(key)
       if elemType.kind in {tyBool, tyChar, tyInt..tyUInt64}:
         # no canonicalization for integral types, so that e.g. ``ptr pid_t`` is
@@ -126,7 +126,7 @@ proc getUniqueType*(key: PType): PType =
         result = slowSearch(key, k)
     of tyGenericInvocation, tyGenericBody,
        tyOpenArray, tyArray, tySet, tyRange, tyTuple,
-       tySequence, tyForward, tyVarargs, tyProxy:
+       tySequence, tyForward, tyVarargs, tyProxy, tyOpt:
       # we have to do a slow linear search because types may need
       # to be compared by their structure:
       result = slowSearch(key, k)
@@ -157,7 +157,7 @@ proc getUniqueType*(key: PType): PType =
       else:
         # ugh, we need the canon here:
         result = slowSearch(key, k)
-    of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("getUniqueType")
+    of tyUnused, tyOptAsRef, tyUnused1, tyUnused2: internalError("getUniqueType")
 
 proc makeSingleLineCString*(s: string): string =
   result = "\""

compiler/cgen.nim

@@ -11,7 +11,7 @@
 
 import
   ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets,
-  nversion, nimsets, msgs, securehash, bitsets, idents, types,
+  nversion, nimsets, msgs, std / sha1, bitsets, idents, types,
   ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth,
   condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases,
   lowerings, semparallel, tables, sets, ndi
@@ -47,21 +47,31 @@ proc findPendingModule(m: BModule, s: PSym): BModule =
   var ms = getModule(s)
   result = m.g.modules[ms.position]
 
-proc initLoc(result: var TLoc, k: TLocKind, typ: PType, s: TStorageLoc) =
+proc initLoc(result: var TLoc, k: TLocKind, lode: PNode, s: TStorageLoc) =
   result.k = k
-  result.s = s
-  result.t = typ
+  result.storage = s
+  result.lode = lode
   result.r = nil
   result.flags = {}
 
-proc fillLoc(a: var TLoc, k: TLocKind, typ: PType, r: Rope, s: TStorageLoc) =
+proc fillLoc(a: var TLoc, k: TLocKind, lode: PNode, r: Rope, s: TStorageLoc) =
   # fills the loc if it is not already initialized
   if a.k == locNone:
     a.k = k
-    a.t = typ
-    a.s = s
+    a.lode = lode
+    a.storage = s
     if a.r == nil: a.r = r
 
+proc t(a: TLoc): PType {.inline.} =
+  if a.lode.kind == nkSym:
+    result = a.lode.sym.typ
+  else:
+    result = a.lode.typ
+
+proc lodeTyp(t: PType): PNode =
+  result = newNode(nkEmpty)
+  result.typ = t
+
 proc isSimpleConst(typ: PType): bool =
   let t = skipTypes(typ, abstractVar)
   result = t.kind notin
@@ -250,6 +260,11 @@ proc rdCharLoc(a: TLoc): Rope =
 
 proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: TLoc,
                    takeAddr: bool) =
+  if p.module.compileToCpp and t.isException:
+    # init vtable in Exception object for polymorphic exceptions
+    includeHeader(p.module, "<new>")
+    linefmt(p, section, "new ($1) $2;$n", rdLoc(a), getTypeDesc(p.module, t))
+
   case analyseObjectWithTypeField(t)
   of frNone:
     discard
@@ -261,11 +276,11 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: TLoc,
       while (s.kind == tyObject) and (s.sons[0] != nil):
         add(r, ".Sup")
         s = skipTypes(s.sons[0], skipPtrs)
-    linefmt(p, section, "$1.m_type = $2;$n", r, genTypeInfo(p.module, t))
+    linefmt(p, section, "$1.m_type = $2;$n", r, genTypeInfo(p.module, t, a.lode.info))
   of frEmbedded:
     # worst case for performance:
     var r = if takeAddr: addrLoc(a) else: rdLoc(a)
-    linefmt(p, section, "#objectInit($1, $2);$n", r, genTypeInfo(p.module, t))
+    linefmt(p, section, "#objectInit($1, $2);$n", r, genTypeInfo(p.module, t, a.lode.info))
 
 type
   TAssignmentFlag = enum
@@ -286,7 +301,7 @@ proc resetLoc(p: BProc, loc: var TLoc) =
   if not isComplexValueType(typ):
     if containsGcRef:
       var nilLoc: TLoc
-      initLoc(nilLoc, locTemp, loc.t, OnStack)
+      initLoc(nilLoc, locTemp, loc.lode, OnStack)
       nilLoc.r = rope("NIM_NIL")
       genRefAssign(p, loc, nilLoc, {afSrcIsNil})
     else:
@@ -294,9 +309,9 @@ proc resetLoc(p: BProc, loc: var TLoc) =
   else:
     if optNilCheck in p.options:
       linefmt(p, cpsStmts, "#chckNil((void*)$1);$n", addrLoc(loc))
-    if loc.s != OnStack:
+    if loc.storage != OnStack:
       linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
-              addrLoc(loc), genTypeInfo(p.module, loc.t))
+              addrLoc(loc), genTypeInfo(p.module, loc.t, loc.lode.info))
       # XXX: generated reset procs should not touch the m_type
       # field, so disabling this should be safe:
       genObjectInit(p, cpsStmts, loc.t, loc, true)
@@ -340,8 +355,8 @@ proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) =
   result.r = "T" & rope(p.labels) & "_"
   linefmt(p, cpsLocals, "$1 $2;$n", getTypeDesc(p.module, t), result.r)
   result.k = locTemp
-  result.t = t
-  result.s = OnStack
+  result.lode = lodeTyp t
+  result.storage = OnStack
   result.flags = {}
   constructLoc(p, result, not needsInit)
 
@@ -350,8 +365,8 @@ proc getIntTemp(p: BProc, result: var TLoc) =
   result.r = "T" & rope(p.labels) & "_"
   linefmt(p, cpsLocals, "NI $1;$n", result.r)
   result.k = locTemp
-  result.s = OnStack
-  result.t = getSysType(tyInt)
+  result.storage = OnStack
+  result.lode = lodeTyp getSysType(tyInt)
   result.flags = {}
 
 proc initGCFrame(p: BProc): Rope =
@@ -371,13 +386,14 @@ proc localDebugInfo(p: BProc, s: PSym) =
   lineF(p, cpsInit,
        "FR_.s[$1].address = (void*)$3; FR_.s[$1].typ = $4; FR_.s[$1].name = $2;$n",
        [p.maxFrameLen.rope, makeCString(normalize(s.name.s)), a,
-        genTypeInfo(p.module, s.loc.t)])
+        genTypeInfo(p.module, s.loc.t, s.info)])
   inc(p.maxFrameLen)
   inc p.blocks[p.blocks.len-1].frameLen
 
-proc localVarDecl(p: BProc; s: PSym): Rope =
+proc localVarDecl(p: BProc; n: PNode): Rope =
+  let s = n.sym
   if s.loc.k == locNone:
-    fillLoc(s.loc, locLocalVar, s.typ, mangleLocalName(p, s), OnStack)
+    fillLoc(s.loc, locLocalVar, n, mangleLocalName(p, s), OnStack)
     if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy)
   result = getTypeDesc(p.module, s.typ)
   if s.constraint.isNil:
@@ -390,23 +406,24 @@ proc localVarDecl(p: BProc; s: PSym): Rope =
   else:
     result = s.cgDeclFrmt % [result, s.loc.r]
 
-proc assignLocalVar(p: BProc, s: PSym) =
+proc assignLocalVar(p: BProc, n: PNode) =
   #assert(s.loc.k == locNone) # not yet assigned
   # this need not be fulfilled for inline procs; they are regenerated
   # for each module that uses them!
   let nl = if optLineDir in gOptions: "" else: tnl
-  let decl = localVarDecl(p, s) & ";" & nl
+  let decl = localVarDecl(p, n) & ";" & nl
   line(p, cpsLocals, decl)
-  localDebugInfo(p, s)
+  localDebugInfo(p, n.sym)
 
 include ccgthreadvars
 
 proc varInDynamicLib(m: BModule, sym: PSym)
 proc mangleDynLibProc(sym: PSym): Rope
 
-proc assignGlobalVar(p: BProc, s: PSym) =
+proc assignGlobalVar(p: BProc, n: PNode) =
+  let s = n.sym
   if s.loc.k == locNone:
-    fillLoc(s.loc, locGlobalVar, s.typ, mangleName(p.module, s), OnHeap)
+    fillLoc(s.loc, locGlobalVar, n, mangleName(p.module, s), OnHeap)
 
   if lfDynamicLib in s.loc.flags:
     var q = findPendingModule(p.module, s)
@@ -439,16 +456,17 @@ proc assignGlobalVar(p: BProc, s: PSym) =
     appcg(p.module, p.module.s[cfsDebugInit],
           "#dbgRegisterGlobal($1, &$2, $3);$n",
          [makeCString(normalize(s.owner.name.s & '.' & s.name.s)),
-          s.loc.r, genTypeInfo(p.module, s.typ)])
+          s.loc.r, genTypeInfo(p.module, s.typ, n.info)])
 
 proc assignParam(p: BProc, s: PSym) =
   assert(s.loc.r != nil)
   scopeMangledParam(p, s)
   localDebugInfo(p, s)
 
-proc fillProcLoc(m: BModule; sym: PSym) =
+proc fillProcLoc(m: BModule; n: PNode) =
+  let sym = n.sym
   if sym.loc.k == locNone:
-    fillLoc(sym.loc, locProc, sym.typ, mangleName(m, sym), OnStack)
+    fillLoc(sym.loc, locProc, n, mangleName(m, sym), OnStack)
 
 proc getLabel(p: BProc): TLabel =
   inc(p.labels)
@@ -457,7 +475,7 @@ proc getLabel(p: BProc): TLabel =
 proc fixLabel(p: BProc, labl: TLabel) =
   lineF(p, cpsStmts, "$1: ;$n", [labl])
 
-proc genVarPrototype(m: BModule, sym: PSym)
+proc genVarPrototype(m: BModule, n: PNode)
 proc requestConstImpl(p: BProc, sym: PSym)
 proc genStmts(p: BProc, t: PNode)
 proc expr(p: BProc, n: PNode, d: var TLoc)
@@ -469,18 +487,43 @@ proc genLiteral(p: BProc, n: PNode): Rope
 proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope
 
 proc initLocExpr(p: BProc, e: PNode, result: var TLoc) =
-  initLoc(result, locNone, e.typ, OnUnknown)
+  initLoc(result, locNone, e, OnUnknown)
   expr(p, e, result)
 
 proc initLocExprSingleUse(p: BProc, e: PNode, result: var TLoc) =
-  initLoc(result, locNone, e.typ, OnUnknown)
+  initLoc(result, locNone, e, OnUnknown)
   result.flags.incl lfSingleUse
   expr(p, e, result)
 
 proc lenField(p: BProc): Rope =
   result = rope(if p.module.compileToCpp: "len" else: "Sup.len")
 
-include ccgcalls, "ccgstmts.nim", "ccgexprs.nim"
+include ccgcalls, "ccgstmts.nim"
+
+proc initFrame(p: BProc, procname, filename: Rope): Rope =
+  discard cgsym(p.module, "nimFrame")
+  if p.maxFrameLen > 0:
+    discard cgsym(p.module, "VarSlot")
+    result = rfmt(nil, "\tnimfrs_($1, $2, $3, $4);$n",
+                  procname, filename, p.maxFrameLen.rope,
+                  p.blocks[0].frameLen.rope)
+  else:
+    result = rfmt(nil, "\tnimfr_($1, $2);$n", procname, filename)
+
+proc initFrameNoDebug(p: BProc; frame, procname, filename: Rope; line: int): Rope =
+  discard cgsym(p.module, "nimFrame")
+  addf(p.blocks[0].sections[cpsLocals], "TFrame $1;$n", [frame])
+  result = rfmt(nil, "\t$1.procname = $2; $1.filename = $3; " &
+                      " $1.line = $4; $1.len = -1; nimFrame(&$1);$n",
+                      frame, procname, filename, rope(line))
+
+proc deinitFrameNoDebug(p: BProc; frame: Rope): Rope =
+  result = rfmt(p.module, "\t#popFrameOfAddr(&$1);$n", frame)
+
+proc deinitFrame(p: BProc): Rope =
+  result = rfmt(p.module, "\t#popFrame();$n")
+
+include ccgexprs
 
 # ----------------------------- dynamic library handling -----------------
 # We don't finalize dynamic libs as the OS does this for us.
@@ -587,11 +630,11 @@ proc symInDynamicLibPartial(m: BModule, sym: PSym) =
   sym.typ.sym = nil           # generate a new name
 
 proc cgsym(m: BModule, name: string): Rope =
-  var sym = magicsys.getCompilerProc(name)
+  let sym = magicsys.getCompilerProc(name)
   if sym != nil:
     case sym.kind
-    of skProc, skMethod, skConverter, skIterator: genProc(m, sym)
-    of skVar, skResult, skLet: genVarPrototype(m, sym)
+    of skProc, skFunc, skMethod, skConverter, skIterator: genProc(m, sym)
+    of skVar, skResult, skLet: genVarPrototype(m, newSymNode sym)
     of skType: discard getTypeDesc(m, sym.typ)
     else: internalError("cgsym: " & name & ": " & $sym.kind)
   else:
@@ -624,19 +667,6 @@ proc generateHeaders(m: BModule) =
   add(m.s[cfsHeaders], "#undef powerpc" & tnl)
   add(m.s[cfsHeaders], "#undef unix" & tnl)
 
-proc initFrame(p: BProc, procname, filename: Rope): Rope =
-  discard cgsym(p.module, "nimFrame")
-  if p.maxFrameLen > 0:
-    discard cgsym(p.module, "VarSlot")
-    result = rfmt(nil, "\tnimfrs_($1, $2, $3, $4);$n",
-                  procname, filename, p.maxFrameLen.rope,
-                  p.blocks[0].frameLen.rope)
-  else:
-    result = rfmt(nil, "\tnimfr_($1, $2);$n", procname, filename)
-
-proc deinitFrame(p: BProc): Rope =
-  result = rfmt(p.module, "\t#popFrame();$n")
-
 proc closureSetup(p: BProc, prc: PSym) =
   if tfCapturesEnv notin prc.typ.flags: return
   # prc.ast[paramsPos].last contains the type we're after:
@@ -645,11 +675,18 @@ proc closureSetup(p: BProc, prc: PSym) =
     internalError(prc.info, "closure generation failed")
   var env = ls.sym
   #echo "created environment: ", env.id, " for ", prc.name.s
-  assignLocalVar(p, env)
+  assignLocalVar(p, ls)
   # generate cast assignment:
   linefmt(p, cpsStmts, "$1 = ($2) ClE_0;$n",
           rdLoc(env.loc), getTypeDesc(p.module, env.typ))
 
+proc containsResult(n: PNode): bool =
+  if n.kind == nkSym and n.sym.kind == skResult:
+    result = true
+  else:
+    for i in 0..<n.safeLen:
+      if containsResult(n[i]): return true
+
 proc easyResultAsgn(n: PNode): PNode =
   const harmless = {nkConstSection, nkTypeSection, nkEmpty, nkCommentStmt} +
                     declarativeDefs
@@ -659,7 +696,7 @@ proc easyResultAsgn(n: PNode): PNode =
     while i < n.len and n[i].kind in harmless: inc i
     if i < n.len: result = easyResultAsgn(n[i])
   of nkAsgn, nkFastAsgn:
-    if n[0].kind == nkSym and skResult == n[0].sym.kind:
+    if n[0].kind == nkSym and n[0].sym.kind == skResult and not containsResult(n[1]):
       incl n.flags, nfPreventCg
       return n[1]
   of nkReturnStmt:
@@ -676,29 +713,31 @@ proc genProcAux(m: BModule, prc: PSym) =
   if sfPure notin prc.flags and prc.typ.sons[0] != nil:
     if resultPos >= prc.ast.len:
       internalError(prc.info, "proc has no result symbol")
-    var res = prc.ast.sons[resultPos].sym # get result symbol
+    let resNode = prc.ast.sons[resultPos]
+    let res = resNode.sym # get result symbol
     if not isInvalidReturnType(prc.typ.sons[0]):
       if sfNoInit in prc.flags: incl(res.flags, sfNoInit)
       if sfNoInit in prc.flags and p.module.compileToCpp and (let val = easyResultAsgn(prc.getBody); val != nil):
-        var decl = localVarDecl(p, res)
+        var decl = localVarDecl(p, resNode)
         var a: TLoc
         initLocExprSingleUse(p, val, a)
         linefmt(p, cpsStmts, "$1 = $2;$n", decl, rdLoc(a))
       else:
         # declare the result symbol:
-        assignLocalVar(p, res)
+        assignLocalVar(p, resNode)
         assert(res.loc.r != nil)
         initLocalVar(p, res, immediateAsgn=false)
       returnStmt = rfmt(nil, "\treturn $1;$n", rdLoc(res.loc))
     else:
-      fillResult(res)
+      fillResult(resNode)
       assignParam(p, res)
+      resetLoc(p, res.loc)
       if skipTypes(res.typ, abstractInst).kind == tyArray:
         #incl(res.loc.flags, lfIndirect)
-        res.loc.s = OnUnknown
+        res.loc.storage = OnUnknown
 
   for i in countup(1, sonsLen(prc.typ.n) - 1):
-    var param = prc.typ.n.sons[i].sym
+    let param = prc.typ.n.sons[i].sym
     if param.typ.isCompileTimeOnly: continue
     assignParam(p, param)
   closureSetup(p, prc)
@@ -764,13 +803,13 @@ proc genProcPrototype(m: BModule, sym: PSym) =
 
 proc genProcNoForward(m: BModule, prc: PSym) =
   if lfImportCompilerProc in prc.loc.flags:
-    fillProcLoc(m, prc)
+    fillProcLoc(m, prc.ast[namePos])
     useHeader(m, prc)
     # dependency to a compilerproc:
     discard cgsym(m, prc.name.s)
     return
   if lfNoDecl in prc.loc.flags:
-    fillProcLoc(m, prc)
+    fillProcLoc(m, prc.ast[namePos])
     useHeader(m, prc)
     genProcPrototype(m, prc)
   elif prc.typ.callConv == ccInline:
@@ -779,7 +818,7 @@ proc genProcNoForward(m: BModule, prc: PSym) =
     # a check for ``m.declaredThings``.
     if not containsOrIncl(m.declaredThings, prc.id):
       #if prc.loc.k == locNone:
-      fillProcLoc(m, prc)
+      fillProcLoc(m, prc.ast[namePos])
       #elif {sfExportc, sfImportc} * prc.flags == {}:
       #  # reset name to restore consistency in case of hashing collisions:
       #  echo "resetting ", prc.id, " by ", m.module.name.s
@@ -790,7 +829,7 @@ proc genProcNoForward(m: BModule, prc: PSym) =
       genProcAux(m, prc)
   elif lfDynamicLib in prc.loc.flags:
     var q = findPendingModule(m, prc)
-    fillProcLoc(q, prc)
+    fillProcLoc(q, prc.ast[namePos])
     useHeader(m, prc)
     genProcPrototype(m, prc)
     if q != nil and not containsOrIncl(q.declaredThings, prc.id):
@@ -799,13 +838,13 @@ proc genProcNoForward(m: BModule, prc: PSym) =
       symInDynamicLibPartial(m, prc)
   elif sfImportc notin prc.flags:
     var q = findPendingModule(m, prc)
-    fillProcLoc(q, prc)
+    fillProcLoc(q, prc.ast[namePos])
     useHeader(m, prc)
     genProcPrototype(m, prc)
     if q != nil and not containsOrIncl(q.declaredThings, prc.id):
       genProcAux(q, prc)
   else:
-    fillProcLoc(m, prc)
+    fillProcLoc(m, prc.ast[namePos])
     useHeader(m, prc)
     if sfInfixCall notin prc.flags: genProcPrototype(m, prc)
 
@@ -813,7 +852,7 @@ proc requestConstImpl(p: BProc, sym: PSym) =
   var m = p.module
   useHeader(m, sym)
   if sym.loc.k == locNone:
-    fillLoc(sym.loc, locData, sym.typ, mangleName(p.module, sym), OnStatic)
+    fillLoc(sym.loc, locData, sym.ast, mangleName(p.module, sym), OnStatic)
   if lfNoDecl in sym.loc.flags: return
   # declare implementation:
   var q = findPendingModule(m, sym)
@@ -836,7 +875,7 @@ proc genProc(m: BModule, prc: PSym) =
   if sfBorrow in prc.flags or not isActivated(prc): return
   if sfForward in prc.flags:
     addForwardedProc(m, prc)
-    fillProcLoc(m, prc)
+    fillProcLoc(m, prc.ast[namePos])
   else:
     genProcNoForward(m, prc)
     if {sfExportc, sfCompilerProc} * prc.flags == {sfExportc} and
@@ -846,10 +885,11 @@ proc genProc(m: BModule, prc: PSym) =
         if not containsOrIncl(m.g.generatedHeader.declaredThings, prc.id):
           genProcAux(m.g.generatedHeader, prc)
 
-proc genVarPrototypeAux(m: BModule, sym: PSym) =
+proc genVarPrototypeAux(m: BModule, n: PNode) =
   #assert(sfGlobal in sym.flags)
+  let sym = n.sym
   useHeader(m, sym)
-  fillLoc(sym.loc, locGlobalVar, sym.typ, mangleName(m, sym), OnHeap)
+  fillLoc(sym.loc, locGlobalVar, n, mangleName(m, sym), OnHeap)
   if (lfNoDecl in sym.loc.flags) or containsOrIncl(m.declaredThings, sym.id):
     return
   if sym.owner.id != m.module.id:
@@ -865,8 +905,8 @@ proc genVarPrototypeAux(m: BModule, sym: PSym) =
       if sfVolatile in sym.flags: add(m.s[cfsVars], " volatile")
       addf(m.s[cfsVars], " $1;$n", [sym.loc.r])
 
-proc genVarPrototype(m: BModule, sym: PSym) =
-  genVarPrototypeAux(m, sym)
+proc genVarPrototype(m: BModule, n: PNode) =
+  genVarPrototypeAux(m, n)
 
 proc addIntTypes(result: var Rope) {.inline.} =
   addf(result, "#define NIM_NEW_MANGLING_RULES" & tnl &
@@ -876,12 +916,12 @@ proc addIntTypes(result: var Rope) {.inline.} =
 proc getCopyright(cfile: Cfile): Rope =
   if optCompileOnly in gGlobalOptions:
     result = ("/* Generated by Nim Compiler v$1 */$N" &
-        "/*   (c) " & CompileDate.substr(0, 3) & " Andreas Rumpf */$N" &
+        "/*   (c) " & copyrightYear & " Andreas Rumpf */$N" &
         "/* The generated code is subject to the original license. */$N") %
         [rope(VersionAsString)]
   else:
     result = ("/* Generated by Nim Compiler v$1 */$N" &
-        "/*   (c) " & CompileDate.substr(0, 3) & " Andreas Rumpf */$N" &
+        "/*   (c) " & copyrightYear & " Andreas Rumpf */$N" &
         "/* The generated code is subject to the original license. */$N" &
         "/* Compiled for: $2, $3, $4 */$N" &
         "/* Command for C compiler:$n   $5 */$N") %
@@ -898,7 +938,7 @@ proc getFileHeader(cfile: Cfile): Rope =
 proc genFilenames(m: BModule): Rope =
   discard cgsym(m, "dbgRegisterFilename")
   result = nil
-  for i in 0.. <fileInfos.len:
+  for i in 0..<fileInfos.len:
     result.addf("dbgRegisterFilename($1);$N", [fileInfos[i].projPath.makeCString])
 
 proc genMainProc(m: BModule) =
@@ -991,10 +1031,9 @@ proc genMainProc(m: BModule) =
     ComponentConstruct =
       "void Libc::Component::construct(Libc::Env &env) {$N" &
       "\tgenodeEnv = &env;$N" &
-      "\tLibc::with_libc([&] () {$n\t" &
+      "\tLibc::with_libc([&] () {$N\t" &
       MainProcs &
       "\t});$N" &
-      "\tenv.parent().exit(0);$N" &
       "}$N$N"
 
   var nimMain, otherMain: FormatStr
@@ -1132,7 +1171,7 @@ proc genInitCode(m: BModule) =
 
   for i, el in pairs(m.extensionLoaders):
     if el != nil:
-      let ex = "N_NIMCALL(void, nimLoadProcs$1)(void) {$2}$N$N" %
+      let ex = "NIM_EXTERNC N_NIMCALL(void, nimLoadProcs$1)(void) {$2}$N$N" %
         [(i.ord - '0'.ord).rope, el]
       add(m.s[cfsInitProc], ex)
 
@@ -1224,7 +1263,7 @@ proc resetModule*(m: BModule) =
 
   # indicate that this is now cached module
   # the cache will be invalidated by nullifying gModules
-  m.fromCache = true
+  #m.fromCache = true
   m.g = nil
 
   # we keep only the "merge info" information for the module
@@ -1271,7 +1310,7 @@ proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
 
 proc writeHeader(m: BModule) =
   var result = ("/* Generated by Nim Compiler v$1 */$N" &
-        "/*   (c) " & CompileDate.substr(0, 3) & " Andreas Rumpf */$N" &
+        "/*   (c) 2017 Andreas Rumpf */$N" &
         "/* The generated code is subject to the original license. */$N") %
         [rope(VersionAsString)]
 
@@ -1302,7 +1341,6 @@ proc getCFile(m: BModule): string =
 
 proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext =
   injectG(graph.config)
-  assert optSymbolFiles in gGlobalOptions
   var m = newModule(g, module)
   readMergeInfo(getCFile(m), m)
   result = m
@@ -1356,7 +1394,7 @@ proc writeModule(m: BModule, pending: bool) =
   # generate code for the init statements of the module:
   let cfile = getCFile(m)
 
-  if not m.fromCache or optForceFullMake in gGlobalOptions:
+  if m.rd == nil or optForceFullMake in gGlobalOptions:
     genInitCode(m)
     finishTypeDescriptions(m)
     if sfMainModule in m.module.flags:
@@ -1409,6 +1447,10 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
   result = n
   if b == nil or passes.skipCodegen(n): return
   var m = BModule(b)
+  # if the module is cached, we don't regenerate the main proc
+  # nor the dispatchers? But if the dispatchers changed?
+  # XXX emit the dispatchers into its own .c file?
+  if b.rd != nil: return
   if n != nil:
     m.initProc.options = initProcOptions(m)
     genStmts(m.initProc, n)
@@ -1431,10 +1473,10 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) =
   if g.generatedHeader != nil: finishModule(g.generatedHeader)
   while g.forwardedProcsCounter > 0:
     for m in cgenModules(g):
-      if not m.fromCache:
+      if m.rd == nil:
         finishModule(m)
   for m in cgenModules(g):
-    if m.fromCache:
+    if m.rd != nil:
       m.updateCachedModule
     else:
       m.writeModule(pending=true)

compiler/cgendata.nim

@@ -54,7 +54,7 @@ type
   TCProcSections* = array[TCProcSection, Rope] # represents a generated C proc
   BModule* = ref TCGen
   BProc* = ref TCProc
-  TBlock*{.final.} = object
+  TBlock* = object
     id*: int                  # the ID of the label; positive means that it
     label*: Rope              # generated text for the label
                               # nil if label is not used
@@ -64,7 +64,7 @@ type
     nestedExceptStmts*: int16 # how many except statements is it nested into
     frameLen*: int16
 
-  TCProc{.final.} = object    # represents C proc that is currently generated
+  TCProc = object             # represents C proc that is currently generated
     prc*: PSym                # the Nim proc that this C proc belongs to
     beforeRetNeeded*: bool    # true iff 'BeforeRet' label for proc is needed
     threadVarAccessed*: bool  # true if the proc already accessed some threadvar
@@ -154,7 +154,7 @@ proc includeHeader*(this: BModule; header: string) =
 
 proc s*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # section in the current block
-  result = p.blocks[^1].sections[s]
+  result = p.blocks[p.blocks.len-1].sections[s]
 
 proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # top level proc sections

compiler/cgmeth.nim

@@ -68,7 +68,7 @@ proc sameMethodBucket(a, b: PSym): MethodResult =
     while true:
       aa = skipTypes(aa, {tyGenericInst, tyAlias})
       bb = skipTypes(bb, {tyGenericInst, tyAlias})
-      if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef}:
+      if aa.kind == bb.kind and aa.kind in {tyVar, tyPtr, tyRef, tyLent}:
         aa = aa.lastSon
         bb = bb.lastSon
       else:

compiler/commands.nim

@@ -53,8 +53,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
 # implementation
 
 const
-  HelpMessage = "Nim Compiler Version $1 (" & CompileDate & ") [$2: $3]\n" &
-      "Copyright (c) 2006-" & CompileDate.substr(0, 3) & " by Andreas Rumpf\n"
+  HelpMessage = "Nim Compiler Version $1 [$2: $3]\n" &
+      "Copyright (c) 2006-" & copyrightYear & " by Andreas Rumpf\n"
 
 const
   Usage = slurp"../doc/basicopt.txt".replace("//", "")
@@ -81,15 +81,17 @@ proc writeVersionInfo(pass: TCmdLinePass) =
   if pass == passCmd1:
     msgWriteln(`%`(HelpMessage, [VersionAsString,
                                  platform.OS[platform.hostOS].name,
-                                 CPU[platform.hostCPU].name]))
+                                 CPU[platform.hostCPU].name]),
+               {msgStdout})
 
     const gitHash = gorge("git log -n 1 --format=%H").strip
     when gitHash.len == 40:
-      msgWriteln("git hash: " & gitHash)
+      msgWriteln("git hash: " & gitHash, {msgStdout})
 
     msgWriteln("active boot switches:" & usedRelease & usedAvoidTimeMachine &
       usedTinyC & usedGnuReadline & usedNativeStacktrace & usedNoCaas &
-      usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC)
+      usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC,
+               {msgStdout})
     msgQuit(0)
 
 var
@@ -249,6 +251,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool =
     result = gOptions * {optNaNCheck, optInfCheck} == {optNaNCheck, optInfCheck}
   of "infchecks": result = contains(gOptions, optInfCheck)
   of "nanchecks": result = contains(gOptions, optNaNCheck)
+  of "nilchecks": result = contains(gOptions, optNilCheck)
   of "objchecks": result = contains(gOptions, optObjCheck)
   of "fieldchecks": result = contains(gOptions, optFieldCheck)
   of "rangechecks": result = contains(gOptions, optRangeCheck)
@@ -258,7 +261,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool =
   of "assertions", "a": result = contains(gOptions, optAssert)
   of "deadcodeelim": result = contains(gGlobalOptions, optDeadCodeElim)
   of "run", "r": result = contains(gGlobalOptions, optRun)
-  of "symbolfiles": result = contains(gGlobalOptions, optSymbolFiles)
+  of "symbolfiles": result = gSymbolFiles != disabledSf
   of "genscript": result = contains(gGlobalOptions, optGenScript)
   of "threads": result = contains(gGlobalOptions, optThreads)
   of "taintmode": result = contains(gGlobalOptions, optTaintMode)
@@ -271,9 +274,10 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool =
 
 proc processPath(path: string, info: TLineInfo,
                  notRelativeToProj = false): string =
-  let p = if notRelativeToProj or os.isAbsolute(path) or
-              '$' in path:
+  let p = if os.isAbsolute(path) or '$' in path:
             path
+          elif notRelativeToProj:
+            getCurrentDir() / path
           else:
             options.gProjectPath / path
   try:
@@ -340,7 +344,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     # keep the old name for compat
     if pass in {passCmd2, passPP} and not options.gNoNimblePath:
       expectArg(switch, arg, pass, info)
-      let path = processPath(arg, info, notRelativeToProj=true)
+      var path = processPath(arg, info, notRelativeToProj=true)
+      let nimbleDir = getEnv("NIMBLE_DIR")
+      if nimbleDir.len > 0 and pass == passPP: path = nimbleDir / "pkgs"
       nimblePath(path, info)
   of "nonimblepath", "nobabelpath":
     expectNoArg(switch, arg, pass, info)
@@ -466,11 +472,22 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitch({optMemTracker}, arg, pass, info)
     if optMemTracker in gOptions: defineSymbol("memtracker")
     else: undefSymbol("memtracker")
+  of "oldnewlines":
+    case arg.normalize
+    of "on":
+      options.gOldNewlines = true
+      defineSymbol("nimOldNewlines")
+    of "off":
+      options.gOldNewlines = false
+      undefSymbol("nimOldNewlines")
+    else:
+      localError(info, errOnOrOffExpectedButXFound, arg)
   of "checks", "x": processOnOffSwitch(ChecksOptions, arg, pass, info)
   of "floatchecks":
     processOnOffSwitch({optNaNCheck, optInfCheck}, arg, pass, info)
   of "infchecks": processOnOffSwitch({optInfCheck}, arg, pass, info)
   of "nanchecks": processOnOffSwitch({optNaNCheck}, arg, pass, info)
+  of "nilchecks": processOnOffSwitch({optNilCheck}, arg, pass, info)
   of "objchecks": processOnOffSwitch({optObjCheck}, arg, pass, info)
   of "fieldchecks": processOnOffSwitch({optFieldCheck}, arg, pass, info)
   of "rangechecks": processOnOffSwitch({optRangeCheck}, arg, pass, info)
@@ -592,7 +609,13 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectNoArg(switch, arg, pass, info)
     helpOnError(pass)
   of "symbolfiles":
-    processOnOffSwitchG({optSymbolFiles}, arg, pass, info)
+    case arg.normalize
+    of "on": gSymbolFiles = enabledSf
+    of "off": gSymbolFiles = disabledSf
+    of "writeonly": gSymbolFiles = writeOnlySf
+    of "readonly": gSymbolFiles = readOnlySf
+    of "v2": gSymbolFiles = v2Sf
+    else: localError(info, errOnOrOffExpectedButXFound, arg)
   of "skipcfg":
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optSkipConfigFile)
@@ -605,7 +628,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "skipparentcfg":
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optSkipParentConfigFiles)
-  of "genscript":
+  of "genscript", "gendeps":
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optGenScript)
   of "colors": processOnOffSwitchG({optUseColors}, arg, pass, info)
@@ -648,6 +671,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     gListFullPaths = true
   of "dynliboverride":
     dynlibOverride(switch, arg, pass, info)
+  of "dynliboverrideall":
+    expectNoArg(switch, arg, pass, info)
+    gDynlibOverrideAll = true
   of "cs":
     # only supported for compatibility. Does nothing.
     expectArg(switch, arg, pass, info)
@@ -662,6 +688,10 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectArg(switch, arg, pass, info)
     if config != nil:
       config.cppDefine(arg)
+  of "newruntime":
+    expectNoArg(switch, arg, pass, info)
+    newDestructors = true
+    defineSymbol("nimNewRuntime")
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(switch, arg)
     else: invalidCmdLineOption(pass, switch, info)

compiler/condsyms.nim

@@ -107,3 +107,9 @@ proc initDefines*() =
   defineSymbol("nimDistros")
   defineSymbol("nimHasCppDefine")
   defineSymbol("nimGenericInOutFlags")
+  when false: defineSymbol("nimHasOpt")
+  defineSymbol("nimNoArrayToCstringConversion")
+  defineSymbol("nimNewRoof")
+  defineSymbol("nimHasRunnableExamples")
+  defineSymbol("nimNewDot")
+  defineSymbol("nimHasNilChecks")

compiler/depends.nim

@@ -10,7 +10,7 @@
 # This module implements a dependency file generator.
 
 import
-  os, options, ast, astalgo, msgs, ropes, idents, passes, importer
+  os, options, ast, astalgo, msgs, ropes, idents, passes, modulepaths
 
 from modulegraphs import ModuleGraph
 

compiler/destroyer.nim

@@ -0,0 +1,329 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Injects destructor calls into Nim code as well as
+## an optimizer that optimizes copies to moves. This is implemented as an
+## AST to AST transformation so that every backend benefits from it.
+
+## Rules for destructor injections:
+##
+## foo(bar(X(), Y()))
+## X and Y get destroyed after bar completes:
+##
+## foo( (tmpX = X(); tmpY = Y(); tmpBar = bar(tmpX, tmpY);
+##       destroy(tmpX); destroy(tmpY);
+##       tmpBar))
+## destroy(tmpBar)
+##
+## var x = f()
+## body
+##
+## is the same as:
+##
+##  var x;
+##  try:
+##    move(x, f())
+##  finally:
+##    destroy(x)
+##
+## But this really just an optimization that tries to avoid to
+## introduce too many temporaries, the 'destroy' is caused by
+## the 'f()' call. No! That is not true for 'result = f()'!
+##
+## x = y where y is read only once
+## is the same as:  move(x, y)
+##
+## Actually the more general rule is: The *last* read of ``y``
+## can become a move if ``y`` is the result of a construction.
+##
+## We also need to keep in mind here that the number of reads is
+## control flow dependent:
+## let x = foo()
+## while true:
+##   y = x  # only one read, but the 2nd iteration will fail!
+## This also affects recursions! Only usages that do not cross
+## a loop boundary (scope) and are not used in function calls
+## are safe.
+##
+##
+## x = f() is the same as:  move(x, f())
+##
+## x = y
+## is the same as:  copy(x, y)
+##
+## Reassignment works under this scheme:
+## var x = f()
+## x = y
+##
+## is the same as:
+##
+##  var x;
+##  try:
+##    move(x, f())
+##    copy(x, y)
+##  finally:
+##    destroy(x)
+##
+##  result = f()  must not destroy 'result'!
+##
+## The produced temporaries clutter up the code and might lead to
+## inefficiencies. A better strategy is to collect all the temporaries
+## in a single object that we put into a single try-finally that
+## surrounds the proc body. This means the code stays quite efficient
+## when compiled to C. In fact, we do the same for variables, so
+## destructors are called when the proc returns, not at scope exit!
+## This makes certains idioms easier to support. (Taking the slice
+## of a temporary object.)
+##
+## foo(bar(X(), Y()))
+## X and Y get destroyed after bar completes:
+##
+## var tmp: object
+## foo( (move tmp.x, X(); move tmp.y, Y(); tmp.bar = bar(tmpX, tmpY);
+##       tmp.bar))
+## destroy(tmp.bar)
+## destroy(tmp.x); destroy(tmp.y)
+
+
+import
+  intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
+  strutils, options, dfa, lowerings, rodread
+
+const
+  InterestingSyms = {skVar, skResult, skLet}
+
+type
+  Con = object
+    owner: PSym
+    g: ControlFlowGraph
+    jumpTargets: IntSet
+    tmpObj: PType
+    tmp: PSym
+    destroys, topLevelVars: PNode
+
+proc isHarmlessVar*(s: PSym; c: Con): bool =
+  # 's' is harmless if it used only once and its
+  # definition/usage are not split by any labels:
+  #
+  # let s = foo()
+  # while true:
+  #   a[i] = s
+  #
+  # produces:
+  #
+  # def s
+  # L1:
+  #   use s
+  # goto L1
+  #
+  # let s = foo()
+  # if cond:
+  #   a[i] = s
+  # else:
+  #   a[j] = s
+  #
+  # produces:
+  #
+  # def s
+  # fork L2
+  # use s
+  # goto L3
+  # L2:
+  # use s
+  # L3
+  #
+  # So this analysis is for now overly conservative, but correct.
+  var defsite = -1
+  var usages = 0
+  for i in 0..<c.g.len:
+    case c.g[i].kind
+    of def:
+      if c.g[i].sym == s:
+        if defsite < 0: defsite = i
+        else: return false
+    of use:
+      if c.g[i].sym == s:
+        if defsite < 0: return false
+        for j in defsite .. i:
+          # not within the same basic block?
+          if j in c.jumpTargets: return false
+        # if we want to die after the first 'use':
+        if usages > 1: return false
+        inc usages
+    of useWithinCall:
+      if c.g[i].sym == s: return false
+    of goto, fork:
+      discard "we do not perform an abstract interpretation yet"
+
+template interestingSym(s: PSym): bool =
+  s.owner == c.owner and s.kind in InterestingSyms and hasDestructor(s.typ)
+
+proc patchHead(n: PNode) =
+  if n.kind in nkCallKinds and n[0].kind == nkSym and n.len > 1:
+    let s = n[0].sym
+    if s.name.s[0] == '=' and s.name.s in ["=sink", "=", "=destroy"]:
+      if sfFromGeneric in s.flags:
+        excl(s.flags, sfFromGeneric)
+        patchHead(s.getBody)
+      if n[1].typ.isNil:
+        # XXX toptree crashes without this workaround. Figure out why.
+        return
+      let t = n[1].typ.skipTypes({tyVar, tyLent, tyGenericInst, tyAlias, tySink, tyInferred})
+      template patch(op, field) =
+        if s.name.s == op and field != nil and field != s:
+          n.sons[0].sym = field
+      patch "=sink", t.sink
+      patch "=", t.assignment
+      patch "=destroy", t.destructor
+  for x in n:
+    patchHead(x)
+
+proc patchHead(s: PSym) =
+  if sfFromGeneric in s.flags:
+    patchHead(s.ast[bodyPos])
+
+template genOp(opr, opname) =
+  let op = opr
+  if op == nil:
+    globalError(dest.info, "internal error: '" & opname & "' operator not found for type " & typeToString(t))
+  elif op.ast[genericParamsPos].kind != nkEmpty:
+    globalError(dest.info, "internal error: '" & opname & "' operator is generic")
+  patchHead op
+  result = newTree(nkCall, newSymNode(op), newTree(nkHiddenAddr, dest))
+
+proc genSink(t: PType; dest: PNode): PNode =
+  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
+  genOp(if t.sink != nil: t.sink else: t.assignment, "=sink")
+
+proc genCopy(t: PType; dest: PNode): PNode =
+  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
+  genOp(t.assignment, "=")
+
+proc genDestroy(t: PType; dest: PNode): PNode =
+  let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
+  genOp(t.destructor, "=destroy")
+
+proc addTopVar(c: var Con; v: PNode) =
+  c.topLevelVars.add newTree(nkIdentDefs, v, emptyNode, emptyNode)
+
+proc p(n: PNode; c: var Con): PNode
+
+template recurse(n, dest) =
+  for i in 0..<n.len:
+    dest.add p(n[i], c)
+
+proc moveOrCopy(dest, ri: PNode; c: var Con): PNode =
+  if ri.kind in nkCallKinds+{nkObjConstr}:
+    result = genSink(ri.typ, dest)
+    # watch out and no not transform 'ri' twice if it's a call:
+    let ri2 = copyNode(ri)
+    recurse(ri, ri2)
+    result.add ri2
+  elif ri.kind == nkSym and isHarmlessVar(ri.sym, c):
+    result = genSink(ri.typ, dest)
+    result.add p(ri, c)
+  else:
+    result = genCopy(ri.typ, dest)
+    result.add p(ri, c)
+
+proc p(n: PNode; c: var Con): PNode =
+  case n.kind
+  of nkVarSection, nkLetSection:
+    discard "transform; var x = y to  var x; x op y  where op is a move or copy"
+    result = newNodeI(nkStmtList, n.info)
+
+    for i in 0..<n.len:
+      let it = n[i]
+      let L = it.len-1
+      let ri = it[L]
+      if it.kind == nkVarTuple and hasDestructor(ri.typ):
+        let x = lowerTupleUnpacking(it, c.owner)
+        result.add p(x, c)
+      elif it.kind == nkIdentDefs and hasDestructor(it[0].typ):
+        for j in 0..L-2:
+          let v = it[j]
+          doAssert v.kind == nkSym
+          # move the variable declaration to the top of the frame:
+          c.addTopVar v
+          # make sure it's destroyed at the end of the proc:
+          c.destroys.add genDestroy(v.typ, v)
+          if ri.kind != nkEmpty:
+            let r = moveOrCopy(v, ri, c)
+            result.add r
+      else:
+        # keep it, but transform 'ri':
+        var varSection = copyNode(n)
+        var itCopy = copyNode(it)
+        for j in 0..L-1:
+          itCopy.add it[j]
+        itCopy.add p(ri, c)
+        varSection.add itCopy
+        result.add varSection
+  of nkCallKinds:
+    if n.typ != nil and hasDestructor(n.typ):
+      discard "produce temp creation"
+      result = newNodeIT(nkStmtListExpr, n.info, n.typ)
+      let f = newSym(skField, getIdent(":d" & $c.tmpObj.n.len), c.owner, n.info)
+      f.typ = n.typ
+      rawAddField c.tmpObj, f
+      var m = genSink(n.typ, rawDirectAccess(c.tmp, f))
+      var call = copyNode(n)
+      recurse(n, call)
+      m.add call
+      result.add m
+      result.add rawDirectAccess(c.tmp, f)
+      c.destroys.add genDestroy(n.typ, rawDirectAccess(c.tmp, f))
+    else:
+      result = copyNode(n)
+      recurse(n, result)
+  of nkAsgn, nkFastAsgn:
+    if hasDestructor(n[0].typ):
+      result = moveOrCopy(n[0], n[1], c)
+    else:
+      result = copyNode(n)
+      recurse(n, result)
+  of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef,
+      nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef:
+    result = n
+  else:
+    result = copyNode(n)
+    recurse(n, result)
+
+proc injectDestructorCalls*(owner: PSym; n: PNode): PNode =
+  when defined(nimDebugDestroys):
+    echo "injecting into ", n
+  var c: Con
+  c.owner = owner
+  c.tmp = newSym(skTemp, getIdent":d", owner, n.info)
+  c.tmpObj = createObj(owner, n.info)
+  c.tmp.typ = c.tmpObj
+  c.destroys = newNodeI(nkStmtList, n.info)
+  c.topLevelVars = newNodeI(nkVarSection, n.info)
+  let cfg = constructCfg(owner, n)
+  shallowCopy(c.g, cfg)
+  c.jumpTargets = initIntSet()
+  for i in 0..<c.g.len:
+    if c.g[i].kind in {goto, fork}:
+      c.jumpTargets.incl(i+c.g[i].dest)
+  let body = p(n, c)
+  if c.tmp.typ.n.len > 0:
+    c.addTopVar(newSymNode c.tmp)
+  result = newNodeI(nkStmtList, n.info)
+  if c.topLevelVars.len > 0:
+    result.add c.topLevelVars
+  if c.destroys.len > 0:
+    result.add newTryFinally(body, c.destroys)
+  else:
+    result.add body
+
+  when defined(nimDebugDestroys):
+    if owner.name.s == "main" or true:
+      echo "------------------------------------"
+      echo owner.name.s, " transformed to: "
+      echo result

compiler/dfa.nim

@@ -0,0 +1,444 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Data flow analysis for Nim. For now the task is to prove that every
+## usage of a local variable 'v' is covered by an initialization to 'v'
+## first.
+## We transform the AST into a linear list of instructions first to
+## make this easier to handle: There are only 2 different branching
+## instructions: 'goto X' is an unconditional goto, 'fork X'
+## is a conditional goto (either the next instruction or 'X' can be
+## taken). Exhaustive case statements are translated
+## so that the last branch is transformed into an 'else' branch.
+## ``return`` and ``break`` are all covered by 'goto'.
+## The case to detect is ``use v`` that is not dominated by
+## a ``def v``.
+## The data structures and algorithms used here are inspired by
+## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
+## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
+
+import ast, astalgo, types, intsets, tables, msgs
+
+type
+  InstrKind* = enum
+    goto, fork, def, use,
+    useWithinCall # this strange special case is used to get more
+                  # move optimizations out of regular code
+                  # XXX This is still overly pessimistic in
+                  # call(let x = foo; bar(x))
+  Instr* = object
+    n*: PNode
+    case kind*: InstrKind
+    of def, use, useWithinCall: sym*: PSym
+    of goto, fork: dest*: int
+
+  ControlFlowGraph* = seq[Instr]
+
+  TPosition = distinct int
+  TBlock = object
+    label: PSym
+    fixups: seq[TPosition]
+
+  ValueKind = enum
+    undef, value, valueOrUndef
+
+  Con = object
+    code: ControlFlowGraph
+    inCall: int
+    blocks: seq[TBlock]
+
+proc debugInfo(info: TLineInfo): string =
+  result = info.toFilename & ":" & $info.line
+
+proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
+  # for debugging purposes
+  # first iteration: compute all necessary labels:
+  var jumpTargets = initIntSet()
+  let last = if last < 0: c.len-1 else: min(last, c.len-1)
+  for i in start..last:
+    if c[i].kind in {goto, fork}:
+      jumpTargets.incl(i+c[i].dest)
+  var i = start
+  while i <= last:
+    if i in jumpTargets: result.add("L" & $i & ":\n")
+    result.add "\t"
+    result.add $c[i].kind
+    result.add "\t"
+    case c[i].kind
+    of def, use, useWithinCall:
+      result.add c[i].sym.name.s
+    of goto, fork:
+      result.add "L"
+      result.add c[i].dest+i
+    result.add("\t#")
+    result.add(debugInfo(c[i].n.info))
+    result.add("\n")
+    inc i
+  if i in jumpTargets: result.add("L" & $i & ": End\n")
+
+
+proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
+  ## echos the ControlFlowGraph for debugging purposes.
+  var buf = ""
+  codeListing(c, buf, start, last)
+  echo buf
+
+proc forkI(c: var Con; n: PNode): TPosition =
+  result = TPosition(c.code.len)
+  c.code.add Instr(n: n, kind: fork, dest: 0)
+
+proc gotoI(c: var Con; n: PNode): TPosition =
+  result = TPosition(c.code.len)
+  c.code.add Instr(n: n, kind: goto, dest: 0)
+
+proc genLabel(c: Con): TPosition =
+  result = TPosition(c.code.len)
+
+proc jmpBack(c: var Con, n: PNode, p = TPosition(0)) =
+  let dist = p.int - c.code.len
+  internalAssert(-0x7fff < dist and dist < 0x7fff)
+  c.code.add Instr(n: n, kind: goto, dest: dist)
+
+proc patch(c: var Con, p: TPosition) =
+  # patch with current index
+  let p = p.int
+  let diff = c.code.len - p
+  internalAssert(-0x7fff < diff and diff < 0x7fff)
+  c.code[p].dest = diff
+
+proc popBlock(c: var Con; oldLen: int) =
+  for f in c.blocks[oldLen].fixups:
+    c.patch(f)
+  c.blocks.setLen(oldLen)
+
+template withBlock(labl: PSym; body: untyped) {.dirty.} =
+  var oldLen {.gensym.} = c.blocks.len
+  c.blocks.add TBlock(label: labl, fixups: @[])
+  body
+  popBlock(c, oldLen)
+
+proc isTrue(n: PNode): bool =
+  n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or
+    n.kind == nkIntLit and n.intVal != 0
+
+proc gen(c: var Con; n: PNode) # {.noSideEffect.}
+
+proc genWhile(c: var Con; n: PNode) =
+  # L1:
+  #   cond, tmp
+  #   fork tmp, L2
+  #   body
+  #   jmp L1
+  # L2:
+  let L1 = c.genLabel
+  withBlock(nil):
+    if isTrue(n.sons[0]):
+      c.gen(n.sons[1])
+      c.jmpBack(n, L1)
+    else:
+      c.gen(n.sons[0])
+      let L2 = c.forkI(n)
+      c.gen(n.sons[1])
+      c.jmpBack(n, L1)
+      c.patch(L2)
+
+proc genBlock(c: var Con; n: PNode) =
+  withBlock(n.sons[0].sym):
+    c.gen(n.sons[1])
+
+proc genBreak(c: var Con; n: PNode) =
+  let L1 = c.gotoI(n)
+  if n.sons[0].kind == nkSym:
+    #echo cast[int](n.sons[0].sym)
+    for i in countdown(c.blocks.len-1, 0):
+      if c.blocks[i].label == n.sons[0].sym:
+        c.blocks[i].fixups.add L1
+        return
+    globalError(n.info, errGenerated, "VM problem: cannot find 'break' target")
+  else:
+    c.blocks[c.blocks.high].fixups.add L1
+
+proc genIf(c: var Con, n: PNode) =
+  var endings: seq[TPosition] = @[]
+  for i in countup(0, len(n) - 1):
+    var it = n.sons[i]
+    c.gen(it.sons[0])
+    if it.len == 2:
+      let elsePos = c.forkI(it.sons[1])
+      c.gen(it.sons[1])
+      if i < sonsLen(n)-1:
+        endings.add(c.gotoI(it.sons[1]))
+      c.patch(elsePos)
+  for endPos in endings: c.patch(endPos)
+
+proc genAndOr(c: var Con; n: PNode) =
+  #   asgn dest, a
+  #   fork L1
+  #   asgn dest, b
+  # L1:
+  c.gen(n.sons[1])
+  let L1 = c.forkI(n)
+  c.gen(n.sons[2])
+  c.patch(L1)
+
+proc genCase(c: var Con; n: PNode) =
+  #  if (!expr1) goto L1;
+  #    thenPart
+  #    goto LEnd
+  #  L1:
+  #  if (!expr2) goto L2;
+  #    thenPart2
+  #    goto LEnd
+  #  L2:
+  #    elsePart
+  #  Lend:
+  var endings: seq[TPosition] = @[]
+  c.gen(n.sons[0])
+  for i in 1 ..< n.len:
+    let it = n.sons[i]
+    if it.len == 1:
+      c.gen(it.sons[0])
+    else:
+      let elsePos = c.forkI(it.lastSon)
+      c.gen(it.lastSon)
+      if i < sonsLen(n)-1:
+        endings.add(c.gotoI(it.lastSon))
+      c.patch(elsePos)
+  for endPos in endings: c.patch(endPos)
+
+proc genTry(c: var Con; n: PNode) =
+  var endings: seq[TPosition] = @[]
+  let elsePos = c.forkI(n)
+  c.gen(n.sons[0])
+  c.patch(elsePos)
+  for i in 1 ..< n.len:
+    let it = n.sons[i]
+    if it.kind != nkFinally:
+      var blen = len(it)
+      let endExcept = c.forkI(it)
+      c.gen(it.lastSon)
+      if i < sonsLen(n)-1:
+        endings.add(c.gotoI(it))
+      c.patch(endExcept)
+  for endPos in endings: c.patch(endPos)
+  let fin = lastSon(n)
+  if fin.kind == nkFinally:
+    c.gen(fin.sons[0])
+
+proc genRaise(c: var Con; n: PNode) =
+  gen(c, n.sons[0])
+  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
+
+proc genReturn(c: var Con; n: PNode) =
+  if n.sons[0].kind != nkEmpty: gen(c, n.sons[0])
+  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
+
+const
+  InterestingSyms = {skVar, skResult, skLet}
+
+proc genUse(c: var Con; n: PNode) =
+  var n = n
+  while n.kind in {nkDotExpr, nkCheckedFieldExpr,
+                   nkBracketExpr, nkDerefExpr, nkHiddenDeref,
+                   nkAddr, nkHiddenAddr}:
+    n = n[0]
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    if c.inCall > 0:
+      c.code.add Instr(n: n, kind: useWithinCall, sym: n.sym)
+    else:
+      c.code.add Instr(n: n, kind: use, sym: n.sym)
+
+proc genDef(c: var Con; n: PNode) =
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    c.code.add Instr(n: n, kind: def, sym: n.sym)
+
+proc genCall(c: var Con; n: PNode) =
+  gen(c, n[0])
+  var t = n[0].typ
+  if t != nil: t = t.skipTypes(abstractInst)
+  inc c.inCall
+  for i in 1..<n.len:
+    gen(c, n[i])
+    if t != nil and i < t.len and t.sons[i].kind == tyVar:
+      genDef(c, n[i])
+  dec c.inCall
+
+proc genMagic(c: var Con; n: PNode; m: TMagic) =
+  case m
+  of mAnd, mOr: c.genAndOr(n)
+  of mNew, mNewFinalize:
+    genDef(c, n[1])
+    for i in 2..<n.len: gen(c, n[i])
+  of mExit:
+    genCall(c, n)
+    c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
+  else:
+    genCall(c, n)
+
+proc genVarSection(c: var Con; n: PNode) =
+  for a in n:
+    if a.kind == nkCommentStmt: continue
+    if a.kind == nkVarTuple:
+      gen(c, a.lastSon)
+      for i in 0 .. a.len-3: genDef(c, a[i])
+    else:
+      gen(c, a.lastSon)
+      if a.lastSon.kind != nkEmpty:
+        genDef(c, a.sons[0])
+
+proc gen(c: var Con; n: PNode) =
+  case n.kind
+  of nkSym: genUse(c, n)
+  of nkCallKinds:
+    if n.sons[0].kind == nkSym:
+      let s = n.sons[0].sym
+      if s.magic != mNone:
+        genMagic(c, n, s.magic)
+      else:
+        genCall(c, n)
+    else:
+      genCall(c, n)
+  of nkCharLit..nkNilLit: discard
+  of nkAsgn, nkFastAsgn:
+    gen(c, n[1])
+    genDef(c, n[0])
+  of nkDotExpr, nkCheckedFieldExpr, nkBracketExpr,
+     nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr:
+    gen(c, n[0])
+  of nkIfStmt, nkIfExpr: genIf(c, n)
+  of nkWhenStmt:
+    # This is "when nimvm" node. Chose the first branch.
+    gen(c, n.sons[0].sons[1])
+  of nkCaseStmt: genCase(c, n)
+  of nkWhileStmt: genWhile(c, n)
+  of nkBlockExpr, nkBlockStmt: genBlock(c, n)
+  of nkReturnStmt: genReturn(c, n)
+  of nkRaiseStmt: genRaise(c, n)
+  of nkBreakStmt: genBreak(c, n)
+  of nkTryStmt: genTry(c, n)
+  of nkStmtList, nkStmtListExpr, nkChckRangeF, nkChckRange64, nkChckRange,
+     nkBracket, nkCurly, nkPar, nkClosure, nkObjConstr:
+    for x in n: gen(c, x)
+  of nkPragmaBlock: gen(c, n.lastSon)
+  of nkDiscardStmt: gen(c, n.sons[0])
+  of nkHiddenStdConv, nkHiddenSubConv, nkConv, nkExprColonExpr, nkExprEqExpr,
+     nkCast:
+    gen(c, n.sons[1])
+  of nkObjDownConv, nkStringToCString, nkCStringToString: gen(c, n.sons[0])
+  of nkVarSection, nkLetSection: genVarSection(c, n)
+  else: discard
+
+proc dfa(code: seq[Instr]) =
+  var u = newSeq[IntSet](code.len) # usages
+  var d = newSeq[IntSet](code.len) # defs
+  var c = newSeq[IntSet](code.len) # consumed
+  var backrefs = initTable[int, int]()
+  for i in 0..<code.len:
+    u[i] = initIntSet()
+    d[i] = initIntSet()
+    c[i] = initIntSet()
+    case code[i].kind
+    of use, useWithinCall: u[i].incl(code[i].sym.id)
+    of def: d[i].incl(code[i].sym.id)
+    of fork, goto:
+      let d = i+code[i].dest
+      backrefs.add(d, i)
+
+  var w = @[0]
+  var maxIters = 50
+  var someChange = true
+  var takenGotos = initIntSet()
+  var consuming = -1
+  while w.len > 0 and maxIters > 0: # and someChange:
+    dec maxIters
+    var pc = w.pop() # w[^1]
+    var prevPc = -1
+    # this simulates a single linear control flow execution:
+    while pc < code.len:
+      if prevPc >= 0:
+        someChange = false
+        # merge step and test for changes (we compute the fixpoints here):
+        # 'u' needs to be the union of prevPc, pc
+        # 'd' needs to be the intersection of 'pc'
+        for id in u[prevPc]:
+          if not u[pc].containsOrIncl(id):
+            someChange = true
+        # in (a; b) if ``a`` sets ``v`` so does ``b``. The intersection
+        # is only interesting on merge points:
+        for id in d[prevPc]:
+          if not d[pc].containsOrIncl(id):
+            someChange = true
+        # if this is a merge point, we take the intersection of the 'd' sets:
+        if backrefs.hasKey(pc):
+          var intersect = initIntSet()
+          assign(intersect, d[pc])
+          var first = true
+          for prevPc in backrefs.allValues(pc):
+            for def in d[pc]:
+              if def notin d[prevPc]:
+                excl(intersect, def)
+                someChange = true
+                when defined(debugDfa):
+                  echo "Excluding ", pc, " prev ", prevPc
+          assign d[pc], intersect
+      if consuming >= 0:
+        if not c[pc].containsOrIncl(consuming):
+          someChange = true
+        consuming = -1
+
+      # our interpretation ![I!]:
+      prevPc = pc
+      case code[pc].kind
+      of goto:
+        # we must leave endless loops eventually:
+        if not takenGotos.containsOrIncl(pc) or someChange:
+          pc = pc + code[pc].dest
+        else:
+          inc pc
+      of fork:
+        # we follow the next instruction but push the dest onto our "work" stack:
+        #if someChange:
+        w.add pc + code[pc].dest
+        inc pc
+      of use, useWithinCall:
+        #if not d[prevPc].missingOrExcl():
+        # someChange = true
+        consuming = code[pc].sym.id
+        inc pc
+      of def:
+        if not d[pc].containsOrIncl(code[pc].sym.id):
+          someChange = true
+        inc pc
+
+  when defined(useDfa) and defined(debugDfa):
+    for i in 0..<code.len:
+      echo "PC ", i, ": defs: ", d[i], "; uses ", u[i], "; consumes ", c[i]
+
+  # now check the condition we're interested in:
+  for i in 0..<code.len:
+    case code[i].kind
+    of use, useWithinCall:
+      let s = code[i].sym
+      if s.id notin d[i]:
+        localError(code[i].n.info, "usage of uninitialized variable: " & s.name.s)
+      if s.id in c[i]:
+        localError(code[i].n.info, "usage of an already consumed variable: " & s.name.s)
+
+    else: discard
+
+proc dataflowAnalysis*(s: PSym; body: PNode) =
+  var c = Con(code: @[], blocks: @[])
+  gen(c, body)
+  when defined(useDfa) and defined(debugDfa): echoCfg(c.code)
+  dfa(c.code)
+
+proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
+  ## constructs a control flow graph for ``body``.
+  var c = Con(code: @[], blocks: @[])
+  shallowCopy(result, c.code)

compiler/docgen.nim

@@ -15,14 +15,13 @@ import
   ast, strutils, strtabs, options, msgs, os, ropes, idents,
   wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast,
   packages/docutils/rst, packages/docutils/rstgen, times,
-  packages/docutils/highlite, importer, sempass2, json, xmltree, cgi,
-  typesrenderer, astalgo
+  packages/docutils/highlite, sempass2, json, xmltree, cgi,
+  typesrenderer, astalgo, modulepaths
 
 type
   TSections = array[TSymKind, Rope]
   TDocumentor = object of rstgen.RstGenerator
     modDesc: Rope           # module description
-    id: int                  # for generating IDs
     toc, section: TSections
     indexValFilename: string
     analytics: string  # Google Analytics javascript, "" if doesn't exist
@@ -109,6 +108,8 @@ proc newDocumentor*(filename: string, config: StringTableRef): PDoc =
   result.id = 100
   result.jArray = newJArray()
   initStrTable result.types
+  result.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string; status: int; content: string) =
+    localError(newLineInfo(d.filename, -1, -1), warnUser, "only 'rst2html' supports the ':test:' attribute")
 
 proc dispA(dest: var Rope, xml, tex: string, args: openArray[Rope]) =
   if gCmd != cmdRst2tex: addf(dest, xml, args)
@@ -204,198 +205,15 @@ proc getPlainDocstring(n: PNode): string =
   if n.comment != nil and startsWith(n.comment, "##"):
     result = n.comment
   if result.len < 1:
-    if n.kind notin {nkEmpty..nkNilLit}:
-      for i in countup(0, len(n)-1):
-        result = getPlainDocstring(n.sons[i])
-        if result.len > 0: return
-
-when false:
-  proc findDocComment(n: PNode): PNode =
-    if n == nil: return nil
-    if not isNil(n.comment) and startsWith(n.comment, "##"): return n
     for i in countup(0, safeLen(n)-1):
-      result = findDocComment(n.sons[i])
-      if result != nil: return
+      result = getPlainDocstring(n.sons[i])
+      if result.len > 0: return
 
-  proc extractDocComment*(s: PSym, d: PDoc = nil): string =
-    let n = findDocComment(s.ast)
-    result = ""
-    if not n.isNil:
-      if not d.isNil:
-        var dummyHasToc: bool
-        renderRstToOut(d[], parseRst(n.comment, toFilename(n.info),
-                                     toLinenumber(n.info), toColumn(n.info),
-                                     dummyHasToc, d.options + {roSkipPounds}),
-                       result)
-      else:
-        result = n.comment.substr(2).replace("\n##", "\n").strip
-
-proc isVisible(n: PNode): bool =
-  result = false
-  if n.kind == nkPostfix:
-    if n.len == 2 and n.sons[0].kind == nkIdent:
-      var v = n.sons[0].ident
-      result = v.id == ord(wStar) or v.id == ord(wMinus)
-  elif n.kind == nkSym:
-    # we cannot generate code for forwarded symbols here as we have no
-    # exception tracking information here. Instead we copy over the comment
-    # from the proc header.
-    result = {sfExported, sfFromGeneric, sfForward}*n.sym.flags == {sfExported}
-  elif n.kind == nkPragmaExpr:
-    result = isVisible(n.sons[0])
-
-proc getName(d: PDoc, n: PNode, splitAfter = -1): string =
-  case n.kind
-  of nkPostfix: result = getName(d, n.sons[1], splitAfter)
-  of nkPragmaExpr: result = getName(d, n.sons[0], splitAfter)
-  of nkSym: result = esc(d.target, n.sym.renderDefinitionName, splitAfter)
-  of nkIdent: result = esc(d.target, n.ident.s, splitAfter)
-  of nkAccQuoted:
-    result = esc(d.target, "`")
-    for i in 0.. <n.len: result.add(getName(d, n[i], splitAfter))
-    result.add esc(d.target, "`")
-  of nkOpenSymChoice, nkClosedSymChoice:
-    result = getName(d, n[0], splitAfter)
-  else:
-    internalError(n.info, "getName()")
-    result = ""
-
-proc getNameIdent(n: PNode): PIdent =
-  case n.kind
-  of nkPostfix: result = getNameIdent(n.sons[1])
-  of nkPragmaExpr: result = getNameIdent(n.sons[0])
-  of nkSym: result = n.sym.name
-  of nkIdent: result = n.ident
-  of nkAccQuoted:
-    var r = ""
-    for i in 0.. <n.len: r.add(getNameIdent(n[i]).s)
-    result = getIdent(r)
-  of nkOpenSymChoice, nkClosedSymChoice:
-    result = getNameIdent(n[0])
-  else:
-    result = nil
-
-proc getRstName(n: PNode): PRstNode =
-  case n.kind
-  of nkPostfix: result = getRstName(n.sons[1])
-  of nkPragmaExpr: result = getRstName(n.sons[0])
-  of nkSym: result = newRstNode(rnLeaf, n.sym.renderDefinitionName)
-  of nkIdent: result = newRstNode(rnLeaf, n.ident.s)
-  of nkAccQuoted:
-    result = getRstName(n.sons[0])
-    for i in 1 .. <n.len: result.text.add(getRstName(n[i]).text)
-  of nkOpenSymChoice, nkClosedSymChoice:
-    result = getRstName(n[0])
-  else:
-    internalError(n.info, "getRstName()")
-    result = nil
-
-proc newUniquePlainSymbol(d: PDoc, original: string): string =
-  ## Returns a new unique plain symbol made up from the original.
-  ##
-  ## When a collision is found in the seenSymbols table, new numerical variants
-  ## with underscore + number will be generated.
-  if not d.seenSymbols.hasKey(original):
-    result = original
-    d.seenSymbols[original] = ""
-    return
-
-  # Iterate over possible numeric variants of the original name.
-  var count = 2
-
-  while true:
-    result = original & "_" & $count
-    if not d.seenSymbols.hasKey(result):
-      d.seenSymbols[result] = ""
-      break
-    count += 1
-
-
-proc complexName(k: TSymKind, n: PNode, baseName: string): string =
-  ## Builds a complex unique href name for the node.
-  ##
-  ## Pass as ``baseName`` the plain symbol obtained from the nodeName. The
-  ## format of the returned symbol will be ``baseName(.callable type)?,(param
-  ## type)?(,param type)*``. The callable type part will be added only if the
-  ## node is not a proc, as those are the common ones. The suffix will be a dot
-  ## and a single letter representing the type of the callable. The parameter
-  ## types will be added with a preceding dash. Return types won't be added.
-  ##
-  ## If you modify the output of this proc, please update the anchor generation
-  ## section of ``doc/docgen.txt``.
-  result = baseName
-  case k:
-  of skProc: result.add(defaultParamSeparator)
-  of skMacro: result.add(".m" & defaultParamSeparator)
-  of skMethod: result.add(".e" & defaultParamSeparator)
-  of skIterator: result.add(".i" & defaultParamSeparator)
-  of skTemplate: result.add(".t" & defaultParamSeparator)
-  of skConverter: result.add(".c" & defaultParamSeparator)
-  else: discard
-
-  if len(n) > paramsPos and n[paramsPos].kind == nkFormalParams:
-    result.add(renderParamTypes(n[paramsPos]))
-
-
-proc isCallable(n: PNode): bool =
-  ## Returns true if `n` contains a callable node.
-  case n.kind
-  of nkProcDef, nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef,
-    nkConverterDef: result = true
-  else:
-    result = false
-
-
-proc docstringSummary(rstText: string): string =
-  ## Returns just the first line or a brief chunk of text from a rst string.
-  ##
-  ## Most docstrings will contain a one liner summary, so stripping at the
-  ## first newline is usually fine. If after that the content is still too big,
-  ## it is stripped at the first comma, colon or dot, usual english sentence
-  ## separators.
-  ##
-  ## No guarantees are made on the size of the output, but it should be small.
-  ## Also, we hope to not break the rst, but maybe we do. If there is any
-  ## trimming done, an ellipsis unicode char is added.
-  const maxDocstringChars = 100
-  assert(rstText.len < 2 or (rstText[0] == '#' and rstText[1] == '#'))
-  result = rstText.substr(2).strip
-  var pos = result.find('\L')
-  if pos > 0:
-    result.delete(pos, result.len - 1)
-    result.add("…")
-  if pos < maxDocstringChars:
-    return
-  # Try to keep trimming at other natural boundaries.
-  pos = result.find({'.', ',', ':'})
-  let last = result.len - 1
-  if pos > 0 and pos < last:
-    result.delete(pos, last)
-    result.add("…")
-
-
-proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
-  if not isVisible(nameNode): return
-  let
-    name = getName(d, nameNode)
-    nameRope = name.rope
-    plainDocstring = getPlainDocstring(n) # call here before genRecComment!
-  var result: Rope = nil
-  var literal, plainName = ""
-  var kind = tkEof
-  var comm = genRecComment(d, n)  # call this here for the side-effect!
+proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRenderFlags = {}) =
   var r: TSrcGen
-  # Obtain the plain rendered string for hyperlink titles.
-  initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
-    renderNoPragmas, renderNoProcDefs})
-  while true:
-    getNextTok(r, kind, literal)
-    if kind == tkEof:
-      break
-    plainName.add(literal)
-
-  # Render the HTML hyperlink.
-  initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments})
+  var literal = ""
+  initTokRender(r, n, renderFlags)
+  var kind = tkEof
   while true:
     getNextTok(r, kind, literal)
     case kind
@@ -443,6 +261,221 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
       dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}",
             [rope(esc(d.target, literal))])
 
+proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
+  case n.kind
+  of nkCallKinds:
+    if n[0].kind == nkSym and n[0].sym.magic == mRunnableExamples and
+        n.len >= 2 and n.lastSon.kind == nkStmtList:
+      dispA(dest, "\n<strong class=\"examples_text\">$1</strong>\n",
+          "\n\\textbf{$1}\n", [rope"Examples:"])
+      inc d.listingCounter
+      let id = $d.listingCounter
+      dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"])
+      # this is a rather hacky way to get rid of the initial indentation
+      # that the renderer currently produces:
+      var i = 0
+      var body = n.lastSon
+      if body.len == 1 and body.kind == nkStmtList and
+          body.lastSon.kind == nkStmtList:
+        body = body.lastSon
+      for b in body:
+        if i > 0: dest.add "\n"
+        inc i
+        nodeToHighlightedHtml(d, b, dest, {})
+      dest.add(d.config.getOrDefault"doc.listing_end" % id)
+  else: discard
+  for i in 0 ..< n.safeLen:
+    getAllRunnableExamples(d, n[i], dest)
+
+when false:
+  proc findDocComment(n: PNode): PNode =
+    if n == nil: return nil
+    if not isNil(n.comment) and startsWith(n.comment, "##"): return n
+    for i in countup(0, safeLen(n)-1):
+      result = findDocComment(n.sons[i])
+      if result != nil: return
+
+  proc extractDocComment*(s: PSym, d: PDoc = nil): string =
+    let n = findDocComment(s.ast)
+    result = ""
+    if not n.isNil:
+      if not d.isNil:
+        var dummyHasToc: bool
+        renderRstToOut(d[], parseRst(n.comment, toFilename(n.info),
+                                     toLinenumber(n.info), toColumn(n.info),
+                                     dummyHasToc, d.options + {roSkipPounds}),
+                       result)
+      else:
+        result = n.comment.substr(2).replace("\n##", "\n").strip
+
+proc isVisible(n: PNode): bool =
+  result = false
+  if n.kind == nkPostfix:
+    if n.len == 2 and n.sons[0].kind == nkIdent:
+      var v = n.sons[0].ident
+      result = v.id == ord(wStar) or v.id == ord(wMinus)
+  elif n.kind == nkSym:
+    # we cannot generate code for forwarded symbols here as we have no
+    # exception tracking information here. Instead we copy over the comment
+    # from the proc header.
+    result = {sfExported, sfFromGeneric, sfForward}*n.sym.flags == {sfExported}
+  elif n.kind == nkPragmaExpr:
+    result = isVisible(n.sons[0])
+
+proc getName(d: PDoc, n: PNode, splitAfter = -1): string =
+  case n.kind
+  of nkPostfix: result = getName(d, n.sons[1], splitAfter)
+  of nkPragmaExpr: result = getName(d, n.sons[0], splitAfter)
+  of nkSym: result = esc(d.target, n.sym.renderDefinitionName, splitAfter)
+  of nkIdent: result = esc(d.target, n.ident.s, splitAfter)
+  of nkAccQuoted:
+    result = esc(d.target, "`")
+    for i in 0..<n.len: result.add(getName(d, n[i], splitAfter))
+    result.add esc(d.target, "`")
+  of nkOpenSymChoice, nkClosedSymChoice:
+    result = getName(d, n[0], splitAfter)
+  else:
+    internalError(n.info, "getName()")
+    result = ""
+
+proc getNameIdent(n: PNode): PIdent =
+  case n.kind
+  of nkPostfix: result = getNameIdent(n.sons[1])
+  of nkPragmaExpr: result = getNameIdent(n.sons[0])
+  of nkSym: result = n.sym.name
+  of nkIdent: result = n.ident
+  of nkAccQuoted:
+    var r = ""
+    for i in 0..<n.len: r.add(getNameIdent(n[i]).s)
+    result = getIdent(r)
+  of nkOpenSymChoice, nkClosedSymChoice:
+    result = getNameIdent(n[0])
+  else:
+    result = nil
+
+proc getRstName(n: PNode): PRstNode =
+  case n.kind
+  of nkPostfix: result = getRstName(n.sons[1])
+  of nkPragmaExpr: result = getRstName(n.sons[0])
+  of nkSym: result = newRstNode(rnLeaf, n.sym.renderDefinitionName)
+  of nkIdent: result = newRstNode(rnLeaf, n.ident.s)
+  of nkAccQuoted:
+    result = getRstName(n.sons[0])
+    for i in 1 ..< n.len: result.text.add(getRstName(n[i]).text)
+  of nkOpenSymChoice, nkClosedSymChoice:
+    result = getRstName(n[0])
+  else:
+    internalError(n.info, "getRstName()")
+    result = nil
+
+proc newUniquePlainSymbol(d: PDoc, original: string): string =
+  ## Returns a new unique plain symbol made up from the original.
+  ##
+  ## When a collision is found in the seenSymbols table, new numerical variants
+  ## with underscore + number will be generated.
+  if not d.seenSymbols.hasKey(original):
+    result = original
+    d.seenSymbols[original] = ""
+    return
+
+  # Iterate over possible numeric variants of the original name.
+  var count = 2
+
+  while true:
+    result = original & "_" & $count
+    if not d.seenSymbols.hasKey(result):
+      d.seenSymbols[result] = ""
+      break
+    count += 1
+
+
+proc complexName(k: TSymKind, n: PNode, baseName: string): string =
+  ## Builds a complex unique href name for the node.
+  ##
+  ## Pass as ``baseName`` the plain symbol obtained from the nodeName. The
+  ## format of the returned symbol will be ``baseName(.callable type)?,(param
+  ## type)?(,param type)*``. The callable type part will be added only if the
+  ## node is not a proc, as those are the common ones. The suffix will be a dot
+  ## and a single letter representing the type of the callable. The parameter
+  ## types will be added with a preceding dash. Return types won't be added.
+  ##
+  ## If you modify the output of this proc, please update the anchor generation
+  ## section of ``doc/docgen.txt``.
+  result = baseName
+  case k:
+  of skProc, skFunc: result.add(defaultParamSeparator)
+  of skMacro: result.add(".m" & defaultParamSeparator)
+  of skMethod: result.add(".e" & defaultParamSeparator)
+  of skIterator: result.add(".i" & defaultParamSeparator)
+  of skTemplate: result.add(".t" & defaultParamSeparator)
+  of skConverter: result.add(".c" & defaultParamSeparator)
+  else: discard
+
+  if len(n) > paramsPos and n[paramsPos].kind == nkFormalParams:
+    result.add(renderParamTypes(n[paramsPos]))
+
+
+proc isCallable(n: PNode): bool =
+  ## Returns true if `n` contains a callable node.
+  case n.kind
+  of nkProcDef, nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef,
+    nkConverterDef, nkFuncDef: result = true
+  else:
+    result = false
+
+
+proc docstringSummary(rstText: string): string =
+  ## Returns just the first line or a brief chunk of text from a rst string.
+  ##
+  ## Most docstrings will contain a one liner summary, so stripping at the
+  ## first newline is usually fine. If after that the content is still too big,
+  ## it is stripped at the first comma, colon or dot, usual english sentence
+  ## separators.
+  ##
+  ## No guarantees are made on the size of the output, but it should be small.
+  ## Also, we hope to not break the rst, but maybe we do. If there is any
+  ## trimming done, an ellipsis unicode char is added.
+  const maxDocstringChars = 100
+  assert(rstText.len < 2 or (rstText[0] == '#' and rstText[1] == '#'))
+  result = rstText.substr(2).strip
+  var pos = result.find('\L')
+  if pos > 0:
+    result.delete(pos, result.len - 1)
+    result.add("…")
+  if pos < maxDocstringChars:
+    return
+  # Try to keep trimming at other natural boundaries.
+  pos = result.find({'.', ',', ':'})
+  let last = result.len - 1
+  if pos > 0 and pos < last:
+    result.delete(pos, last)
+    result.add("…")
+
+
+proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
+  if not isVisible(nameNode): return
+  let
+    name = getName(d, nameNode)
+    nameRope = name.rope
+  var plainDocstring = getPlainDocstring(n) # call here before genRecComment!
+  var result: Rope = nil
+  var literal, plainName = ""
+  var kind = tkEof
+  var comm = genRecComment(d, n)  # call this here for the side-effect!
+  getAllRunnableExamples(d, n, comm)
+  var r: TSrcGen
+  # Obtain the plain rendered string for hyperlink titles.
+  initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
+    renderNoPragmas, renderNoProcDefs})
+  while true:
+    getNextTok(r, kind, literal)
+    if kind == tkEof:
+      break
+    plainName.add(literal)
+
+  # Render the HTML hyperlink.
+  nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments, renderDocComments})
+
   inc(d.id)
   let
     plainNameRope = rope(xmltree.escape(plainName.strip))
@@ -520,12 +553,24 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode =
 proc checkForFalse(n: PNode): bool =
   result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0
 
-proc traceDeps(d: PDoc, n: PNode) =
+proc traceDeps(d: PDoc, it: PNode) =
   const k = skModule
-  if d.section[k] != nil: add(d.section[k], ", ")
-  dispA(d.section[k],
-        "<a class=\"reference external\" href=\"$1.html\">$1</a>",
-        "$1", [rope(getModuleName(n))])
+
+  if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
+    let sep = it[0]
+    let dir = it[1]
+    let a = newNodeI(nkInfix, it.info)
+    a.add sep
+    a.add dir
+    a.add sep # dummy entry, replaced in the loop
+    for x in it[2]:
+      a.sons[2] = x
+      traceDeps(d, a)
+  else:
+    if d.section[k] != nil: add(d.section[k], ", ")
+    dispA(d.section[k],
+          "<a class=\"reference external\" href=\"$1.html\">$1</a>",
+          "$1", [rope(getModuleName(it))])
 
 proc generateDoc*(d: PDoc, n: PNode) =
   case n.kind
@@ -533,6 +578,9 @@ proc generateDoc*(d: PDoc, n: PNode) =
   of nkProcDef:
     when useEffectSystem: documentRaises(n)
     genItem(d, n, n.sons[namePos], skProc)
+  of nkFuncDef:
+    when useEffectSystem: documentRaises(n)
+    genItem(d, n, n.sons[namePos], skFunc)
   of nkMethodDef:
     when useEffectSystem: documentRaises(n)
     genItem(d, n, n.sons[namePos], skMethod)
@@ -574,6 +622,9 @@ proc generateJson*(d: PDoc, n: PNode) =
   of nkProcDef:
     when useEffectSystem: documentRaises(n)
     d.add genJsonItem(d, n, n.sons[namePos], skProc)
+  of nkFuncDef:
+    when useEffectSystem: documentRaises(n)
+    d.add genJsonItem(d, n, n.sons[namePos], skFunc)
   of nkMethodDef:
     when useEffectSystem: documentRaises(n)
     d.add genJsonItem(d, n, n.sons[namePos], skMethod)
@@ -602,10 +653,53 @@ proc generateJson*(d: PDoc, n: PNode) =
       generateJson(d, lastSon(n.sons[0]))
   else: discard
 
+proc genTagsItem(d: PDoc, n, nameNode: PNode, k: TSymKind): string =
+  result = getName(d, nameNode) & "\n"
+
+proc generateTags*(d: PDoc, n: PNode, r: var Rope) =
+  case n.kind
+  of nkCommentStmt:
+    if n.comment != nil and startsWith(n.comment, "##"):
+      let stripped = n.comment.substr(2).strip
+      r.add stripped
+  of nkProcDef:
+    when useEffectSystem: documentRaises(n)
+    r.add genTagsItem(d, n, n.sons[namePos], skProc)
+  of nkFuncDef:
+    when useEffectSystem: documentRaises(n)
+    r.add genTagsItem(d, n, n.sons[namePos], skFunc)
+  of nkMethodDef:
+    when useEffectSystem: documentRaises(n)
+    r.add genTagsItem(d, n, n.sons[namePos], skMethod)
+  of nkIteratorDef:
+    when useEffectSystem: documentRaises(n)
+    r.add genTagsItem(d, n, n.sons[namePos], skIterator)
+  of nkMacroDef:
+    r.add genTagsItem(d, n, n.sons[namePos], skMacro)
+  of nkTemplateDef:
+    r.add genTagsItem(d, n, n.sons[namePos], skTemplate)
+  of nkConverterDef:
+    when useEffectSystem: documentRaises(n)
+    r.add genTagsItem(d, n, n.sons[namePos], skConverter)
+  of nkTypeSection, nkVarSection, nkLetSection, nkConstSection:
+    for i in countup(0, sonsLen(n) - 1):
+      if n.sons[i].kind != nkCommentStmt:
+        # order is always 'type var let const':
+        r.add genTagsItem(d, n.sons[i], n.sons[i].sons[0],
+                succ(skType, ord(n.kind)-ord(nkTypeSection)))
+  of nkStmtList:
+    for i in countup(0, sonsLen(n) - 1):
+      generateTags(d, n.sons[i], r)
+  of nkWhenStmt:
+    # generate documentation for the first branch only:
+    if not checkForFalse(n.sons[0].sons[0]):
+      generateTags(d, lastSon(n.sons[0]), r)
+  else: discard
+
 proc genSection(d: PDoc, kind: TSymKind) =
   const sectionNames: array[skModule..skTemplate, string] = [
-    "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Methods",
-    "Iterators", "Converters", "Macros", "Templates"
+    "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Funcs",
+    "Methods", "Iterators", "Converters", "Macros", "Templates"
   ]
   if d.section[kind] == nil: return
   var title = sectionNames[kind].rope
@@ -706,6 +800,26 @@ proc commandDoc*() =
 proc commandRstAux(filename, outExt: string) =
   var filen = addFileExt(filename, "txt")
   var d = newDocumentor(filen, options.gConfigVars)
+  d.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string;
+                          status: int; content: string) =
+    var outp: string
+    if filename.len == 0:
+      inc(d.id)
+      let nameOnly = splitFile(d.filename).name
+      let subdir = getNimcacheDir() / nameOnly
+      createDir(subdir)
+      outp = subdir / (nameOnly & "_snippet_" & $d.id & ".nim")
+    elif isAbsolute(filename):
+      outp = filename
+    else:
+      # Nim's convention: every path is relative to the file it was written in:
+      outp = splitFile(d.filename).dir / filename
+    writeFile(outp, content)
+    let cmd = unescape(cmd) % quoteShell(outp)
+    rawMessage(hintExecuting, cmd)
+    if execShellCmd(cmd) != status:
+      rawMessage(errExecutionOfProgramFailed, cmd)
+
   d.isPureRst = true
   var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc,
                      {roSupportRawDirective})
@@ -739,6 +853,21 @@ proc commandJson*() =
     #echo getOutFile(gProjectFull, JsonExt)
     writeRope(content, getOutFile(gProjectFull, JsonExt), useWarning = false)
 
+proc commandTags*() =
+  var ast = parseFile(gProjectMainIdx, newIdentCache())
+  if ast == nil: return
+  var d = newDocumentor(gProjectFull, options.gConfigVars)
+  d.hasToc = true
+  var
+    content: Rope
+  generateTags(d, ast, content)
+
+  if optStdout in gGlobalOptions:
+    writeRope(stdout, content)
+  else:
+    #echo getOutFile(gProjectFull, TagsExt)
+    writeRope(content, getOutFile(gProjectFull, TagsExt), useWarning = false)
+
 proc commandBuildIndex*() =
   var content = mergeIndexes(gProjectFull).rope
 

compiler/evalffi.nim

@@ -86,10 +86,10 @@ proc mapType(t: ast.PType): ptr libffi.TType =
     else: result = nil
   of tyFloat, tyFloat64: result = addr libffi.type_double
   of tyFloat32: result = addr libffi.type_float
-  of tyVar, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr,
+  of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr,
      tyStmt, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
     result = addr libffi.type_pointer
-  of tyDistinct, tyAlias:
+  of tyDistinct, tyAlias, tySink:
     result = mapType(t.sons[0])
   else:
     result = nil
@@ -112,12 +112,12 @@ template `+!`(x, y: untyped): untyped =
 proc packSize(v: PNode, typ: PType): int =
   ## computes the size of the blob
   case typ.kind
-  of tyPtr, tyRef, tyVar:
+  of tyPtr, tyRef, tyVar, tyLent:
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     else:
       result = sizeof(pointer) + packSize(v.sons[0], typ.lastSon)
-  of tyDistinct, tyGenericInst, tyAlias:
+  of tyDistinct, tyGenericInst, tyAlias, tySink:
     result = packSize(v, typ.sons[0])
   of tyArray:
     # consider: ptr array[0..1000_000, int] which is common for interfacing;
@@ -209,7 +209,7 @@ proc pack(v: PNode, typ: PType, res: pointer) =
       awr(cstring, cstring(v.strVal))
     else:
       globalError(v.info, "cannot map pointer/proc value to FFI")
-  of tyPtr, tyRef, tyVar:
+  of tyPtr, tyRef, tyVar, tyLent:
     if v.kind == nkNilLit:
       # nothing to do since the memory is 0 initialized anyway
       discard
@@ -225,13 +225,13 @@ proc pack(v: PNode, typ: PType, res: pointer) =
       awr(pointer, res +! sizeof(pointer))
   of tyArray:
     let baseSize = typ.sons[1].getSize
-    for i in 0 .. <v.len:
+    for i in 0 ..< v.len:
       pack(v.sons[i], typ.sons[1], res +! i * baseSize)
   of tyObject, tyTuple:
     packObject(v, typ, res)
   of tyNil:
     discard
-  of tyDistinct, tyGenericInst, tyAlias:
+  of tyDistinct, tyGenericInst, tyAlias, tySink:
     pack(v, typ.sons[0], res)
   else:
     globalError(v.info, "cannot map value to FFI " & typeToString(v.typ))
@@ -291,7 +291,7 @@ proc unpackArray(x: pointer, typ: PType, n: PNode): PNode =
     if result.kind != nkBracket:
       globalError(n.info, "cannot map value from FFI")
   let baseSize = typ.sons[1].getSize
-  for i in 0 .. < result.len:
+  for i in 0 ..< result.len:
     result.sons[i] = unpack(x +! i * baseSize, typ.sons[1], result.sons[i])
 
 proc canonNodeKind(k: TNodeKind): TNodeKind =
@@ -364,7 +364,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
       result = n
     else:
       awi(nkPtrLit, cast[ByteAddress](p))
-  of tyPtr, tyRef, tyVar:
+  of tyPtr, tyRef, tyVar, tyLent:
     let p = rd(pointer, x)
     if p.isNil:
       setNil()
@@ -388,14 +388,14 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode =
       aws(nkStrLit, $p)
   of tyNil:
     setNil()
-  of tyDistinct, tyGenericInst, tyAlias:
+  of tyDistinct, tyGenericInst, tyAlias, tySink:
     result = unpack(x, typ.lastSon, n)
   else:
     # XXX what to do with 'array' here?
     globalError(n.info, "cannot map value from FFI " & typeToString(typ))
 
 proc fficast*(x: PNode, destTyp: PType): PNode =
-  if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyPointer,
+  if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyLent, tyPointer,
                                            tyProc, tyCString, tyString,
                                            tySequence}:
     result = newNodeIT(x.kind, x.info, destTyp)

compiler/evaltempl.nim

@@ -42,7 +42,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
            s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam:
         handleParam actual.sons[s.owner.typ.len + s.position - 1]
       else:
-        internalAssert sfGenSym in s.flags
+        internalAssert sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
         if x == nil:
           x = copySym(s, false)
@@ -77,7 +77,7 @@ proc evalTemplateArgs(n: PNode, s: PSym; fromHlo: bool): PNode =
     # now that we have working untyped parameters.
     genericParams = if sfImmediate in s.flags or fromHlo: 0
                     else: s.ast[genericParamsPos].len
-    expectedRegularParams = <s.typ.len
+    expectedRegularParams = s.typ.len-1
     givenRegularParams = totalParams - genericParams
   if givenRegularParams < 0: givenRegularParams = 0
 
@@ -109,7 +109,7 @@ proc evalTemplateArgs(n: PNode, s: PSym; fromHlo: bool): PNode =
 var evalTemplateCounter* = 0
   # to prevent endless recursion in templates instantiation
 
-proc wrapInComesFrom*(info: TLineInfo; res: PNode): PNode =
+proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
   when true:
     result = res
     result.info = info
@@ -124,8 +124,12 @@ proc wrapInComesFrom*(info: TLineInfo; res: PNode): PNode =
           if x[i].kind in nkCallKinds:
             x.sons[i].info = info
   else:
-    result = newNodeI(nkPar, info)
+    result = newNodeI(nkStmtListExpr, info)
+    var d = newNodeI(nkComesFrom, info)
+    d.add newSymNode(sym, info)
+    result.add d
     result.add res
+    result.typ = res.typ
 
 proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym; fromHlo=false): PNode =
   inc(evalTemplateCounter)
@@ -156,6 +160,6 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym; fromHlo=false): PNode =
     for i in countup(0, safeLen(body) - 1):
       evalTemplateAux(body.sons[i], args, ctx, result)
   result.flags.incl nfFromTemplate
-  result = wrapInComesFrom(n.info, result)
+  result = wrapInComesFrom(n.info, tmpl, result)
   dec(evalTemplateCounter)
 

compiler/extccomp.nim

@@ -14,14 +14,14 @@
 
 import
   ropes, os, strutils, osproc, platform, condsyms, options, msgs,
-  securehash, streams
+  std / sha1, streams
 
 #from debuginfo import writeDebugInfo
 
 type
   TSystemCC* = enum
     ccNone, ccGcc, ccLLVM_Gcc, ccCLang, ccLcc, ccBcc, ccDmc, ccWcc, ccVcc,
-    ccTcc, ccPcc, ccUcc, ccIcl
+    ccTcc, ccPcc, ccUcc, ccIcl, ccIcc
   TInfoCCProp* = enum         # properties of the C compiler:
     hasSwitchRange,           # CC allows ranges in switch statements (GNU C)
     hasComputedGoto,          # CC has computed goto (GNU C extension)
@@ -95,7 +95,11 @@ compiler llvmGcc:
   result.name = "llvm_gcc"
   result.compilerExe = "llvm-gcc"
   result.cppCompiler = "llvm-g++"
-  result.buildLib = "llvm-ar rcs $libfile $objfiles"
+  when defined(macosx):
+    # OS X has no 'llvm-ar' tool:
+    result.buildLib = "ar rcs $libfile $objfiles"
+  else:
+    result.buildLib = "llvm-ar rcs $libfile $objfiles"
 
 # Clang (LLVM) C/C++ Compiler
 compiler clang:
@@ -131,16 +135,18 @@ compiler vcc:
 
 # Intel C/C++ Compiler
 compiler icl:
-  # Intel compilers try to imitate the native ones (gcc and msvc)
-  when defined(windows):
-    result = vcc()
-  else:
-    result = gcc()
-
+  result = vcc()
   result.name = "icl"
   result.compilerExe = "icl"
   result.linkerExe = "icl"
 
+# Intel compilers try to imitate the native ones (gcc and msvc)
+compiler icc:
+  result = gcc()
+  result.name = "icc"
+  result.compilerExe = "icc"
+  result.linkerExe = "icc"
+
 # Local C Compiler
 compiler lcc:
   result = (
@@ -247,7 +253,7 @@ compiler tcc:
     compilerExe: "tcc",
     cppCompiler: "",
     compileTmpl: "-c $options $include -o $objfile $file",
-    buildGui: "UNAVAILABLE!",
+    buildGui: "-Wl,-subsystem=gui",
     buildDll: " -shared",
     buildLib: "", # XXX: not supported yet
     linkerExe: "tcc",
@@ -323,7 +329,8 @@ const
     tcc(),
     pcc(),
     ucc(),
-    icl()]
+    icl(),
+    icc()]
 
   hExt* = ".h"
 
@@ -652,9 +659,10 @@ proc getLinkCmd(projectfile, objfiles: string): string =
   else:
     var linkerExe = getConfigVar(cCompiler, ".linkerexe")
     if len(linkerExe) == 0: linkerExe = cCompiler.getLinkerExe
+    # bug #6452: We must not use ``quoteShell`` here for ``linkerExe``
     if needsExeExt(): linkerExe = addFileExt(linkerExe, "exe")
-    if noAbsolutePaths(): result = quoteShell(linkerExe)
-    else: result = quoteShell(joinPath(ccompilerpath, linkerExe))
+    if noAbsolutePaths(): result = linkerExe
+    else: result = joinPath(ccompilerpath, linkerExe)
     let buildgui = if optGenGuiApp in gGlobalOptions: CC[cCompiler].buildGui
                    else: ""
     var exefile, builddll: string
@@ -701,6 +709,40 @@ template tryExceptOSErrorMessage(errorPrefix: string = "", body: untyped): typed
       rawMessage(errExecutionOfProgramFailed, ose.msg & " " & $ose.errorCode)
     raise
 
+proc execLinkCmd(linkCmd: string) =
+  tryExceptOSErrorMessage("invocation of external linker program failed."):
+    execExternalProgram(linkCmd,
+      if optListCmd in gGlobalOptions or gVerbosity > 1: hintExecuting else: hintLinking)
+
+proc execCmdsInParallel(cmds: seq[string]; prettyCb: proc (idx: int)) =
+  let runCb = proc (idx: int, p: Process) =
+    let exitCode = p.peekExitCode
+    if exitCode != 0:
+      rawMessage(errGenerated, "execution of an external compiler program '" &
+        cmds[idx] & "' failed with exit code: " & $exitCode & "\n\n" &
+        p.outputStream.readAll.strip)
+  if gNumberOfProcessors == 0: gNumberOfProcessors = countProcessors()
+  var res = 0
+  if gNumberOfProcessors <= 1:
+    for i in countup(0, high(cmds)):
+      tryExceptOSErrorMessage("invocation of external compiler program failed."):
+        res = execWithEcho(cmds[i])
+      if res != 0: rawMessage(errExecutionOfProgramFailed, cmds[i])
+  else:
+    tryExceptOSErrorMessage("invocation of external compiler program failed."):
+      if optListCmd in gGlobalOptions or gVerbosity > 1:
+        res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath},
+                            gNumberOfProcessors, afterRunEvent=runCb)
+      elif gVerbosity == 1:
+        res = execProcesses(cmds, {poStdErrToStdOut, poUsePath},
+                            gNumberOfProcessors, prettyCb, afterRunEvent=runCb)
+      else:
+        res = execProcesses(cmds, {poStdErrToStdOut, poUsePath},
+                            gNumberOfProcessors, afterRunEvent=runCb)
+  if res != 0:
+    if gNumberOfProcessors <= 1:
+      rawMessage(errExecutionOfProgramFailed, cmds.join())
+
 proc callCCompiler*(projectfile: string) =
   var
     linkCmd: string
@@ -713,35 +755,9 @@ proc callCCompiler*(projectfile: string) =
   var prettyCmds: TStringSeq = @[]
   let prettyCb = proc (idx: int) =
     echo prettyCmds[idx]
-  let runCb = proc (idx: int, p: Process) =
-    let exitCode = p.peekExitCode
-    if exitCode != 0:
-      rawMessage(errGenerated, "execution of an external compiler program '" &
-        cmds[idx] & "' failed with exit code: " & $exitCode & "\n\n" &
-        p.outputStream.readAll.strip)
   compileCFile(toCompile, script, cmds, prettyCmds)
   if optCompileOnly notin gGlobalOptions:
-    if gNumberOfProcessors == 0: gNumberOfProcessors = countProcessors()
-    var res = 0
-    if gNumberOfProcessors <= 1:
-      for i in countup(0, high(cmds)):
-        tryExceptOSErrorMessage("invocation of external compiler program failed."):
-          res = execWithEcho(cmds[i])
-        if res != 0: rawMessage(errExecutionOfProgramFailed, cmds[i])
-    else:
-      tryExceptOSErrorMessage("invocation of external compiler program failed."):
-        if optListCmd in gGlobalOptions or gVerbosity > 1:
-          res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams},
-                              gNumberOfProcessors, afterRunEvent=runCb)
-        elif gVerbosity == 1:
-          res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams},
-                              gNumberOfProcessors, prettyCb, afterRunEvent=runCb)
-        else:
-          res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams},
-                              gNumberOfProcessors, afterRunEvent=runCb)
-    if res != 0:
-      if gNumberOfProcessors <= 1:
-        rawMessage(errExecutionOfProgramFailed, cmds.join())
+    execCmdsInParallel(cmds, prettyCb)
   if optNoLinking notin gGlobalOptions:
     # call the linker:
     var objfiles = ""
@@ -751,14 +767,13 @@ proc callCCompiler*(projectfile: string) =
       add(objfiles, quoteShell(
           addFileExt(objFile, CC[cCompiler].objExt)))
     for x in toCompile:
+      let objFile = if noAbsolutePaths(): x.obj.extractFilename else: x.obj
       add(objfiles, ' ')
-      add(objfiles, quoteShell(x.obj))
+      add(objfiles, quoteShell(objFile))
 
     linkCmd = getLinkCmd(projectfile, objfiles)
     if optCompileOnly notin gGlobalOptions:
-      tryExceptOSErrorMessage("invocation of external linker program failed."):
-        execExternalProgram(linkCmd,
-          if optListCmd in gGlobalOptions or gVerbosity > 1: hintExecuting else: hintLinking)
+      execLinkCmd(linkCmd)
   else:
     linkCmd = ""
   if optGenScript in gGlobalOptions:
@@ -766,7 +781,8 @@ proc callCCompiler*(projectfile: string) =
     add(script, tnl)
     generateScript(projectfile, script)
 
-from json import escapeJson
+#from json import escapeJson
+import json
 
 proc writeJsonBuildInstructions*(projectfile: string) =
   template lit(x: untyped) = f.write x
@@ -778,42 +794,40 @@ proc writeJsonBuildInstructions*(projectfile: string) =
     else:
       f.write escapeJson(x)
 
-  proc cfiles(f: File; buf: var string; list: CfileList, isExternal: bool) =
-    var i = 0
-    for it in list:
+  proc cfiles(f: File; buf: var string; clist: CfileList, isExternal: bool) =
+    var pastStart = false
+    for it in clist:
       if CfileFlag.Cached in it.flags: continue
       let compileCmd = getCompileCFileCmd(it)
+      if pastStart: lit "],\L"
       lit "["
       str it.cname
       lit ", "
       str compileCmd
-      inc i
-      if i == list.len:
-        lit "]\L"
-      else:
-        lit "],\L"
+      pastStart = true
+    lit "]\L"
 
-  proc linkfiles(f: File; buf, objfiles: var string) =
-    for i, it in externalToLink:
-      let
-        objFile = if noAbsolutePaths(): it.extractFilename else: it
-        objStr = addFileExt(objFile, CC[cCompiler].objExt)
+  proc linkfiles(f: File; buf, objfiles: var string; clist: CfileList;
+                 llist: seq[string]) =
+    var pastStart = false
+    for it in llist:
+      let objfile = if noAbsolutePaths(): it.extractFilename
+                    else: it
+      let objstr = addFileExt(objfile, CC[cCompiler].objExt)
       add(objfiles, ' ')
-      add(objfiles, objStr)
-      str objStr
-      if toCompile.len == 0 and i == externalToLink.high:
-        lit "\L"
-      else:
-        lit ",\L"
-    for i, x in toCompile:
-      let objStr = quoteShell(x.obj)
+      add(objfiles, objstr)
+      if pastStart: lit ",\L"
+      str objstr
+      pastStart = true
+
+    for it in clist:
+      let objstr = quoteShell(it.obj)
       add(objfiles, ' ')
-      add(objfiles, objStr)
-      str objStr
-      if i == toCompile.high:
-        lit "\L"
-      else:
-        lit ",\L"
+      add(objfiles, objstr)
+      if pastStart: lit ",\L"
+      str objstr
+      pastStart = true
+    lit "\L"
 
   var buf = newStringOfCap(50)
 
@@ -827,13 +841,41 @@ proc writeJsonBuildInstructions*(projectfile: string) =
     lit "],\L\"link\":[\L"
     var objfiles = ""
     # XXX add every file here that is to link
-    linkfiles(f, buf, objfiles)
+    linkfiles(f, buf, objfiles, toCompile, externalToLink)
 
     lit "],\L\"linkcmd\": "
     str getLinkCmd(projectfile, objfiles)
     lit "\L}\L"
     close(f)
 
+proc runJsonBuildInstructions*(projectfile: string) =
+  let file = projectfile.splitFile.name
+  let jsonFile = toGeneratedFile(file, "json")
+  try:
+    let data = json.parseFile(jsonFile)
+    let toCompile = data["compile"]
+    doAssert toCompile.kind == JArray
+    var cmds: TStringSeq = @[]
+    var prettyCmds: TStringSeq = @[]
+    for c in toCompile:
+      doAssert c.kind == JArray
+      doAssert c.len >= 2
+
+      add(cmds, c[1].getStr)
+      let (_, name, _) = splitFile(c[0].getStr)
+      add(prettyCmds, "CC: " & name)
+
+    let prettyCb = proc (idx: int) =
+      echo prettyCmds[idx]
+    execCmdsInParallel(cmds, prettyCb)
+
+    let linkCmd = data["linkcmd"]
+    doAssert linkCmd.kind == JString
+    execLinkCmd(linkCmd.getStr)
+  except:
+    echo getCurrentException().getStackTrace()
+    quit "error evaluating JSON file: " & jsonFile
+
 proc genMappingFiles(list: CFileList): Rope =
   for it in list:
     addf(result, "--file:r\"$1\"$N", [rope(it.cname)])

compiler/filter_tmpl.nim

@@ -13,14 +13,10 @@ import
   llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
   renderer, filters
 
-proc filterTmpl*(stdin: PLLStream, filename: string, call: PNode): PLLStream
-  # #! template(subsChar='$', metaChar='#') | standard(version="0.7.2")
-# implementation
-
 type
   TParseState = enum
     psDirective, psTempl
-  TTmplParser{.final.} = object
+  TTmplParser = object
     inp: PLLStream
     state: TParseState
     info: TLineInfo
@@ -61,6 +57,10 @@ proc scanPar(p: var TTmplParser, d: int) =
 proc withInExpr(p: TTmplParser): bool {.inline.} =
   result = p.par > 0 or p.bracket > 0 or p.curly > 0
 
+const
+  LineContinuationOprs = {'+', '-', '*', '/', '\\', '<', '>', '^',
+                          '|', '%', '&', '$', '@', '~', ','}
+
 proc parseLine(p: var TTmplParser) =
   var j = 0
   while p.x[j] == ' ': inc(j)
@@ -77,7 +77,7 @@ proc parseLine(p: var TTmplParser) =
       inc(j)
 
     scanPar(p, j)
-    p.pendingExprLine = withInExpr(p) or llstream.endsWithOpr(p.x)
+    p.pendingExprLine = withInExpr(p) or p.x.endsWith(LineContinuationOprs)
     case keyw
     of "end":
       if p.indent >= 2:
@@ -88,14 +88,14 @@ proc parseLine(p: var TTmplParser) =
       llStreamWrite(p.outp, spaces(p.indent))
       llStreamWrite(p.outp, "#end")
     of "if", "when", "try", "while", "for", "block", "case", "proc", "iterator",
-       "converter", "macro", "template", "method":
+       "converter", "macro", "template", "method", "func":
       llStreamWrite(p.outp, spaces(p.indent))
       llStreamWrite(p.outp, substr(p.x, d))
       inc(p.indent, 2)
     of "elif", "of", "else", "except", "finally":
       llStreamWrite(p.outp, spaces(p.indent - 2))
       llStreamWrite(p.outp, substr(p.x, d))
-    of "wLet", "wVar", "wConst", "wType":
+    of "let", "var", "const", "type":
       llStreamWrite(p.outp, spaces(p.indent))
       llStreamWrite(p.outp, substr(p.x, d))
       if not p.x.contains({':', '='}):
@@ -199,7 +199,7 @@ proc parseLine(p: var TTmplParser) =
           inc(j)
     llStreamWrite(p.outp, "\\n\"")
 
-proc filterTmpl(stdin: PLLStream, filename: string, call: PNode): PLLStream =
+proc filterTmpl*(stdin: PLLStream, filename: string, call: PNode): PLLStream =
   var p: TTmplParser
   p.info = newLineInfo(filename, 0, 0)
   p.outp = llStreamOpen("")

compiler/filters.nim

@@ -13,14 +13,6 @@ import
   llstream, os, wordrecg, idents, strutils, ast, astalgo, msgs, options,
   renderer
 
-proc filterReplace*(stdin: PLLStream, filename: string, call: PNode): PLLStream
-proc filterStrip*(stdin: PLLStream, filename: string, call: PNode): PLLStream
-  # helpers to retrieve arguments:
-proc charArg*(n: PNode, name: string, pos: int, default: char): char
-proc strArg*(n: PNode, name: string, pos: int, default: string): string
-proc boolArg*(n: PNode, name: string, pos: int, default: bool): bool
-# implementation
-
 proc invalidPragma(n: PNode) =
   localError(n.info, errXNotAllowedHere, renderTree(n, {renderNoComments}))
 
@@ -35,26 +27,26 @@ proc getArg(n: PNode, name: string, pos: int): PNode =
     elif i == pos:
       return n.sons[i]
 
-proc charArg(n: PNode, name: string, pos: int, default: char): char =
+proc charArg*(n: PNode, name: string, pos: int, default: char): char =
   var x = getArg(n, name, pos)
   if x == nil: result = default
   elif x.kind == nkCharLit: result = chr(int(x.intVal))
   else: invalidPragma(n)
 
-proc strArg(n: PNode, name: string, pos: int, default: string): string =
+proc strArg*(n: PNode, name: string, pos: int, default: string): string =
   var x = getArg(n, name, pos)
   if x == nil: result = default
   elif x.kind in {nkStrLit..nkTripleStrLit}: result = x.strVal
   else: invalidPragma(n)
 
-proc boolArg(n: PNode, name: string, pos: int, default: bool): bool =
+proc boolArg*(n: PNode, name: string, pos: int, default: bool): bool =
   var x = getArg(n, name, pos)
   if x == nil: result = default
   elif x.kind == nkIdent and cmpIgnoreStyle(x.ident.s, "true") == 0: result = true
   elif x.kind == nkIdent and cmpIgnoreStyle(x.ident.s, "false") == 0: result = false
   else: invalidPragma(n)
 
-proc filterStrip(stdin: PLLStream, filename: string, call: PNode): PLLStream =
+proc filterStrip*(stdin: PLLStream, filename: string, call: PNode): PLLStream =
   var pattern = strArg(call, "startswith", 1, "")
   var leading = boolArg(call, "leading", 2, true)
   var trailing = boolArg(call, "trailing", 3, true)
@@ -68,7 +60,7 @@ proc filterStrip(stdin: PLLStream, filename: string, call: PNode): PLLStream =
       llStreamWriteln(result, line)
   llStreamClose(stdin)
 
-proc filterReplace(stdin: PLLStream, filename: string, call: PNode): PLLStream =
+proc filterReplace*(stdin: PLLStream, filename: string, call: PNode): PLLStream =
   var sub = strArg(call, "sub", 1, "")
   if len(sub) == 0: invalidPragma(call)
   var by = strArg(call, "by", 2, "")

compiler/forloops.nim

@@ -45,13 +45,13 @@ proc counterInTree(n, loop: PNode; counter: PSym): bool =
     for it in n:
       if counterInTree(it.lastSon): return true
   else:
-    for i in 0 .. <safeLen(n):
+    for i in 0 ..< safeLen(n):
       if counterInTree(n[i], loop, counter): return true
 
 proc copyExcept(n: PNode, x, dest: PNode) =
   if x == n: return
   if n.kind in {nkStmtList, nkStmtListExpr}:
-    for i in 0 .. <n.len: copyExcept(n[i], x, dest)
+    for i in 0 ..< n.len: copyExcept(n[i], x, dest)
   else:
     dest.add n
 

compiler/gorgeimpl.nim

@@ -0,0 +1,57 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Module that implements ``gorge`` for the compiler.
+
+import msgs, std / sha1, os, osproc, streams, strutils, options
+
+proc readOutput(p: Process): (string, int) =
+  result[0] = ""
+  var output = p.outputStream
+  while not output.atEnd:
+    result[0].add(output.readLine)
+    result[0].add("\n")
+  if result[0].len > 0:
+    result[0].setLen(result[0].len - "\n".len)
+  result[1] = p.waitForExit
+
+proc opGorge*(cmd, input, cache: string, info: TLineInfo): (string, int) =
+  let workingDir = parentDir(info.toFullPath)
+  if cache.len > 0:# and optForceFullMake notin gGlobalOptions:
+    let h = secureHash(cmd & "\t" & input & "\t" & cache)
+    let filename = options.toGeneratedFile("gorge_" & $h, "txt")
+    var f: File
+    if open(f, filename):
+      result = (f.readAll, 0)
+      f.close
+      return
+    var readSuccessful = false
+    try:
+      var p = startProcess(cmd, workingDir,
+                           options={poEvalCommand, poStderrToStdout})
+      if input.len != 0:
+        p.inputStream.write(input)
+        p.inputStream.close()
+      result = p.readOutput
+      readSuccessful = true
+      # only cache successful runs:
+      if result[1] == 0:
+        writeFile(filename, result[0])
+    except IOError, OSError:
+      if not readSuccessful: result = ("", -1)
+  else:
+    try:
+      var p = startProcess(cmd, workingDir,
+                           options={poEvalCommand, poStderrToStdout})
+      if input.len != 0:
+        p.inputStream.write(input)
+        p.inputStream.close()
+      result = p.readOutput
+    except IOError, OSError:
+      result = ("", -1)

compiler/guards.nim

@@ -52,7 +52,7 @@ proc isLet(n: PNode): bool =
 
 proc isVar(n: PNode): bool =
   n.kind == nkSym and n.sym.kind in {skResult, skVar} and
-      {sfGlobal, sfAddrTaken} * n.sym.flags == {}
+      {sfAddrTaken} * n.sym.flags == {}
 
 proc isLetLocation(m: PNode, isApprox: bool): bool =
   # consider: 'n[].kind' --> we really need to support 1 deref op even if this
@@ -247,7 +247,7 @@ proc canon*(n: PNode): PNode =
   # XXX for now only the new code in 'semparallel' uses this
   if n.safeLen >= 1:
     result = shallowCopy(n)
-    for i in 0 .. < n.len:
+    for i in 0 ..< n.len:
       result.sons[i] = canon(n.sons[i])
   elif n.kind == nkSym and n.sym.kind == skLet and
       n.sym.ast.getMagic in (someEq + someAdd + someMul + someMin +
@@ -768,8 +768,10 @@ macro `=~`(x: PNode, pat: untyped): bool =
 
   var conds = newTree(nnkBracket)
   m(x, pat, conds)
-  result = nestList(!"and", conds)
-
+  when declared(macros.toNimIdent):
+    result = nestList(toNimIdent"and", conds)
+  else:
+    result = nestList(!"and", conds)
 
 proc isMinusOne(n: PNode): bool =
   n.kind in {nkCharLit..nkUInt64Lit} and n.intVal == -1

compiler/hlo.nim

@@ -36,7 +36,7 @@ proc applyPatterns(c: PContext, n: PNode): PNode =
   # we apply the last pattern first, so that pattern overriding is possible;
   # however the resulting AST would better not trigger the old rule then
   # anymore ;-)
-  for i in countdown(<c.patterns.len, 0):
+  for i in countdown(c.patterns.len-1, 0):
     let pattern = c.patterns[i]
     if not isNil(pattern):
       let x = applyRule(c, pattern, result)
@@ -75,7 +75,7 @@ proc hlo(c: PContext, n: PNode): PNode =
     result = applyPatterns(c, n)
     if result == n:
       # no optimization applied, try subtrees:
-      for i in 0 .. < safeLen(result):
+      for i in 0 ..< safeLen(result):
         let a = result.sons[i]
         let h = hlo(c, a)
         if h != a: result.sons[i] = h

compiler/importer.nim

@@ -11,54 +11,15 @@
 
 import
   intsets, strutils, os, ast, astalgo, msgs, options, idents, rodread, lookups,
-  semdata, passes, renderer
+  semdata, passes, renderer, modulepaths, sigmatch
 
 proc evalImport*(c: PContext, n: PNode): PNode
 proc evalFrom*(c: PContext, n: PNode): PNode
 
-proc getModuleName*(n: PNode): string =
-  # This returns a short relative module name without the nim extension
-  # e.g. like "system", "importer" or "somepath/module"
-  # The proc won't perform any checks that the path is actually valid
-  case n.kind
-  of nkStrLit, nkRStrLit, nkTripleStrLit:
-    try:
-      result = pathSubs(n.strVal, n.info.toFullPath().splitFile().dir)
-    except ValueError:
-      localError(n.info, "invalid path: " & n.strVal)
-      result = n.strVal
-  of nkIdent:
-    result = n.ident.s
-  of nkSym:
-    result = n.sym.name.s
-  of nkInfix, nkPrefix:
-    if n.sons[0].kind == nkIdent and n.sons[0].ident.id == getIdent("as").id:
-      # XXX hack ahead:
-      n.kind = nkImportAs
-      n.sons[0] = n.sons[1]
-      n.sons[1] = n.sons[2]
-      n.sons.setLen(2)
-      return getModuleName(n.sons[0])
-    # hacky way to implement 'x / y /../ z':
-    result = renderTree(n, {renderNoComments}).replace(" ")
-  of nkDotExpr:
-    result = renderTree(n, {renderNoComments}).replace(".", "/")
-  of nkImportAs:
-    result = getModuleName(n.sons[0])
-  else:
-    localError(n.info, errGenerated, "invalid module name: '$1'" % n.renderTree)
-    result = ""
-
-proc checkModuleName*(n: PNode; doLocalError=true): int32 =
-  # This returns the full canonical path for a given module import
-  let modulename = n.getModuleName
-  let fullPath = findModule(modulename, n.info.toFullPath)
-  if fullPath.len == 0:
-    if doLocalError:
-      localError(n.info, errCannotOpenFile, modulename)
-    result = InvalidFileIDX
-  else:
-    result = fullPath.fileInfoIdx
+proc importPureEnumField*(c: PContext; s: PSym) =
+  var check = strTableGet(c.importTable.symbols, s.name)
+  if check == nil:
+    strTableAdd(c.pureEnumFields, s)
 
 proc rawImportSymbol(c: PContext, s: PSym) =
   # This does not handle stubs, because otherwise loading on demand would be
@@ -66,7 +27,7 @@ proc rawImportSymbol(c: PContext, s: PSym) =
   # check if we have already a symbol of the same name:
   var check = strTableGet(c.importTable.symbols, s.name)
   if check != nil and check.id != s.id:
-    if s.kind notin OverloadableSyms:
+    if s.kind notin OverloadableSyms or check.kind notin OverloadableSyms:
       # s and check need to be qualified:
       incl(c.ambiguousSymbols, s.id)
       incl(c.ambiguousSymbols, check.id)
@@ -75,7 +36,7 @@ proc rawImportSymbol(c: PContext, s: PSym) =
   strTableAdd(c.importTable.symbols, s)
   if s.kind == skType:
     var etyp = s.typ
-    if etyp.kind in {tyBool, tyEnum} and sfPure notin s.flags:
+    if etyp.kind in {tyBool, tyEnum}:
       for j in countup(0, sonsLen(etyp.n) - 1):
         var e = etyp.n.sons[j].sym
         if e.kind != skEnumField:
@@ -91,7 +52,10 @@ proc rawImportSymbol(c: PContext, s: PSym) =
             break
           check = nextIdentIter(it, c.importTable.symbols)
         if e != nil:
-          rawImportSymbol(c, e)
+          if sfPure notin s.flags:
+            rawImportSymbol(c, e)
+          else:
+            importPureEnumField(c, e)
   else:
     # rodgen assures that converters and patterns are no stubs
     if s.kind == skConverter: addConverter(c, s)
@@ -185,7 +149,7 @@ proc myImportModule(c: PContext, n: PNode): PSym =
         localError(n.info, errGenerated, "A module cannot import itself")
     if sfDeprecated in result.flags:
       message(n.info, warnDeprecated, result.name.s)
-    #suggestSym(n.info, result, false)
+    suggestSym(n.info, result, c.graph.usageSym, false)
 
 proc impMod(c: PContext; it: PNode) =
   let m = myImportModule(c, it)
@@ -201,12 +165,14 @@ proc evalImport(c: PContext, n: PNode): PNode =
   for i in countup(0, sonsLen(n) - 1):
     let it = n.sons[i]
     if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
-      let sep = renderTree(it.sons[0], {renderNoComments})
-      let dir = renderTree(it.sons[1], {renderNoComments})
+      let sep = it[0]
+      let dir = it[1]
+      let a = newNodeI(nkInfix, it.info)
+      a.add sep
+      a.add dir
+      a.add sep # dummy entry, replaced in the loop
       for x in it[2]:
-        let f = renderTree(x, {renderNoComments})
-        let a = newStrNode(nkStrLit, (dir & sep & f).replace(" "))
-        a.info = it.info
+        a.sons[2] = x
         impMod(c, a)
     else:
       impMod(c, it)

compiler/jsgen.nim

@@ -31,7 +31,7 @@ implements the required case distinction.
 
 import
   ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options,
-  nversion, nimsets, msgs, securehash, bitsets, idents, types, os,
+  nversion, nimsets, msgs, std / sha1, bitsets, idents, types, os,
   times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils,
   intsets, cgmeth, lowerings
 
@@ -173,7 +173,7 @@ const
 proc mapType(typ: PType): TJSTypeKind =
   let t = skipTypes(typ, abstractInst)
   case t.kind
-  of tyVar, tyRef, tyPtr:
+  of tyVar, tyRef, tyPtr, tyLent:
     if skipTypes(t.lastSon, abstractInst).kind in MappedToObject:
       result = etyObject
     else:
@@ -187,22 +187,24 @@ proc mapType(typ: PType): TJSTypeKind =
   of tyBool: result = etyBool
   of tyFloat..tyFloat128: result = etyFloat
   of tySet: result = etyObject # map a set to a table
-  of tyString, tySequence: result = etySeq
+  of tyString, tySequence, tyOpt: result = etySeq
   of tyObject, tyArray, tyTuple, tyOpenArray, tyVarargs:
     result = etyObject
   of tyNil: result = etyNull
-  of tyGenericInst, tyGenericParam, tyGenericBody, tyGenericInvocation,
-     tyNone, tyFromExpr, tyForward, tyEmpty, tyFieldAccessor,
-     tyExpr, tyStmt, tyTypeDesc, tyTypeClasses, tyVoid, tyAlias:
+  of tyGenericParam, tyGenericBody, tyGenericInvocation,
+     tyNone, tyFromExpr, tyForward, tyEmpty,
+     tyExpr, tyStmt, tyTypeDesc, tyBuiltInTypeClass, tyCompositeTypeClass,
+     tyAnd, tyOr, tyNot, tyAnything, tyVoid:
     result = etyNone
-  of tyInferred:
+  of tyGenericInst, tyInferred, tyAlias, tyUserTypeClass, tyUserTypeClassInst,
+     tySink:
     result = mapType(typ.lastSon)
   of tyStatic:
     if t.n != nil: result = mapType(lastSon t)
     else: result = etyNone
   of tyProc: result = etyProc
   of tyCString: result = etyString
-  of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("mapType")
+  of tyUnused, tyOptAsRef: internalError("mapType")
 
 proc mapType(p: PProc; typ: PType): TJSTypeKind =
   if p.target == targetPHP: result = etyObject
@@ -292,7 +294,7 @@ proc useMagic(p: PProc, name: string) =
   if name.len == 0: return
   var s = magicsys.getCompilerProc(name)
   if s != nil:
-    internalAssert s.kind in {skProc, skMethod, skConverter}
+    internalAssert s.kind in {skProc, skFunc, skMethod, skConverter}
     if not p.g.generatedSyms.containsOrIncl(s.id):
       let code = genProc(p, s)
       add(p.g.constants, code)
@@ -376,8 +378,8 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["addInt", "", "addInt($1, $2)", "($1 + $2)"], # AddI
     ["subInt", "", "subInt($1, $2)", "($1 - $2)"], # SubI
     ["mulInt", "", "mulInt($1, $2)", "($1 * $2)"], # MulI
-    ["divInt", "", "divInt($1, $2)", "Math.floor($1 / $2)"], # DivI
-    ["modInt", "", "modInt($1, $2)", "Math.floor($1 % $2)"], # ModI
+    ["divInt", "", "divInt($1, $2)", "Math.trunc($1 / $2)"], # DivI
+    ["modInt", "", "modInt($1, $2)", "Math.trunc($1 % $2)"], # ModI
     ["addInt", "", "addInt($1, $2)", "($1 + $2)"], # Succ
     ["subInt", "", "subInt($1, $2)", "($1 - $2)"], # Pred
     ["", "", "($1 + $2)", "($1 + $2)"], # AddF64
@@ -444,8 +446,8 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["toU32", "toU32", "toU32($1)", "toU32($1)"], # toU32
     ["", "", "$1", "$1"],     # ToFloat
     ["", "", "$1", "$1"],     # ToBiggestFloat
-    ["", "", "Math.floor($1)", "Math.floor($1)"], # ToInt
-    ["", "", "Math.floor($1)", "Math.floor($1)"], # ToBiggestInt
+    ["", "", "Math.trunc($1)", "Math.trunc($1)"], # ToInt
+    ["", "", "Math.trunc($1)", "Math.trunc($1)"], # ToBiggestInt
     ["nimCharToStr", "nimCharToStr", "nimCharToStr($1)", "nimCharToStr($1)"],
     ["nimBoolToStr", "nimBoolToStr", "nimBoolToStr($1)", "nimBoolToStr($1)"],
     ["cstrToNimstr", "cstrToNimstr", "cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")"],
@@ -868,8 +870,8 @@ proc generateHeader(p: PProc, typ: PType): Rope =
         add(result, name)
         add(result, "_Idx")
     elif not (i == 1 and param.name.s == "this"):
-      let k = param.typ.skipTypes({tyGenericInst, tyAlias}).kind
-      if k in {tyVar, tyRef, tyPtr, tyPointer}:
+      let k = param.typ.skipTypes({tyGenericInst, tyAlias, tySink}).kind
+      if k in {tyVar, tyRef, tyPtr, tyLent, tyPointer}:
         add(result, "&")
       add(result, "$")
       add(result, name)
@@ -898,7 +900,7 @@ const
 
 proc needsNoCopy(p: PProc; y: PNode): bool =
   result = (y.kind in nodeKindsNeedNoCopy) or
-      (skipTypes(y.typ, abstractInst).kind in {tyRef, tyPtr, tyVar}) or
+      (skipTypes(y.typ, abstractInst).kind in {tyRef, tyPtr, tyLent, tyVar}) or
       p.target == targetPHP
 
 proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
@@ -927,7 +929,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
 
   # we don't care if it's an etyBaseIndex (global) of a string, it's
   # still a string that needs to be copied properly:
-  if x.typ.skipTypes(abstractInst).kind in {tySequence, tyString}:
+  if x.typ.skipTypes(abstractInst).kind in {tySequence, tyOpt, tyString}:
     xtyp = etySeq
   case xtyp
   of etySeq:
@@ -971,7 +973,7 @@ proc genFastAsgn(p: PProc, n: PNode) =
   # See bug #5933. So we try to be more compatible with the C backend semantics
   # here for 'shallowCopy'. This is an educated guess and might require further
   # changes later:
-  let noCopy = n[0].typ.skipTypes(abstractInst).kind in {tySequence, tyString}
+  let noCopy = n[0].typ.skipTypes(abstractInst).kind in {tySequence, tyOpt, tyString}
   genAsgnAux(p, n.sons[0], n.sons[1], noCopyNeeded=noCopy)
 
 proc genSwap(p: PProc, n: PNode) =
@@ -1067,7 +1069,7 @@ proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) =
       else:
         r.res = "chckIndx($1, $2, strlen($3))-$2" % [b.res, rope(first), a.res]
     else:
-      r.res = "chckIndx($1, $2, $3.length-1)-$2" % [b.res, rope(first), a.res]
+      r.res = "chckIndx($1, $2, $3.length+$2-1)-$2" % [b.res, rope(first), a.res]
   elif first != 0:
     r.res = "($1)-$2" % [b.res, rope(first)]
   else:
@@ -1076,7 +1078,7 @@ proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) =
 
 proc genArrayAccess(p: PProc, n: PNode, r: var TCompRes) =
   var ty = skipTypes(n.sons[0].typ, abstractVarRange)
-  if ty.kind in {tyRef, tyPtr}: ty = skipTypes(ty.lastSon, abstractVarRange)
+  if ty.kind in {tyRef, tyPtr, tyLent}: ty = skipTypes(ty.lastSon, abstractVarRange)
   case ty.kind
   of tyArray, tyOpenArray, tySequence, tyString, tyCString, tyVarargs:
     genArrayAddr(p, n, r)
@@ -1111,7 +1113,7 @@ template isIndirect(x: PSym): bool =
   ({sfAddrTaken, sfGlobal} * v.flags != {} and
     #(mapType(v.typ) != etyObject) and
     {sfImportc, sfVolatile, sfExportc} * v.flags == {} and
-    v.kind notin {skProc, skConverter, skMethod, skIterator,
+    v.kind notin {skProc, skFunc, skConverter, skMethod, skIterator,
                   skConst, skTemp, skLet} and p.target == targetJS)
 
 proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
@@ -1237,7 +1239,7 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
     else:
       r.res = "$" & s.loc.r
       p.declareGlobal(s.id, r.res)
-  of skProc, skConverter, skMethod:
+  of skProc, skFunc, skConverter, skMethod:
     discard mangleName(s, p.target)
     if p.target == targetPHP and r.kind != resCallee:
       r.res = makeJsString($s.loc.r)
@@ -1299,7 +1301,7 @@ proc genArg(p: PProc, n: PNode, param: PSym, r: var TCompRes; emitted: ptr int =
     add(r.res, ", ")
     add(r.res, a.res)
     if emitted != nil: inc emitted[]
-  elif n.typ.kind == tyVar and n.kind in nkCallKinds and mapType(param.typ) == etyBaseIndex:
+  elif n.typ.kind in {tyVar, tyLent} and n.kind in nkCallKinds and mapType(param.typ) == etyBaseIndex:
     # this fixes bug #5608:
     let tmp = getTemp(p)
     add(r.res, "($1 = $2, $1[0]), $1[1]" % [tmp, a.rdLoc])
@@ -1342,6 +1344,9 @@ proc genArgs(p: PProc, n: PNode, r: var TCompRes; start=1) =
 
 proc genOtherArg(p: PProc; n: PNode; i: int; typ: PType;
                  generated: var int; r: var TCompRes) =
+  if i >= n.len:
+    globalError(n.info, "wrong importcpp pattern; expected parameter at position " & $i &
+        " but got only: " & $(n.len-1))
   let it = n[i]
   var paramType: PNode = nil
   if i < sonsLen(typ):
@@ -1363,7 +1368,7 @@ proc genPatternCall(p: PProc; n: PNode; pat: string; typ: PType;
     case pat[i]
     of '@':
       var generated = 0
-      for k in j .. < n.len:
+      for k in j ..< n.len:
         if generated > 0: add(r.res, ", ")
         genOtherArg(p, n, k, typ, generated, r)
       inc i
@@ -1498,7 +1503,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
     result = putToSeq("0", indirect)
   of tyFloat..tyFloat128:
     result = putToSeq("0.0", indirect)
-  of tyRange, tyGenericInst, tyAlias:
+  of tyRange, tyGenericInst, tyAlias, tySink:
     result = createVar(p, lastSon(typ), indirect)
   of tySet:
     result = putToSeq("{}" | "array()", indirect)
@@ -1528,7 +1533,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   of tyTuple:
     if p.target == targetJS:
       result = rope("{")
-      for i in 0.. <t.sonsLen:
+      for i in 0..<t.sonsLen:
         if i > 0: add(result, ", ")
         addf(result, "Field$1: $2", [i.rope,
              createVar(p, t.sons[i], false)])
@@ -1536,7 +1541,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
       if indirect: result = "[$1]" % [result]
     else:
       result = rope("array(")
-      for i in 0.. <t.sonsLen:
+      for i in 0..<t.sonsLen:
         if i > 0: add(result, ", ")
         add(result, createVar(p, t.sons[i], false))
       add(result, ")")
@@ -1545,12 +1550,12 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
     createObjInitList(p, t, initIntSet(), initList)
     result = ("{$1}" | "array($#)") % [initList]
     if indirect: result = "[$1]" % [result]
-  of tyVar, tyPtr, tyRef:
+  of tyVar, tyPtr, tyLent, tyRef:
     if mapType(p, t) == etyBaseIndex:
       result = putToSeq("[null, 0]", indirect)
     else:
       result = putToSeq("null", indirect)
-  of tySequence, tyString, tyCString, tyPointer, tyProc:
+  of tySequence, tyOpt, tyString, tyCString, tyPointer, tyProc:
     result = putToSeq("null", indirect)
   of tyStatic:
     if t.n != nil:
@@ -1562,15 +1567,23 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
     internalError("createVar: " & $t.kind)
     result = nil
 
+template returnType: untyped =
+  ~""
+
 proc genVarInit(p: PProc, v: PSym, n: PNode) =
   var
     a: TCompRes
     s: Rope
+    varCode: string
+  if v.constraint.isNil:
+    varCode = "var $2"
+  else:
+    varCode = v.constraint.strVal
   if n.kind == nkEmpty:
     let mname = mangleName(v, p.target)
-    lineF(p, "var $1 = $2;$n" | "$$$1 = $2;$n",
-             [mname, createVar(p, v.typ, isIndirect(v))])
-    if v.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, v.typ) == etyBaseIndex:
+    lineF(p, varCode & " = $3;$n" | "$$$2 = $3;$n",
+               [returnType, mname, createVar(p, v.typ, isIndirect(v))])
+    if v.typ.kind in {tyVar, tyPtr, tyLent, tyRef} and mapType(p, v.typ) == etyBaseIndex:
       lineF(p, "var $1_Idx = 0;$n", [ mname ])
   else:
     discard mangleName(v, p.target)
@@ -1586,25 +1599,25 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
       let targetBaseIndex = {sfAddrTaken, sfGlobal} * v.flags == {}
       if a.typ == etyBaseIndex:
         if targetBaseIndex:
-          lineF(p, "var $1 = $2, $1_Idx = $3;$n",
-                   [v.loc.r, a.address, a.res])
+          lineF(p, varCode & " = $3, $2_Idx = $4;$n",
+                   [returnType, v.loc.r, a.address, a.res])
         else:
-          lineF(p, "var $1 = [$2, $3];$n",
-                   [v.loc.r, a.address, a.res])
+          lineF(p, varCode & " = [$3, $4];$n",
+                   [returnType, v.loc.r, a.address, a.res])
       else:
         if targetBaseIndex:
           let tmp = p.getTemp
           lineF(p, "var $1 = $2, $3 = $1[0], $3_Idx = $1[1];$n",
                    [tmp, a.res, v.loc.r])
         else:
-          lineF(p, "var $1 = $2;$n", [v.loc.r, a.res])
+          lineF(p, varCode & " = $3;$n", [returnType, v.loc.r, a.res])
       return
     else:
       s = a.res
     if isIndirect(v):
-      lineF(p, "var $1 = [$2];$n", [v.loc.r, s])
+      lineF(p, varCode & " = [$3];$n", [returnType, v.loc.r, s])
     else:
-      lineF(p, "var $1 = $2;$n" | "$$$1 = $2;$n", [v.loc.r, s])
+      lineF(p, varCode & " = $3;$n" | "$$$2 = $3;$n", [returnType, v.loc.r, s])
 
 proc genVarStmt(p: PProc, n: PNode) =
   for i in countup(0, sonsLen(n) - 1):
@@ -1650,7 +1663,7 @@ proc genNewSeq(p: PProc, n: PNode) =
 
 proc genOrd(p: PProc, n: PNode, r: var TCompRes) =
   case skipTypes(n.sons[1].typ, abstractVar).kind
-  of tyEnum, tyInt..tyInt64, tyChar: gen(p, n.sons[1], r)
+  of tyEnum, tyInt..tyUInt64, tyChar: gen(p, n.sons[1], r)
   of tyBool: unaryExpr(p, n, r, "", "($1 ? 1:0)")
   else: internalError(n.info, "genOrd")
 
@@ -1765,7 +1778,7 @@ proc genRepr(p: PProc, n: PNode, r: var TCompRes) =
 
 proc genOf(p: PProc, n: PNode, r: var TCompRes) =
   var x: TCompRes
-  let t = skipTypes(n.sons[2].typ, abstractVarRange+{tyRef, tyPtr, tyTypeDesc})
+  let t = skipTypes(n.sons[2].typ, abstractVarRange+{tyRef, tyPtr, tyLent, tyTypeDesc})
   gen(p, n.sons[1], x)
   if tfFinal in t.flags:
     r.res = "($1.m_type == $2)" % [x.res, genTypeInfo(p, t)]
@@ -2042,10 +2055,10 @@ proc genConv(p: PProc, n: PNode, r: var TCompRes) =
     return
   case dest.kind:
   of tyBool:
-    r.res = "(($1)? 1:0)" % [r.res]
+    r.res = "(!!($1))" % [r.res]
     r.kind = resExpr
   of tyInt:
-    r.res = "($1|0)" % [r.res]
+    r.res = "(($1)|0)" % [r.res]
   else:
     # TODO: What types must we handle here?
     discard
@@ -2152,7 +2165,8 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
     let mname = mangleName(resultSym, p.target)
     let resVar = createVar(p, resultSym.typ, isIndirect(resultSym))
     resultAsgn = p.indentLine(("var $# = $#;$n" | "$$$# = $#;$n") % [mname, resVar])
-    if resultSym.typ.kind in { tyVar, tyPtr, tyRef } and mapType(p, resultSym.typ) == etyBaseIndex:
+    if resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef} and
+        mapType(p, resultSym.typ) == etyBaseIndex:
       resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname])
     gen(p, prc.ast.sons[resultPos], a)
     if mapType(p, resultSym.typ) == etyBaseIndex:
@@ -2161,8 +2175,22 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
       returnStmt = "return $#;$n" % [a.res]
 
   p.nested: genStmt(p, prc.getBody)
-  let def = "function $#($#) {$n$#$#$#$#$#" %
-            [name, header,
+
+  var def: Rope
+  if not prc.constraint.isNil:
+    def = (prc.constraint.strVal & " {$n$#$#$#$#$#") %
+            [ returnType,
+              name,
+              header,
+              optionaLine(p.globals),
+              optionaLine(p.locals),
+              optionaLine(resultAsgn),
+              optionaLine(genProcBody(p, prc)),
+              optionaLine(p.indentLine(returnStmt))]
+  else:
+    def = "function $#($#) {$n$#$#$#$#$#" %
+            [ name,
+              header,
               optionaLine(p.globals),
               optionaLine(p.locals),
               optionaLine(resultAsgn),
@@ -2195,10 +2223,10 @@ proc genCast(p: PProc, n: PNode, r: var TCompRes) =
   if dest.kind == src.kind:
     # no-op conversion
     return
-  let toInt = (dest.kind in tyInt .. tyInt32)
-  let toUint = (dest.kind in tyUInt .. tyUInt32)
-  let fromInt = (src.kind in tyInt .. tyInt32)
-  let fromUint = (src.kind in tyUInt .. tyUInt32)
+  let toInt = (dest.kind in tyInt..tyInt32)
+  let toUint = (dest.kind in tyUInt..tyUInt32)
+  let fromInt = (src.kind in tyInt..tyInt32)
+  let fromUint = (src.kind in tyUInt..tyUInt32)
 
   if toUint and (fromInt or fromUint):
     let trimmer = unsignedTrimmer(dest.size)
@@ -2229,7 +2257,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   case n.kind
   of nkSym:
     genSym(p, n, r)
-  of nkCharLit..nkUInt32Lit:
+  of nkCharLit..nkUInt64Lit:
     if n.typ.kind == tyBool:
       r.res = if n.intVal == 0: rope"false" else: rope"true"
     else:
@@ -2256,11 +2284,17 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     r.kind = resExpr
   of nkFloatLit..nkFloat64Lit:
     let f = n.floatVal
-    if f != f: r.res = rope"NaN"
-    elif f == 0.0: r.res = rope"0.0"
-    elif f == 0.5 * f:
-      if f > 0.0: r.res = rope"Infinity"
-      else: r.res = rope"-Infinity"
+    case classify(f)
+    of fcNaN:
+      r.res = rope"NaN"
+    of fcNegZero:
+      r.res = rope"-0.0"
+    of fcZero:
+      r.res = rope"0.0"
+    of fcInf:
+      r.res = rope"Infinity"
+    of fcNegInf:
+      r.res = rope"-Infinity"
     else: r.res = rope(f.toStrMaxPrecision)
     r.kind = resExpr
   of nkCallKinds:
@@ -2338,7 +2372,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
      nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
      nkFromStmt, nkTemplateDef, nkMacroDef: discard
   of nkPragma: genPragma(p, n)
-  of nkProcDef, nkMethodDef, nkConverterDef:
+  of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef:
     var s = n.sons[namePos].sym
     if {sfExportc, sfCompilerProc} * s.flags == {sfExportc}:
       genSym(p, n.sons[namePos], r)
@@ -2346,6 +2380,8 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkGotoState, nkState:
     internalError(n.info, "first class iterators not implemented")
   of nkPragmaBlock: gen(p, n.lastSon, r)
+  of nkComesFrom:
+    discard "XXX to implement for better stack traces"
   else: internalError(n.info, "gen: unknown node type: " & $n.kind)
 
 var globals: PGlobals
@@ -2360,7 +2396,7 @@ proc genHeader(target: TTarget): Rope =
   if target == targetJS:
     result = (
       "/* Generated by the Nim Compiler v$1 */$n" &
-      "/*   (c) 2017 Andreas Rumpf */$n$n" &
+      "/*   (c) " & copyrightYear & " Andreas Rumpf */$n$n" &
       "var framePtr = null;$n" &
       "var excHandler = 0;$n" &
       "var lastJSError = null;$n" &
@@ -2376,7 +2412,7 @@ proc genHeader(target: TTarget): Rope =
   else:
     result = ("<?php$n" &
               "/* Generated by the Nim Compiler v$1 */$n" &
-              "/*   (c) 2017 Andreas Rumpf */$n$n" &
+              "/*   (c) " & copyrightYear & " Andreas Rumpf */$n$n" &
               "$$framePtr = null;$n" &
               "$$excHandler = 0;$n" &
               "$$lastJSError = null;$n") %
@@ -2434,7 +2470,7 @@ proc genClass(obj: PType; content: Rope; ext: string) =
     else: nil
   let result = ("<?php$n" &
             "/* Generated by the Nim Compiler v$# */$n" &
-            "/*   (c) 2017 Andreas Rumpf */$n$n" &
+            "/*   (c) " & copyrightYear & " Andreas Rumpf */$n$n" &
             "require_once \"nimsystem.php\";$n" &
             "class $#$# {$n$#$n}$n") %
            [rope(VersionAsString), cls, extends, content]

compiler/jstypes.nim

@@ -84,7 +84,7 @@ proc genObjectInfo(p: PProc, typ: PType, name: Rope) =
 
 proc genTupleFields(p: PProc, typ: PType): Rope =
   var s: Rope = nil
-  for i in 0 .. <typ.len:
+  for i in 0 ..< typ.len:
     if i > 0: add(s, ", " & tnl)
     s.addf("{kind: 1, offset: \"Field$1\", len: 0, " &
            "typ: $2, name: \"Field$1\", sons: null}",
@@ -122,7 +122,7 @@ proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
          [name, genTypeInfo(p, typ.sons[0])])
 
 proc genEnumInfoPHP(p: PProc; t: PType): Rope =
-  let t = t.skipTypes({tyGenericInst, tyDistinct, tyAlias})
+  let t = t.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink})
   result = "$$NTI$1" % [rope(t.id)]
   p.declareGlobal(t.id, result)
   if containsOrIncl(p.g.typeInfoGenerated, t.id): return
@@ -141,7 +141,7 @@ proc genEnumInfoPHP(p: PProc; t: PType): Rope =
 proc genTypeInfo(p: PProc, typ: PType): Rope =
   if p.target == targetPHP:
     return makeJSString(typeToString(typ, preferModuleInfo))
-  let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias})
+  let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink})
   result = "NTI$1" % [rope(t.id)]
   if containsOrIncl(p.g.typeInfoGenerated, t.id): return
   case t.kind
@@ -152,7 +152,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
       [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-  of tyVar, tyRef, tyPtr, tySequence, tyRange, tySet:
+  of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]

compiler/lambdalifting.nim

@@ -190,11 +190,11 @@ proc interestingVar(s: PSym): bool {.inline.} =
 
 proc illegalCapture(s: PSym): bool {.inline.} =
   result = skipTypes(s.typ, abstractInst).kind in
-                   {tyVar, tyOpenArray, tyVarargs} or
+                   {tyVar, tyOpenArray, tyVarargs, tyLent} or
       s.kind == skResult
 
 proc isInnerProc(s: PSym): bool =
-  if s.kind in {skProc, skMethod, skConverter, skIterator} and s.magic == mNone:
+  if s.kind in {skProc, skFunc, skMethod, skConverter, skIterator} and s.magic == mNone:
     result = s.skipGenericOwner.kind in routineKinds
 
 proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode =
@@ -371,7 +371,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
   case n.kind
   of nkSym:
     let s = n.sym
-    if s.kind in {skProc, skMethod, skConverter, skIterator} and s.typ != nil and s.typ.callConv == ccClosure:
+    if s.kind in {skProc, skFunc, skMethod, skConverter, skIterator} and
+        s.typ != nil and s.typ.callConv == ccClosure:
       # this handles the case that the inner proc was declared as
       # .closure but does not actually capture anything:
       addClosureParam(c, s, n.info)
@@ -443,7 +444,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     discard
   of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef:
     discard
-  of nkLambdaKinds, nkIteratorDef:
+  of nkLambdaKinds, nkIteratorDef, nkFuncDef:
     if n.typ != nil:
       detectCapturedVars(n[namePos], owner, c)
   else:
@@ -454,6 +455,7 @@ type
   LiftingPass = object
     processed: IntSet
     envVars: Table[int, PNode]
+    inContainer: int
 
 proc initLiftingPass(fn: PSym): LiftingPass =
   result.processed = initIntSet()
@@ -596,6 +598,8 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
 
 proc transformYield(n: PNode; owner: PSym; d: DetectionPass;
                     c: var LiftingPass): PNode =
+  if c.inContainer > 0:
+    localError(n.info, "invalid control flow: 'yield' within a constructor")
   let state = getStateField(owner)
   assert state != nil
   assert state.typ != nil
@@ -702,11 +706,14 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
       if not c.processed.containsOrIncl(s.id):
         #if s.name.s == "temp":
         #  echo renderTree(s.getBody, {renderIds})
+        let oldInContainer = c.inContainer
+        c.inContainer = 0
         let body = wrapIterBody(liftCapturedVars(s.getBody, s, d, c), s)
         if c.envvars.getOrDefault(s.id).isNil:
           s.ast.sons[bodyPos] = body
         else:
           s.ast.sons[bodyPos] = newTree(nkStmtList, rawClosureCreation(s, d, c), body)
+        c.inContainer = oldInContainer
       if s.typ.callConv == ccClosure:
         result = symToClosure(n, owner, d, c)
     elif s.id in d.capturedVars:
@@ -716,7 +723,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         result = accessViaEnvParam(n, owner)
       else:
         result = accessViaEnvVar(n, owner, d, c)
-  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit,
+  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom,
      nkTemplateDef, nkTypeSection:
     discard
   of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef:
@@ -730,11 +737,14 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         # now we know better, so patch it:
         n.sons[0] = x.sons[0]
         n.sons[1] = x.sons[1]
-  of nkLambdaKinds, nkIteratorDef:
+  of nkLambdaKinds, nkIteratorDef, nkFuncDef:
     if n.typ != nil and n[namePos].kind == nkSym:
+      let oldInContainer = c.inContainer
+      c.inContainer = 0
       let m = newSymNode(n[namePos].sym)
       m.typ = n.typ
       result = liftCapturedVars(m, owner, d, c)
+      c.inContainer = oldInContainer
   of nkHiddenStdConv:
     if n.len == 2:
       n.sons[1] = liftCapturedVars(n[1], owner, d, c)
@@ -749,8 +759,12 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         # special case 'when nimVm' due to bug #3636:
         n.sons[1] = liftCapturedVars(n[1], owner, d, c)
         return
+
+    let inContainer = n.kind in {nkObjConstr, nkBracket}
+    if inContainer: inc c.inContainer
     for i in 0..<n.len:
       n.sons[i] = liftCapturedVars(n[i], owner, d, c)
+    if inContainer: dec c.inContainer
 
 # ------------------ old stuff -------------------------------------------
 
@@ -763,7 +777,10 @@ proc semCaptureSym*(s, owner: PSym) =
       var o = owner.skipGenericOwner
       while o.kind != skModule and o != nil:
         if s.owner == o:
-          owner.typ.callConv = ccClosure
+          if owner.typ.callConv in {ccClosure, ccDefault} or owner.kind == skIterator:
+            owner.typ.callConv = ccClosure
+          else:
+            discard "do not produce an error here, but later"
           #echo "computing .closure for ", owner.name.s, " ", owner.info, " because of ", s.name.s
         o = o.skipGenericOwner
     # since the analysis is not entirely correct, we don't set 'tfCapturesEnv'

compiler/lexer.nim

@@ -33,13 +33,13 @@ type
   TTokType* = enum
     tkInvalid, tkEof,         # order is important here!
     tkSymbol, # keywords:
-    tkAddr, tkAnd, tkAs, tkAsm, tkAtomic,
+    tkAddr, tkAnd, tkAs, tkAsm,
     tkBind, tkBlock, tkBreak, tkCase, tkCast,
     tkConcept, tkConst, tkContinue, tkConverter,
     tkDefer, tkDiscard, tkDistinct, tkDiv, tkDo,
     tkElif, tkElse, tkEnd, tkEnum, tkExcept, tkExport,
     tkFinally, tkFor, tkFrom, tkFunc,
-    tkGeneric, tkIf, tkImport, tkIn, tkInclude, tkInterface,
+    tkIf, tkImport, tkIn, tkInclude, tkInterface,
     tkIs, tkIsnot, tkIterator,
     tkLet,
     tkMacro, tkMethod, tkMixin, tkMod, tkNil, tkNot, tkNotin,
@@ -48,7 +48,7 @@ type
     tkShl, tkShr, tkStatic,
     tkTemplate,
     tkTry, tkTuple, tkType, tkUsing,
-    tkVar, tkWhen, tkWhile, tkWith, tkWithout, tkXor,
+    tkVar, tkWhen, tkWhile, tkXor,
     tkYield, # end of keywords
     tkIntLit, tkInt8Lit, tkInt16Lit, tkInt32Lit, tkInt64Lit,
     tkUIntLit, tkUInt8Lit, tkUInt16Lit, tkUInt32Lit, tkUInt64Lit,
@@ -75,12 +75,12 @@ const
   tokKeywordHigh* = pred(tkIntLit)
   TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]",
     "tkSymbol",
-    "addr", "and", "as", "asm", "atomic",
+    "addr", "and", "as", "asm",
     "bind", "block", "break", "case", "cast",
     "concept", "const", "continue", "converter",
     "defer", "discard", "distinct", "div", "do",
     "elif", "else", "end", "enum", "except", "export",
-    "finally", "for", "from", "func", "generic", "if",
+    "finally", "for", "from", "func", "if",
     "import", "in", "include", "interface", "is", "isnot", "iterator",
     "let",
     "macro", "method", "mixin", "mod",
@@ -89,7 +89,7 @@ const
     "shl", "shr", "static",
     "template",
     "try", "tuple", "type", "using",
-    "var", "when", "while", "with", "without", "xor",
+    "var", "when", "while", "xor",
     "yield",
     "tkIntLit", "tkInt8Lit", "tkInt16Lit", "tkInt32Lit", "tkInt64Lit",
     "tkUIntLit", "tkUInt8Lit", "tkUInt16Lit", "tkUInt32Lit", "tkUInt64Lit",
@@ -126,6 +126,10 @@ type
     literal*: string          # the parsed (string) literal; and
                               # documentation comments are here too
     line*, col*: int
+    when defined(nimpretty):
+      offsetA*, offsetB*: int   # used for pretty printing so that literals
+                                # like 0b01 or  r"\L" are unaffected
+      commentOffsetA*, commentOffsetB*: int
 
   TErrorHandler* = proc (info: TLineInfo; msg: TMsgKind; arg: string)
   TLexer* = object of TBaseLexer
@@ -141,10 +145,19 @@ type
     when defined(nimsuggest):
       previousToken: TLineInfo
 
+when defined(nimpretty):
+  var
+    gIndentationWidth*: int
+
 var gLinesCompiled*: int  # all lines that have been compiled
 
 proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
-  newLineInfo(L.fileIdx, tok.line, tok.col)
+  result = newLineInfo(L.fileIdx, tok.line, tok.col)
+  when defined(nimpretty):
+    result.offsetA = tok.offsetA
+    result.offsetB = tok.offsetB
+    result.commentOffsetA = tok.commentOffsetA
+    result.commentOffsetB = tok.commentOffsetB
 
 proc isKeyword*(kind: TTokType): bool =
   result = (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)
@@ -192,6 +205,9 @@ proc initToken*(L: var TToken) =
   L.fNumber = 0.0
   L.base = base10
   L.ident = nil
+  when defined(nimpretty):
+    L.commentOffsetA = 0
+    L.commentOffsetB = 0
 
 proc fillToken(L: var TToken) =
   L.tokType = tkInvalid
@@ -202,6 +218,9 @@ proc fillToken(L: var TToken) =
   L.fNumber = 0.0
   L.base = base10
   L.ident = nil
+  when defined(nimpretty):
+    L.commentOffsetA = 0
+    L.commentOffsetB = 0
 
 proc openLexer*(lex: var TLexer, fileIdx: int32, inputstream: PLLStream;
                  cache: IdentCache) =
@@ -245,11 +264,13 @@ proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
 proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
   result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
 
-template tokenBegin(pos) {.dirty.} =
+template tokenBegin(tok, pos) {.dirty.} =
   when defined(nimsuggest):
     var colA = getColNumber(L, pos)
+  when defined(nimpretty):
+    tok.offsetA = L.offsetBase + pos
 
-template tokenEnd(pos) {.dirty.} =
+template tokenEnd(tok, pos) {.dirty.} =
   when defined(nimsuggest):
     let colB = getColNumber(L, pos)+1
     if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and
@@ -257,8 +278,10 @@ template tokenEnd(pos) {.dirty.} =
       L.cursor = CursorPosition.InToken
       gTrackPos.col = colA.int16
     colA = 0
+  when defined(nimpretty):
+    tok.offsetB = L.offsetBase + pos
 
-template tokenEndIgnore(pos) =
+template tokenEndIgnore(tok, pos) =
   when defined(nimsuggest):
     let colB = getColNumber(L, pos)
     if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and
@@ -266,8 +289,10 @@ template tokenEndIgnore(pos) =
       gTrackPos.fileIndex = trackPosInvalidFileIdx
       gTrackPos.line = -1
     colA = 0
+  when defined(nimpretty):
+    tok.offsetB = L.offsetBase + pos
 
-template tokenEndPrevious(pos) =
+template tokenEndPrevious(tok, pos) =
   when defined(nimsuggest):
     # when we detect the cursor in whitespace, we attach the track position
     # to the token that came before that, but only if we haven't detected
@@ -279,6 +304,8 @@ template tokenEndPrevious(pos) =
       gTrackPos = L.previousToken
       gTrackPosAttached = true
     colA = 0
+  when defined(nimpretty):
+    tok.offsetB = L.offsetBase + pos
 
 {.push overflowChecks: off.}
 # We need to parse the largest uint literal without overflow checks
@@ -363,7 +390,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
   result.literal = ""
   result.base = base10
   startpos = L.bufpos
-  tokenBegin(startPos)
+  tokenBegin(result, startPos)
 
   # First stage: find out base, make verifications, build token literal string
   if L.buf[L.bufpos] == '0' and L.buf[L.bufpos + 1] in baseCodeChars + {'O'}:
@@ -573,7 +600,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
     lexMessageLitNum(L, errInvalidNumber, startpos)
   except OverflowError, RangeError:
     lexMessageLitNum(L, errNumberOutOfRange, startpos)
-  tokenEnd(postPos-1)
+  tokenEnd(result, postPos-1)
   L.bufpos = postPos
 
 proc handleHexChar(L: var TLexer, xi: var int) =
@@ -598,7 +625,15 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
   inc(L.bufpos)               # skip '\'
   case L.buf[L.bufpos]
   of 'n', 'N':
-    if tok.tokType == tkCharLit: lexMessage(L, errNnotAllowedInCharacter)
+    if gOldNewlines:
+      if tok.tokType == tkCharLit: lexMessage(L, errNnotAllowedInCharacter)
+      add(tok.literal, tnl)
+    else:
+      add(tok.literal, '\L')
+    inc(L.bufpos)
+  of 'p', 'P':
+    if tok.tokType == tkCharLit:
+      lexMessage(L, errGenerated, "\\p not allowed in character literal")
     add(tok.literal, tnl)
     inc(L.bufpos)
   of 'r', 'R', 'c', 'C':
@@ -666,7 +701,7 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
 proc newString(s: cstring, len: int): string =
   ## XXX, how come there is no support for this?
   result = newString(len)
-  for i in 0 .. <len:
+  for i in 0 ..< len:
     result[i] = s[i]
 
 proc handleCRLF(L: var TLexer, pos: int): int =
@@ -691,10 +726,11 @@ proc handleCRLF(L: var TLexer, pos: int): int =
   else: result = pos
 
 proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
-  var pos = L.bufpos + 1          # skip "
+  var pos = L.bufpos
   var buf = L.buf                 # put `buf` in a register
   var line = L.lineNumber         # save linenumber for better error message
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
+  inc pos # skip "
   if buf[pos] == '\"' and buf[pos+1] == '\"':
     tok.tokType = tkTripleStrLit # long string literal:
     inc(pos, 2)               # skip ""
@@ -710,18 +746,18 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
       of '\"':
         if buf[pos+1] == '\"' and buf[pos+2] == '\"' and
             buf[pos+3] != '\"':
-          tokenEndIgnore(pos+2)
+          tokenEndIgnore(tok, pos+2)
           L.bufpos = pos + 3 # skip the three """
           break
         add(tok.literal, '\"')
         inc(pos)
       of CR, LF:
-        tokenEndIgnore(pos)
+        tokenEndIgnore(tok, pos)
         pos = handleCRLF(L, pos)
         buf = L.buf
-        add(tok.literal, tnl)
+        add(tok.literal, "\n")
       of nimlexbase.EndOfFile:
-        tokenEndIgnore(pos)
+        tokenEndIgnore(tok, pos)
         var line2 = L.lineNumber
         L.lineNumber = line
         lexMessagePos(L, errClosingTripleQuoteExpected, L.lineStart)
@@ -742,11 +778,11 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
           inc(pos, 2)
           add(tok.literal, '"')
         else:
-          tokenEndIgnore(pos)
+          tokenEndIgnore(tok, pos)
           inc(pos) # skip '"'
           break
       elif c in {CR, LF, nimlexbase.EndOfFile}:
-        tokenEndIgnore(pos)
+        tokenEndIgnore(tok, pos)
         lexMessage(L, errClosingQuoteExpected)
         break
       elif (c == '\\') and not rawMode:
@@ -759,7 +795,7 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
     L.bufpos = pos
 
 proc getCharacter(L: var TLexer, tok: var TToken) =
-  tokenBegin(L.bufpos)
+  tokenBegin(tok, L.bufpos)
   inc(L.bufpos)               # skip '
   var c = L.buf[L.bufpos]
   case c
@@ -769,14 +805,14 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
     tok.literal = $c
     inc(L.bufpos)
   if L.buf[L.bufpos] != '\'': lexMessage(L, errMissingFinalQuote)
-  tokenEndIgnore(L.bufpos)
+  tokenEndIgnore(tok, L.bufpos)
   inc(L.bufpos)               # skip '
 
 proc getSymbol(L: var TLexer, tok: var TToken) =
   var h: Hash = 0
   var pos = L.bufpos
   var buf = L.buf
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
   while true:
     var c = buf[pos]
     case c
@@ -793,7 +829,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
         break
       inc(pos)
     else: break
-  tokenEnd(pos-1)
+  tokenEnd(tok, pos-1)
   h = !$h
   tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
   L.bufpos = pos
@@ -814,7 +850,7 @@ proc endOperator(L: var TLexer, tok: var TToken, pos: int,
 proc getOperator(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
   var buf = L.buf
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
   var h: Hash = 0
   while true:
     var c = buf[pos]
@@ -822,7 +858,7 @@ proc getOperator(L: var TLexer, tok: var TToken) =
     h = h !& ord(c)
     inc(pos)
   endOperator(L, tok, pos, h)
-  tokenEnd(pos-1)
+  tokenEnd(tok, pos-1)
   # advance pos but don't store it in L.bufpos so the next token (which might
   # be an operator too) gets the preceding spaces:
   tok.strongSpaceB = 0
@@ -832,12 +868,29 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   if buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
+proc newlineFollows*(L: var TLexer): bool =
+  var pos = L.bufpos
+  var buf = L.buf
+  while true:
+    case buf[pos]
+    of ' ', '\t':
+      inc(pos)
+    of CR, LF:
+      result = true
+      break
+    of '#':
+      inc(pos)
+      if buf[pos] == '#': inc(pos)
+      if buf[pos] != '[': return true
+    else:
+      break
+
 proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
                           isDoc: bool) =
   var pos = start
   var buf = L.buf
   var toStrip = 0
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
   # detect the amount of indentation:
   if isDoc:
     toStrip = getColNumber(L, pos)
@@ -864,36 +917,37 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
       if isDoc:
         if buf[pos+1] == '#' and buf[pos+2] == '#':
           if nesting == 0:
-            tokenEndIgnore(pos+2)
+            tokenEndIgnore(tok, pos+2)
             inc(pos, 3)
             break
           dec nesting
         tok.literal.add ']'
       elif buf[pos+1] == '#':
         if nesting == 0:
-          tokenEndIgnore(pos+1)
+          tokenEndIgnore(tok, pos+1)
           inc(pos, 2)
           break
         dec nesting
       inc pos
     of CR, LF:
-      tokenEndIgnore(pos)
+      tokenEndIgnore(tok, pos)
       pos = handleCRLF(L, pos)
       buf = L.buf
       # strip leading whitespace:
+      when defined(nimpretty): tok.literal.add "\L"
       if isDoc:
-        tok.literal.add "\n"
+        when not defined(nimpretty): tok.literal.add "\n"
         inc tok.iNumber
         var c = toStrip
         while buf[pos] == ' ' and c > 0:
           inc pos
           dec c
     of nimlexbase.EndOfFile:
-      tokenEndIgnore(pos)
+      tokenEndIgnore(tok, pos)
       lexMessagePos(L, errGenerated, pos, "end of multiline comment expected")
       break
     else:
-      if isDoc: tok.literal.add buf[pos]
+      if isDoc or defined(nimpretty): tok.literal.add buf[pos]
       inc(pos)
   L.bufpos = pos
 
@@ -907,7 +961,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
   if buf[pos+2] == '[':
     skipMultiLineComment(L, tok, pos+3, true)
     return
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
   inc(pos, 2)
 
   var toStrip = 0
@@ -921,7 +975,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
       if buf[pos] == '\\': lastBackslash = pos+1
       add(tok.literal, buf[pos])
       inc(pos)
-    tokenEndIgnore(pos)
+    tokenEndIgnore(tok, pos)
     pos = handleCRLF(L, pos)
     buf = L.buf
     var indent = 0
@@ -940,14 +994,14 @@ proc scanComment(L: var TLexer, tok: var TToken) =
     else:
       if buf[pos] > ' ':
         L.indentAhead = indent
-      tokenEndIgnore(pos)
+      tokenEndIgnore(tok, pos)
       break
   L.bufpos = pos
 
 proc skip(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
   var buf = L.buf
-  tokenBegin(pos)
+  tokenBegin(tok, pos)
   tok.strongSpaceA = 0
   while true:
     case buf[pos]
@@ -958,7 +1012,7 @@ proc skip(L: var TLexer, tok: var TToken) =
       if not L.allowTabs: lexMessagePos(L, errTabulatorsAreNotAllowed, pos)
       inc(pos)
     of CR, LF:
-      tokenEndPrevious(pos)
+      tokenEndPrevious(tok, pos)
       pos = handleCRLF(L, pos)
       buf = L.buf
       var indent = 0
@@ -980,18 +1034,27 @@ proc skip(L: var TLexer, tok: var TToken) =
     of '#':
       # do not skip documentation comment:
       if buf[pos+1] == '#': break
+      when defined(nimpretty):
+        tok.commentOffsetA = L.offsetBase + pos
       if buf[pos+1] == '[':
         skipMultiLineComment(L, tok, pos+2, false)
         pos = L.bufpos
         buf = L.buf
+        when defined(nimpretty):
+          tok.commentOffsetB = L.offsetBase + pos
       else:
-        tokenBegin(pos)
+        tokenBegin(tok, pos)
         while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos)
-        tokenEndIgnore(pos+1)
+        tokenEndIgnore(tok, pos+1)
+        when defined(nimpretty):
+          tok.commentOffsetB = L.offsetBase + pos + 1
     else:
       break                   # EndOfFile also leaves the loop
-  tokenEndPrevious(pos-1)
+  tokenEndPrevious(tok, pos-1)
   L.bufpos = pos
+  when defined(nimpretty):
+    if gIndentationWidth <= 0:
+      gIndentationWidth = tok.indent
 
 proc rawGetTok*(L: var TLexer, tok: var TToken) =
   template atTokenEnd() {.dirty.} =
@@ -1014,7 +1077,7 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
   var c = L.buf[L.bufpos]
   tok.line = L.lineNumber
   tok.col = getColNumber(L, L.bufpos)
-  if c in SymStartChars - {'r', 'R', 'l'}:
+  if c in SymStartChars - {'r', 'R'}:
     getSymbol(L, tok)
   else:
     case c
@@ -1031,12 +1094,6 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
     of ',':
       tok.tokType = tkComma
       inc(L.bufpos)
-    of 'l':
-      # if we parsed exactly one character and its a small L (l), this
-      # is treated as a warning because it may be confused with the number 1
-      if L.buf[L.bufpos+1] notin (SymChars + {'_'}):
-        lexMessage(L, warnSmallLshouldNotBeUsed)
-      getSymbol(L, tok)
     of 'r', 'R':
       if L.buf[L.bufpos + 1] == '\"':
         inc(L.bufpos)

compiler/liftlocals.nim

@@ -0,0 +1,70 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements the '.liftLocals' pragma.
+
+import
+  intsets, strutils, options, ast, astalgo, msgs,
+  idents, renderer, types, lowerings
+
+from pragmas import getPragmaVal
+from wordrecg import wLiftLocals
+
+type
+  Ctx = object
+    partialParam: PSym
+    objType: PType
+
+proc interestingVar(s: PSym): bool {.inline.} =
+  result = s.kind in {skVar, skLet, skTemp, skForVar, skResult} and
+    sfGlobal notin s.flags
+
+proc lookupOrAdd(c: var Ctx; s: PSym; info: TLineInfo): PNode =
+  let field = addUniqueField(c.objType, s)
+  var deref = newNodeI(nkHiddenDeref, info)
+  deref.typ = c.objType
+  add(deref, newSymNode(c.partialParam, info))
+  result = newNodeI(nkDotExpr, info)
+  add(result, deref)
+  add(result, newSymNode(field))
+  result.typ = field.typ
+
+proc liftLocals(n: PNode; i: int; c: var Ctx) =
+  let it = n[i]
+  case it.kind
+  of nkSym:
+    if interestingVar(it.sym):
+      n[i] = lookupOrAdd(c, it.sym, it.info)
+  of procDefs, nkTypeSection: discard
+  else:
+    for i in 0 ..< it.safeLen:
+      liftLocals(it, i, c)
+
+proc lookupParam(params, dest: PNode): PSym =
+  if dest.kind != nkIdent: return nil
+  for i in 1 ..< params.len:
+    if params[i].kind == nkSym and params[i].sym.name.id == dest.ident.id:
+      return params[i].sym
+
+proc liftLocalsIfRequested*(prc: PSym; n: PNode): PNode =
+  let liftDest = getPragmaVal(prc.ast, wLiftLocals)
+  if liftDest == nil: return n
+  let partialParam = lookupParam(prc.typ.n, liftDest)
+  if partialParam.isNil:
+    localError(liftDest.info, "'$1' is not a parameter of '$2'" %
+              [$liftDest, prc.name.s])
+    return n
+  let objType = partialParam.typ.skipTypes(abstractPtrs)
+  if objType.kind != tyObject or tfPartial notin objType.flags:
+    localError(liftDest.info, "parameter '$1' is not a pointer to a partial object" % $liftDest)
+    return n
+  var c = Ctx(partialParam: partialParam, objType: objType)
+  let w = newTree(nkStmtList, n)
+  liftLocals(w, 0, c)
+  result = w[0]

compiler/llstream.nim

@@ -84,7 +84,6 @@ const
   AdditionalLineContinuationOprs = {'#', ':', '='}
 
 proc endsWithOpr*(x: string): bool =
-  # also used by the standard template filter:
   result = x.endsWith(LineContinuationOprs)
 
 proc continueLine(line: string, inTripleString: bool): bool {.inline.} =

compiler/lookups.nim

@@ -39,14 +39,18 @@ proc considerQuotedIdent*(n: PNode, origin: PNode = nil): PIdent =
     of 1: result = considerQuotedIdent(n.sons[0], origin)
     else:
       var id = ""
-      for i in 0.. <n.len:
+      for i in 0..<n.len:
         let x = n.sons[i]
         case x.kind
         of nkIdent: id.add(x.ident.s)
         of nkSym: id.add(x.sym.name.s)
         else: handleError(n, origin)
       result = getIdent(id)
-  of nkOpenSymChoice, nkClosedSymChoice: result = n.sons[0].sym.name
+  of nkOpenSymChoice, nkClosedSymChoice:
+    if n[0].kind == nkSym:
+      result = n.sons[0].sym.name
+    else:
+      handleError(n, origin)
   else:
     handleError(n, origin)
 
@@ -144,8 +148,10 @@ type
 
 proc getSymRepr*(s: PSym): string =
   case s.kind
-  of skProc, skMethod, skConverter, skIterator: result = getProcHeader(s)
-  else: result = s.name.s
+  of skProc, skFunc, skMethod, skConverter, skIterator:
+    result = getProcHeader(s)
+  else:
+    result = s.name.s
 
 proc ensureNoMissingOrUnusedSymbols(scope: PScope) =
   # check if all symbols have been used and defined:
@@ -153,7 +159,7 @@ proc ensureNoMissingOrUnusedSymbols(scope: PScope) =
   var s = initTabIter(it, scope.symbols)
   var missingImpls = 0
   while s != nil:
-    if sfForward in s.flags:
+    if sfForward in s.flags and s.kind != skType:
       # too many 'implementation of X' errors are annoying
       # and slow 'suggest' down:
       if missingImpls == 0:
@@ -286,7 +292,7 @@ proc lookUp*(c: PContext, n: PNode): PSym =
 
 type
   TLookupFlag* = enum
-    checkAmbiguity, checkUndeclared, checkModule
+    checkAmbiguity, checkUndeclared, checkModule, checkPureEnumFields
 
 proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
   const allExceptModule = {low(TSymKind)..high(TSymKind)}-{skModule,skPackage}
@@ -297,6 +303,8 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
       result = searchInScopes(c, ident).skipAlias(n)
     else:
       result = searchInScopes(c, ident, allExceptModule).skipAlias(n)
+    if result == nil and checkPureEnumFields in flags:
+      result = strTableGet(c.pureEnumFields, ident)
     if result == nil and checkUndeclared in flags:
       fixSpelling(n, ident, searchInScopes)
       errorUndeclaredIdentifier(c, n.info, ident.s)
@@ -375,7 +383,11 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         result = errorSym(c, n.sons[1])
   of nkClosedSymChoice, nkOpenSymChoice:
     o.mode = oimSymChoice
-    result = n.sons[0].sym
+    if n[0].kind == nkSym:
+      result = n.sons[0].sym
+    else:
+      o.mode = oimDone
+      return nil
     o.symChoiceIndex = 1
     o.inSymChoice = initIntSet()
     incl(o.inSymChoice, result.id)
@@ -433,13 +445,14 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
 
   if result != nil and result.kind == skStub: loadStub(result)
 
-proc pickSym*(c: PContext, n: PNode; kind: TSymKind;
+proc pickSym*(c: PContext, n: PNode; kinds: set[TSymKind];
               flags: TSymFlags = {}): PSym =
   var o: TOverloadIter
   var a = initOverloadIter(o, c, n)
   while a != nil:
-    if a.kind == kind and flags <= a.flags:
-      return a
+    if a.kind in kinds and flags <= a.flags:
+      if result == nil: result = a
+      else: return nil # ambiguous
     a = nextOverloadIter(o, c, n)
 
 proc isInfixAs*(n: PNode): bool =

compiler/lowerings.nim

@@ -70,6 +70,9 @@ proc newTupleAccessRaw*(tup: PNode, i: int): PNode =
   lit.intVal = i
   addSon(result, lit)
 
+proc newTryFinally*(body, final: PNode): PNode =
+  result = newTree(nkTryStmt, body, newTree(nkFinally, final))
+
 proc lowerTupleUnpackingForAsgn*(n: PNode; owner: PSym): PNode =
   let value = n.lastSon
   result = newNodeI(nkStmtList, n.info)
@@ -139,6 +142,14 @@ proc rawIndirectAccess*(a: PNode; field: PSym; info: TLineInfo): PNode =
   addSon(result, newSymNode(field))
   result.typ = field.typ
 
+proc rawDirectAccess*(obj, field: PSym): PNode =
+  # returns a.field as a node
+  assert field.kind == skField
+  result = newNodeI(nkDotExpr, field.info)
+  addSon(result, newSymNode obj)
+  addSon(result, newSymNode field)
+  result.typ = field.typ
+
 proc lookupInRecord(n: PNode, id: int): PSym =
   result = nil
   case n.kind
@@ -171,8 +182,9 @@ proc addField*(obj: PType; s: PSym) =
   field.position = sonsLen(obj.n)
   addSon(obj.n, newSymNode(field))
 
-proc addUniqueField*(obj: PType; s: PSym) =
-  if lookupInRecord(obj.n, s.id) == nil:
+proc addUniqueField*(obj: PType; s: PSym): PSym {.discardable.} =
+  result = lookupInRecord(obj.n, s.id)
+  if result == nil:
     var field = newSym(skField, getIdent(s.name.s & $obj.n.len), s.owner, s.info)
     field.id = -s.id
     let t = skipIntLit(s.typ)
@@ -180,6 +192,7 @@ proc addUniqueField*(obj: PType; s: PSym) =
     assert t.kind != tyStmt
     field.position = sonsLen(obj.n)
     addSon(obj.n, newSymNode(field))
+    result = field
 
 proc newDotExpr(obj, b: PSym): PNode =
   result = newNodeI(nkDotExpr, obj.info)
@@ -317,7 +330,7 @@ proc typeNeedsNoDeepCopy(t: PType): bool =
   # note that seq[T] is fine, but 'var seq[T]' is not, so we need to skip 'var'
   # for the stricter check and likewise we can skip 'seq' for a less
   # strict check:
-  if t.kind in {tyVar, tySequence}: t = t.sons[0]
+  if t.kind in {tyVar, tyLent, tySequence}: t = t.lastSon
   result = not containsGarbageCollectedRef(t)
 
 proc addLocalVar(varSection, varInit: PNode; owner: PSym; typ: PType;
@@ -452,11 +465,11 @@ proc setupArgsForConcurrency(n: PNode; objType: PType; scratchObj: PSym,
                              varSection, varInit, result: PNode) =
   let formals = n[0].typ.n
   let tmpName = getIdent(genPrefix)
-  for i in 1 .. <n.len:
+  for i in 1 ..< n.len:
     # we pick n's type here, which hopefully is 'tyArray' and not
     # 'tyOpenArray':
     var argType = n[i].typ.skipTypes(abstractInst)
-    if i < formals.len and formals[i].typ.kind == tyVar:
+    if i < formals.len and formals[i].typ.kind in {tyVar, tyLent}:
       localError(n[i].info, "'spawn'ed function cannot have a 'var' parameter")
     #elif containsTyRef(argType):
     #  localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure")
@@ -508,7 +521,7 @@ proc setupArgsForParallelism(n: PNode; objType: PType; scratchObj: PSym;
   let tmpName = getIdent(genPrefix)
   # we need to copy the foreign scratch object fields into local variables
   # for correctness: These are called 'threadLocal' here.
-  for i in 1 .. <n.len:
+  for i in 1 ..< n.len:
     let n = n[i]
     let argType = skipTypes(if i < formals.len: formals[i].typ else: n.typ,
                             abstractInst)
@@ -633,7 +646,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType;
   if fn.kind == nkClosure:
     localError(n.info, "closure in spawn environment is not allowed")
   if not (fn.kind == nkSym and fn.sym.kind in {skProc, skTemplate, skMacro,
-                                               skMethod, skConverter}):
+                                               skFunc, skMethod, skConverter}):
     # for indirect calls we pass the function pointer in the scratchObj
     var argType = n[0].typ.skipTypes(abstractInst)
     var field = newSym(skField, getIdent"fn", owner, n.info)

compiler/main.nim

@@ -16,12 +16,12 @@ import
   cgen, jsgen, json, nversion,
   platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
   docgen2, service, parser, modules, ccgutils, sigmatch, ropes,
-  modulegraphs
+  modulegraphs, tables, rod
 
 from magicsys import systemModule, resetSysTypes
 
 proc rodPass =
-  if optSymbolFiles in gGlobalOptions:
+  if gSymbolFiles in {enabledSf, writeOnlySf}:
     registerPass(rodwritePass)
 
 proc codegenPass =
@@ -36,6 +36,9 @@ proc writeDepsFile(g: ModuleGraph; project: string) =
   for m in g.modules:
     if m != nil:
       f.writeLine(toFullPath(m.position.int32))
+  for k in g.inclToMod.keys:
+    if g.getModule(k).isNil:  # don't repeat includes which are also modules
+      f.writeLine(k.toFullPath)
   f.close()
 
 proc commandGenDepend(graph: ModuleGraph; cache: IdentCache) =
@@ -77,6 +80,12 @@ proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) =
     let proj = changeFileExt(gProjectFull, "")
     extccomp.callCCompiler(proj)
     extccomp.writeJsonBuildInstructions(proj)
+    if optGenScript in gGlobalOptions:
+      writeDepsFile(graph, toGeneratedFile(proj, ""))
+
+proc commandJsonScript(graph: ModuleGraph; cache: IdentCache) =
+  let proj = changeFileExt(gProjectFull, "")
+  extccomp.runJsonBuildInstructions(proj)
 
 proc commandCompileToJS(graph: ModuleGraph; cache: IdentCache) =
   #incl(gGlobalOptions, optSafeCode)
@@ -148,6 +157,7 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
   when SimulateCaasMemReset:
     gGlobalOptions.incl(optCaasEnabled)
 
+  setupModuleCache()
   # In "nim serve" scenario, each command must reset the registered passes
   clearPasses()
   gLastCmdTime = epochTime()
@@ -182,12 +192,12 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
   of "php":
     gCmd = cmdCompileToPHP
     commandCompileToJS(graph, cache)
-  of "doc":
+  of "doc0":
     wantMainModule()
     gCmd = cmdDoc
     loadConfigs(DocConfig, cache)
     commandDoc()
-  of "doc2":
+  of "doc2", "doc":
     gCmd = cmdDoc
     loadConfigs(DocConfig, cache)
     defineSymbol("nimdoc")
@@ -213,6 +223,13 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
     wantMainModule()
     defineSymbol("nimdoc")
     commandDoc2(graph, cache, true)
+  of "ctags":
+    wantMainModule()
+    gCmd = cmdDoc
+    loadConfigs(DocConfig, cache)
+    wantMainModule()
+    defineSymbol("nimdoc")
+    commandTags()
   of "buildindex":
     gCmd = cmdDoc
     loadConfigs(DocConfig, cache)
@@ -266,6 +283,9 @@ proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
   of "nop", "help":
     # prevent the "success" message:
     gCmd = cmdDump
+  of "jsonscript":
+    gCmd = cmdJsonScript
+    commandJsonScript(graph, cache)
   else:
     rawMessage(errInvalidCommandX, command)
 

compiler/modulegraphs.nim

@@ -25,7 +25,7 @@
 ## - Its dependent module stays the same.
 ##
 
-import ast, intsets, tables, options
+import ast, intsets, tables, options, rod
 
 type
   ModuleGraph* = ref object
@@ -81,6 +81,8 @@ proc getModule*(g: ModuleGraph; fileIdx: int32): PSym =
 proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b
 
 proc addDep*(g: ModuleGraph; m: PSym, dep: int32) =
+  assert m.position == m.info.fileIndex
+  addModuleDep(m.info.fileIndex, dep, isIncludeFile = false)
   if suggestMode:
     deps.incl m.position.dependsOn(dep)
     # we compute the transitive closure later when quering the graph lazily.
@@ -88,6 +90,7 @@ proc addDep*(g: ModuleGraph; m: PSym, dep: int32) =
     #invalidTransitiveClosure = true
 
 proc addIncludeDep*(g: ModuleGraph; module, includeFile: int32) =
+  addModuleDep(module, includeFile, isIncludeFile = true)
   discard hasKeyOrPut(inclToMod, includeFile, module)
 
 proc parentModule*(g: ModuleGraph; fileIdx: int32): int32 =

compiler/modulepaths.nim

@@ -0,0 +1,187 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import ast, renderer, strutils, msgs, options, idents, os
+
+import nimblecmd
+
+when false:
+  const
+    considerParentDirs = not defined(noParentProjects)
+    considerNimbleDirs = not defined(noNimbleDirs)
+
+  proc findInNimbleDir(pkg, subdir, dir: string): string =
+    var best = ""
+    var bestv = ""
+    for k, p in os.walkDir(dir, relative=true):
+      if k == pcDir and p.len > pkg.len+1 and
+          p[pkg.len] == '-' and p.startsWith(pkg):
+        let (_, a) = getPathVersion(p)
+        if bestv.len == 0 or bestv < a:
+          bestv = a
+          best = dir / p
+
+    if best.len > 0:
+      var f: File
+      if open(f, best / changeFileExt(pkg, ".nimble-link")):
+        # the second line contains what we're interested in, see:
+        # https://github.com/nim-lang/nimble#nimble-link
+        var override = ""
+        discard readLine(f, override)
+        discard readLine(f, override)
+        close(f)
+        if not override.isAbsolute():
+          best = best / override
+        else:
+          best = override
+    let f = if subdir.len == 0: pkg else: subdir
+    let res = addFileExt(best / f, "nim")
+    if best.len > 0 and fileExists(res):
+      result = res
+
+const stdlibDirs = [
+  "pure", "core", "arch",
+  "pure/collections",
+  "pure/concurrency", "impure",
+  "wrappers", "wrappers/linenoise",
+  "windows", "posix", "js"]
+
+when false:
+  proc resolveDollar(project, source, pkg, subdir: string; info: TLineInfo): string =
+    template attempt(a) =
+      let x = addFileExt(a, "nim")
+      if fileExists(x): return x
+
+    case pkg
+    of "stdlib":
+      if subdir.len == 0:
+        return options.libpath
+      else:
+        for candidate in stdlibDirs:
+          attempt(options.libpath / candidate / subdir)
+    of "root":
+      let root = project.splitFile.dir
+      if subdir.len == 0:
+        return root
+      else:
+        attempt(root / subdir)
+    else:
+      when considerParentDirs:
+        var p = parentDir(source.splitFile.dir)
+        # support 'import $karax':
+        let f = if subdir.len == 0: pkg else: subdir
+
+        while p.len > 0:
+          let dir = p / pkg
+          if dirExists(dir):
+            attempt(dir / f)
+            # 2nd attempt: try to use 'karax/karax'
+            attempt(dir / pkg / f)
+            # 3rd attempt: try to use 'karax/src/karax'
+            attempt(dir / "src" / f)
+            attempt(dir / "src" / pkg / f)
+          p = parentDir(p)
+
+      when considerNimbleDirs:
+        if not options.gNoNimblePath:
+          var nimbleDir = getEnv("NIMBLE_DIR")
+          if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
+          result = findInNimbleDir(pkg, subdir, nimbleDir / "pkgs")
+          if result.len > 0: return result
+          when not defined(windows):
+            result = findInNimbleDir(pkg, subdir, "/opt/nimble/pkgs")
+            if result.len > 0: return result
+
+  proc scriptableImport(pkg, sub: string; info: TLineInfo): string =
+    result = resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info)
+    if result.isNil: result = ""
+
+  proc lookupPackage(pkg, subdir: PNode): string =
+    let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: ""
+    case pkg.kind
+    of nkStrLit, nkRStrLit, nkTripleStrLit:
+      result = scriptableImport(pkg.strVal, sub, pkg.info)
+    of nkIdent:
+      result = scriptableImport(pkg.ident.s, sub, pkg.info)
+    else:
+      localError(pkg.info, "package name must be an identifier or string literal")
+      result = ""
+
+proc getModuleName*(n: PNode): string =
+  # This returns a short relative module name without the nim extension
+  # e.g. like "system", "importer" or "somepath/module"
+  # The proc won't perform any checks that the path is actually valid
+  case n.kind
+  of nkStrLit, nkRStrLit, nkTripleStrLit:
+    try:
+      result = pathSubs(n.strVal, n.info.toFullPath().splitFile().dir)
+    except ValueError:
+      localError(n.info, "invalid path: " & n.strVal)
+      result = n.strVal
+  of nkIdent:
+    result = n.ident.s
+  of nkSym:
+    result = n.sym.name.s
+  of nkInfix:
+    let n0 = n[0]
+    let n1 = n[1]
+    if n0.kind == nkIdent and n0.ident.id == getIdent("as").id:
+      # XXX hack ahead:
+      n.kind = nkImportAs
+      n.sons[0] = n.sons[1]
+      n.sons[1] = n.sons[2]
+      n.sons.setLen(2)
+      return getModuleName(n.sons[0])
+    when false:
+      if n1.kind == nkPrefix and n1[0].kind == nkIdent and n1[0].ident.s == "$":
+        if n0.kind == nkIdent and n0.ident.s == "/":
+          result = lookupPackage(n1[1], n[2])
+        else:
+          localError(n.info, "only '/' supported with $package notation")
+          result = ""
+    else:
+      let modname = getModuleName(n[2])
+      if $n1 == "std":
+        template attempt(a) =
+          let x = addFileExt(a, "nim")
+          if fileExists(x): return x
+        for candidate in stdlibDirs:
+          attempt(options.libpath / candidate / modname)
+
+      # hacky way to implement 'x / y /../ z':
+      result = getModuleName(n1)
+      result.add renderTree(n0, {renderNoComments})
+      result.add modname
+  of nkPrefix:
+    when false:
+      if n.sons[0].kind == nkIdent and n.sons[0].ident.s == "$":
+        result = lookupPackage(n[1], nil)
+      else:
+        discard
+    # hacky way to implement 'x / y /../ z':
+    result = renderTree(n, {renderNoComments}).replace(" ")
+  of nkDotExpr:
+    result = renderTree(n, {renderNoComments}).replace(".", "/")
+  of nkImportAs:
+    result = getModuleName(n.sons[0])
+  else:
+    localError(n.info, errGenerated, "invalid module name: '$1'" % n.renderTree)
+    result = ""
+
+proc checkModuleName*(n: PNode; doLocalError=true): int32 =
+  # This returns the full canonical path for a given module import
+  let modulename = n.getModuleName
+  let fullPath = findModule(modulename, n.info.toFullPath)
+  if fullPath.len == 0:
+    if doLocalError:
+      let m = if modulename.len > 0: modulename else: $n
+      localError(n.info, errCannotOpenFile, m)
+    result = InvalidFileIDX
+  else:
+    result = fullPath.fileInfoIdx

compiler/modules.nim

@@ -10,8 +10,8 @@
 ## Implements the module handling, including the caching of modules.
 
 import
-  ast, astalgo, magicsys, securehash, rodread, msgs, cgendata, sigmatch, options,
-  idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs
+  ast, astalgo, magicsys, std / sha1, rodread, msgs, cgendata, sigmatch, options,
+  idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod
 
 when false:
   type
@@ -125,7 +125,7 @@ proc newModule(graph: ModuleGraph; fileIdx: int32): PSym =
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID.
   new(result)
-  result.id = - 1             # for better error checking
+  result.id = -1             # for better error checking
   result.kind = skModule
   let filename = fileIdx.toFullPath
   result.name = getIdent(splitFile(filename).name)
@@ -169,13 +169,15 @@ proc compileModule*(graph: ModuleGraph; fileIdx: int32; cache: IdentCache, flags
     if sfMainModule in result.flags:
       gMainPackageId = result.owner.id
 
-    if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}:
-      rd = handleSymbolFile(result, cache)
-      if result.id < 0:
-        internalError("handleSymbolFile should have set the module's ID")
-        return
-    else:
-      result.id = getID()
+    when false:
+      if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}:
+        rd = handleSymbolFile(result, cache)
+        if result.id < 0:
+          internalError("handleSymbolFile should have set the module's ID")
+          return
+      else:
+        discard
+    result.id = getModuleId(fileIdx, toFullPath(fileIdx))
     discard processModule(graph, result,
       if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil,
       rd, cache)

compiler/msgs.nim

@@ -26,7 +26,8 @@ type
     errAtPopWithoutPush, errEmptyAsm, errInvalidIndentation,
     errExceptionExpected, errExceptionAlreadyHandled,
     errYieldNotAllowedHere, errYieldNotAllowedInTryStmt,
-    errInvalidNumberOfYieldExpr, errCannotReturnExpr, errAttemptToRedefine,
+    errInvalidNumberOfYieldExpr, errCannotReturnExpr,
+    errNoReturnWithReturnTypeNotAllowed, errAttemptToRedefine,
     errStmtInvalidAfterReturn, errStmtExpected, errInvalidLabel,
     errInvalidCmdLineOption, errCmdLineArgExpected, errCmdLineNoArgExpected,
     errInvalidVarSubstitution, errUnknownVar, errUnknownCcompiler,
@@ -61,7 +62,7 @@ type
     errBaseTypeMustBeOrdinal, errInheritanceOnlyWithNonFinalObjects,
     errInheritanceOnlyWithEnums, errIllegalRecursionInTypeX,
     errCannotInstantiateX, errExprHasNoAddress, errXStackEscape,
-    errVarForOutParamNeeded,
+    errVarForOutParamNeededX,
     errPureTypeMismatch, errTypeMismatch, errButExpected, errButExpectedX,
     errAmbiguousCallXYZ, errWrongNumberOfArguments,
     errWrongNumberOfArgumentsInCall,
@@ -179,8 +180,9 @@ const
     errYieldNotAllowedInTryStmt: "'yield' cannot be used within 'try' in a non-inlined iterator",
     errInvalidNumberOfYieldExpr: "invalid number of \'yield\' expressions",
     errCannotReturnExpr: "current routine cannot return an expression",
+    errNoReturnWithReturnTypeNotAllowed: "routines with NoReturn pragma are not allowed to have return type",
     errAttemptToRedefine: "redefinition of \'$1\'",
-    errStmtInvalidAfterReturn: "statement not allowed after \'return\', \'break\', \'raise\' or \'continue'",
+    errStmtInvalidAfterReturn: "statement not allowed after \'return\', \'break\', \'raise\', \'continue\' or proc call with noreturn pragma",
     errStmtExpected: "statement expected",
     errInvalidLabel: "\'$1\' is no label",
     errInvalidCmdLineOption: "invalid command line option: \'$1\'",
@@ -268,9 +270,9 @@ const
     errCannotInstantiateX: "cannot instantiate: \'$1\'",
     errExprHasNoAddress: "expression has no address",
     errXStackEscape: "address of '$1' may not escape its stack frame",
-    errVarForOutParamNeeded: "for a \'var\' type a variable needs to be passed",
+    errVarForOutParamNeededX: "for a \'var\' type a variable needs to be passed; but '$1' is immutable",
     errPureTypeMismatch: "type mismatch",
-    errTypeMismatch: "type mismatch: got (",
+    errTypeMismatch: "type mismatch: got <",
     errButExpected: "but expected one of: ",
     errButExpectedX: "but expected \'$1\'",
     errAmbiguousCallXYZ: "ambiguous call; both $1 and $2 match for: $3",
@@ -368,7 +370,7 @@ const
     errXhasSideEffects: "\'$1\' can have side effects",
     errIteratorExpected: "iterator within for loop context expected",
     errLetNeedsInit: "'let' symbol requires an initialization",
-    errThreadvarCannotInit: "a thread var cannot be initialized explicitly",
+    errThreadvarCannotInit: "a thread var cannot be initialized explicitly; this would only run for the main thread",
     errWrongSymbolX: "usage of \'$1\' is a user-defined error",
     errIllegalCaptureX: "illegal capture '$1'",
     errXCannotBeClosure: "'$1' cannot have 'closure' calling convention",
@@ -489,6 +491,7 @@ type
     dirtyfile: string          # the file that is actually read into memory
                                # and parsed; usually 'nil' but is used
                                # for 'nimsuggest'
+    hash*: string              # the checksum of the file
 
   TLineInfo* = object          # This is designed to be as small as possible,
                                # because it is used
@@ -498,6 +501,9 @@ type
                                # only 8 bytes.
     line*, col*: int16
     fileIndex*: int32
+    when defined(nimpretty):
+      offsetA*, offsetB*: int
+      commentOffsetA*, commentOffsetB*: int
 
   TErrorOutput* = enum
     eStdOut
@@ -714,6 +720,14 @@ proc setDirtyFile*(fileIdx: int32; filename: string) =
   assert fileIdx >= 0
   fileInfos[fileIdx].dirtyFile = filename
 
+proc setHash*(fileIdx: int32; hash: string) =
+  assert fileIdx >= 0
+  shallowCopy(fileInfos[fileIdx].hash, hash)
+
+proc getHash*(fileIdx: int32): string =
+  assert fileIdx >= 0
+  shallowCopy(result, fileInfos[fileIdx].hash)
+
 proc toFullPathConsiderDirty*(fileIdx: int32): string =
   if fileIdx < 0:
     result = "???"
@@ -1020,6 +1034,9 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
   handleError(msg, eh, s)
 
 proc fatal*(info: TLineInfo, msg: TMsgKind, arg = "") =
+  # this fixes bug #7080 so that it is at least obvious 'fatal'
+  # was executed.
+  errorOutputs = {eStdOut, eStdErr}
   liMessage(info, msg, arg, doAbort)
 
 proc globalError*(info: TLineInfo, msg: TMsgKind, arg = "") =

compiler/nim.nim

@@ -119,4 +119,6 @@ condsyms.initDefines()
 
 when not defined(selftest):
   handleCmdLine(newIdentCache(), newConfigRef())
+  when declared(GC_setMaxPause):
+    echo GC_getStatistics()
   msgQuit(int8(msgs.gErrorCounter > 0))

compiler/nimblecmd.nim

@@ -16,11 +16,11 @@ proc addPath*(path: string, info: TLineInfo) =
     options.searchPaths.insert(path, 0)
 
 type
-  Version = distinct string
+  Version* = distinct string
 
-proc `$`(ver: Version): string {.borrow.}
+proc `$`*(ver: Version): string {.borrow.}
 
-proc newVersion(ver: string): Version =
+proc newVersion*(ver: string): Version =
   doAssert(ver.len == 0 or ver[0] in {'#', '\0'} + Digits,
            "Wrong version: " & ver)
   return Version(ver)
@@ -28,7 +28,11 @@ proc newVersion(ver: string): Version =
 proc isSpecial(ver: Version): bool =
   return ($ver).len > 0 and ($ver)[0] == '#'
 
-proc `<`(ver: Version, ver2: Version): bool =
+proc isValidVersion(v: string): bool =
+  if v.len > 0:
+    if v[0] in {'#'} + Digits: return true
+
+proc `<`*(ver: Version, ver2: Version): bool =
   ## This is synced from Nimble's version module.
 
   # Handling for special versions such as "#head" or "#branch".
@@ -72,15 +76,23 @@ proc getPathVersion*(p: string): tuple[name, version: string] =
     result.name = p
     return
 
+  for i in sepIdx..<p.len:
+    if p[i] in {DirSep, AltSep}:
+      result.name = p
+      return
+
   result.name = p[0 .. sepIdx - 1]
   result.version = p.substr(sepIdx + 1)
 
-proc addPackage(packages: StringTableRef, p: string) =
+proc addPackage(packages: StringTableRef, p: string; info: TLineInfo) =
   let (name, ver) = getPathVersion(p)
-  let version = newVersion(ver)
-  if packages.getOrDefault(name).newVersion < version or
-     (not packages.hasKey(name)):
-    packages[name] = $version
+  if isValidVersion(ver):
+    let version = newVersion(ver)
+    if packages.getOrDefault(name).newVersion < version or
+      (not packages.hasKey(name)):
+      packages[name] = $version
+  else:
+    localError(info, "invalid package name: " & p)
 
 iterator chosen(packages: StringTableRef): string =
   for key, val in pairs(packages):
@@ -109,7 +121,7 @@ proc addPathRec(dir: string, info: TLineInfo) =
   if dir[pos] in {DirSep, AltSep}: inc(pos)
   for k,p in os.walkDir(dir):
     if k == pcDir and p[pos] != '.':
-      addPackage(packages, p)
+      addPackage(packages, p, info)
   for p in packages.chosen:
     addNimblePath(p, info)
 

compiler/nimlexbase.nim

@@ -46,6 +46,7 @@ type
                               # private data:
     sentinel*: int
     lineStart*: int           # index of last line start in buffer
+    offsetBase*: int          # use ``offsetBase + bufpos`` to get the offset
 
 
 proc openBaseLexer*(L: var TBaseLexer, inputstream: PLLStream,
@@ -122,7 +123,8 @@ proc fillBaseLexer(L: var TBaseLexer, pos: int): int =
     result = pos + 1          # nothing to do
   else:
     fillBuffer(L)
-    L.bufpos = 0              # XXX: is this really correct?
+    L.offsetBase += pos + 1
+    L.bufpos = 0
     result = 0
   L.lineStart = result
 
@@ -146,6 +148,7 @@ proc skipUTF8BOM(L: var TBaseLexer) =
 proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen = 8192) =
   assert(bufLen > 0)
   L.bufpos = 0
+  L.offsetBase = 0
   L.bufLen = bufLen
   L.buf = cast[cstring](alloc(bufLen * chrSize))
   L.sentinel = bufLen - 1

compiler/nimsets.nim

@@ -151,6 +151,11 @@ proc complement*(a: PNode): PNode =
   for i in countup(0, high(x)): x[i] = not x[i]
   result = toTreeSet(x, a.typ, a.info)
 
+proc deduplicate*(a: PNode): PNode =
+  var x: TBitSet
+  toBitSet(a, x)
+  result = toTreeSet(x, a.typ, a.info)
+
 proc cardSet(s: PNode): BiggestInt =
   # here we can do better than converting it into a compact set
   # we just count the elements directly

compiler/nversion.nim

@@ -13,5 +13,5 @@
 const
   MaxSetElements* = 1 shl 16  # (2^16) to support unicode character sets?
   VersionAsString* = system.NimVersion
-  RodFileVersion* = "1222"       # modify this if the rod-format changes!
+  RodFileVersion* = "1223"       # modify this if the rod-format changes!
 

compiler/options.nim

@@ -18,6 +18,7 @@ const
   newScopeForIf* = true
   useCaas* = not defined(noCaas)
   noTimeMachine* = defined(avoidTimeMachine) and defined(macosx)
+  copyrightYear* = "2018"
 
 type                          # please make sure we have under 32 options
                               # (improves code efficiency a lot!)
@@ -48,7 +49,6 @@ type                          # please make sure we have under 32 options
     optGenScript,             # generate a script file to compile the *.c files
     optGenMapping,            # generate a mapping file
     optRun,                   # run the compiled project
-    optSymbolFiles,           # use symbol files for speeding up compilation
     optCaasEnabled            # compiler-as-a-service is running
     optSkipConfigFile,        # skip the general config file
     optSkipProjConfigFile,    # skip the project's config file
@@ -91,7 +91,8 @@ type
     cmdRst2html,              # convert a reStructuredText file to HTML
     cmdRst2tex,               # convert a reStructuredText file to TeX
     cmdInteractive,           # start interactive session
-    cmdRun                    # run the project via TCC backend
+    cmdRun,                   # run the project via TCC backend
+    cmdJsonScript             # compile a .json build file
   TStringSeq* = seq[string]
   TGCMode* = enum             # the selected GC
     gcNone, gcBoehm, gcGo, gcRegions, gcMarkAndSweep, gcRefc,
@@ -114,6 +115,7 @@ proc cppDefine*(c: ConfigRef; define: string) =
 
 var
   gIdeCmd*: IdeCmd
+  gOldNewlines*: bool
 
 const
   ChecksOptions* = {optObjCheck, optFieldCheck, optRangeCheck, optNilCheck,
@@ -140,15 +142,25 @@ var
   gEvalExpr* = ""             # expression for idetools --eval
   gLastCmdTime*: float        # when caas is enabled, we measure each command
   gListFullPaths*: bool
-  isServing*: bool = false
+  gPreciseStack*: bool = false
   gNoNimblePath* = false
   gExperimentalMode*: bool
+  newDestructors*: bool
+  gDynlibOverrideAll*: bool
+
+type
+  SymbolFilesOption* = enum
+    disabledSf, enabledSf, writeOnlySf, readOnlySf, v2Sf
+
+var gSymbolFiles*: SymbolFilesOption
 
 proc importantComments*(): bool {.inline.} = gCmd in {cmdDoc, cmdIdeTools}
 proc usesNativeGC*(): bool {.inline.} = gSelectedGC >= gcRefc
+template preciseStack*(): bool = gPreciseStack
 
 template compilationCachePresent*: untyped =
-  {optCaasEnabled, optSymbolFiles} * gGlobalOptions != {}
+  gSymbolFiles in {enabledSf, writeOnlySf}
+#  {optCaasEnabled, optSymbolFiles} * gGlobalOptions != {}
 
 template optPreserveOrigSource*: untyped =
   optEmbedOrigSrc in gGlobalOptions
@@ -159,6 +171,7 @@ const
   RodExt* = "rod"
   HtmlExt* = "html"
   JsonExt* = "json"
+  TagsExt* = "tags"
   TexExt* = "tex"
   IniExt* = "ini"
   DefaultConfig* = "nim.cfg"
@@ -423,7 +436,7 @@ proc inclDynlibOverride*(lib: string) =
   gDllOverrides[lib.canonDynlibName] = "true"
 
 proc isDynlibOverride*(lib: string): bool =
-  result = gDllOverrides.hasKey(lib.canonDynlibName)
+  result = gDynlibOverrideAll or gDllOverrides.hasKey(lib.canonDynlibName)
 
 proc binaryStrSearch*(x: openArray[string], y: string): int =
   var a = 0

compiler/parampatterns.nim

@@ -122,7 +122,7 @@ proc semNodeKindConstraints*(p: PNode): PNode =
   result.strVal = newStringOfCap(10)
   result.strVal.add(chr(aqNone.ord))
   if p.len >= 2:
-    for i in 1.. <p.len:
+    for i in 1..<p.len:
       compileConstraints(p.sons[i], result.strVal)
     if result.strVal.len > MaxStackSize-1:
       internalError(p.info, "parameter pattern too complex")
@@ -152,7 +152,7 @@ proc checkForSideEffects*(n: PNode): TSideEffectAnalysis =
       # indirect call: assume side effect:
       return seSideEffect
     # we need to check n[0] too: (FwithSideEffectButReturnsProcWithout)(args)
-    for i in 0 .. <n.len:
+    for i in 0 ..< n.len:
       let ret = checkForSideEffects(n.sons[i])
       if ret == seSideEffect: return ret
       elif ret == seUnknown and result == seNoSideEffect:
@@ -163,7 +163,7 @@ proc checkForSideEffects*(n: PNode): TSideEffectAnalysis =
   else:
     # assume no side effect:
     result = seNoSideEffect
-    for i in 0 .. <n.len:
+    for i in 0 ..< n.len:
       let ret = checkForSideEffects(n.sons[i])
       if ret == seSideEffect: return ret
       elif ret == seUnknown and result == seNoSideEffect:
@@ -222,7 +222,10 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
     elif compareTypes(n.typ, n.sons[1].typ, dcEqIgnoreDistinct):
       # types that are equal modulo distinction preserve l-value:
       result = isAssignable(owner, n.sons[1], isUnsafeAddr)
-  of nkHiddenDeref, nkDerefExpr, nkHiddenAddr:
+  of nkHiddenDeref:
+    if n[0].typ.kind == tyLent: result = arDiscriminant
+    else: result = arLValue
+  of nkDerefExpr, nkHiddenAddr:
     result = arLValue
   of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
     result = isAssignable(owner, n.sons[0], isUnsafeAddr)

compiler/parser.nim

@@ -28,7 +28,7 @@ import
   llstream, lexer, idents, strutils, ast, astalgo, msgs
 
 type
-  TParser*{.final.} = object   # A TParser object represents a file that
+  TParser* = object            # A TParser object represents a file that
                                # is being parsed
     currInd: int               # current indentation level
     firstTok, strongSpaces: bool # Has the first token been read?
@@ -72,6 +72,7 @@ proc parseStmtPragma(p: var TParser): PNode
 proc parsePragma(p: var TParser): PNode
 proc postExprBlocks(p: var TParser, x: PNode): PNode
 proc parseExprStmt(p: var TParser): PNode
+proc parseBlock(p: var TParser): PNode
 # implementation
 
 proc getTok(p: var TParser) =
@@ -255,13 +256,6 @@ proc isUnary(p: TParser): bool =
      p.tok.strongSpaceB == 0 and
      p.tok.strongSpaceA > 0:
       result = true
-      # versions prior to 0.13.0 used to do this:
-      when false:
-        if p.strongSpaces:
-          result = true
-        else:
-          parMessage(p, warnDeprecated,
-            "will be parsed as unary operator; inconsistent spacing")
 
 proc checkBinary(p: TParser) {.inline.} =
   ## Check if the current parser token is a binary operator.
@@ -700,12 +694,7 @@ proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode =
   result = r
 
   template somePar() =
-    if p.tok.strongSpaceA > 0:
-      if p.strongSpaces:
-        break
-      else:
-        parMessage(p, warnDeprecated,
-          "a [b] will be parsed as command syntax; spacing")
+    if p.tok.strongSpaceA > 0: break
   # progress guaranteed
   while p.tok.indent < 0 or
        (p.tok.tokType == tkDot and p.tok.indent >= baseIndent):
@@ -797,21 +786,58 @@ proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode =
   #|          'else' colcom expr
   #| ifExpr = 'if' condExpr
   #| whenExpr = 'when' condExpr
-  result = newNodeP(kind, p)
-  while true:
-    getTok(p)                 # skip `if`, `elif`
-    var branch = newNodeP(nkElifExpr, p)
+  when true:
+    result = newNodeP(kind, p)
+    while true:
+      getTok(p)                 # skip `if`, `when`, `elif`
+      var branch = newNodeP(nkElifExpr, p)
+      optInd(p, branch)
+      addSon(branch, parseExpr(p))
+      colcom(p, branch)
+      addSon(branch, parseStmt(p))
+      skipComment(p, branch)
+      addSon(result, branch)
+      if p.tok.tokType != tkElif: break # or not sameOrNoInd(p): break
+    if p.tok.tokType == tkElse: # and sameOrNoInd(p):
+      var branch = newNodeP(nkElseExpr, p)
+      eat(p, tkElse)
+      colcom(p, branch)
+      addSon(branch, parseStmt(p))
+      addSon(result, branch)
+  else:
+    var
+      b: PNode
+      wasIndented = false
+    result = newNodeP(kind, p)
+
+    getTok(p)
+    let branch = newNodeP(nkElifExpr, p)
     addSon(branch, parseExpr(p))
     colcom(p, branch)
+    let oldInd = p.currInd
+    if realInd(p):
+      p.currInd = p.tok.indent
+      wasIndented = true
+      echo result.info, " yes ", p.currInd
     addSon(branch, parseExpr(p))
-    optInd(p, branch)
-    addSon(result, branch)
-    if p.tok.tokType != tkElif: break
-  var branch = newNodeP(nkElseExpr, p)
-  eat(p, tkElse)
-  colcom(p, branch)
-  addSon(branch, parseExpr(p))
-  addSon(result, branch)
+    result.add branch
+    while sameInd(p) or not wasIndented:
+      case p.tok.tokType
+      of tkElif:
+        b = newNodeP(nkElifExpr, p)
+        getTok(p)
+        optInd(p, b)
+        addSon(b, parseExpr(p))
+      of tkElse:
+        b = newNodeP(nkElseExpr, p)
+        getTok(p)
+      else: break
+      colcom(p, b)
+      addSon(b, parseStmt(p))
+      addSon(result, b)
+      if b.kind == nkElseExpr: break
+    if wasIndented:
+      p.currInd = oldInd
 
 proc parsePragma(p: var TParser): PNode =
   #| pragma = '{.' optInd (exprColonExpr comma?)* optPar ('.}' | '}')
@@ -888,7 +914,7 @@ proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode =
     optInd(p, result)
     addSon(result, parseTypeDesc(p))
   else:
-    addSon(result, ast.emptyNode)
+    addSon(result, newNodeP(nkEmpty, p))
     if p.tok.tokType != tkEquals and withBothOptional notin flags:
       parMessage(p, errColonOrEqualsExpected, p.tok)
   if p.tok.tokType == tkEquals:
@@ -896,7 +922,7 @@ proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode =
     optInd(p, result)
     addSon(result, parseExpr(p))
   else:
-    addSon(result, ast.emptyNode)
+    addSon(result, newNodeP(nkEmpty, p))
 
 proc parseTuple(p: var TParser, indentAllowed = false): PNode =
   #| inlTupleDecl = 'tuple'
@@ -991,7 +1017,7 @@ proc parseDoBlock(p: var TParser; info: TLineInfo): PNode =
   if params.kind != nkEmpty:
     result = newProcNode(nkDo, info, result, params = params, pragmas = pragmas)
 
-proc parseProcExpr(p: var TParser, isExpr: bool): PNode =
+proc parseProcExpr(p: var TParser; isExpr: bool; kind: TNodeKind): PNode =
   #| procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)?
   # either a proc type or a anonymous proc
   let info = parLineInfo(p)
@@ -1002,7 +1028,7 @@ proc parseProcExpr(p: var TParser, isExpr: bool): PNode =
   if p.tok.tokType == tkEquals and isExpr:
     getTok(p)
     skipComment(p, result)
-    result = newProcNode(nkLambda, info, parseStmt(p),
+    result = newProcNode(kind, info, parseStmt(p),
                          params = params,
                          pragmas = pragmas)
   else:
@@ -1014,7 +1040,7 @@ proc parseProcExpr(p: var TParser, isExpr: bool): PNode =
 proc isExprStart(p: TParser): bool =
   case p.tok.tokType
   of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf,
-     tkProc, tkIterator, tkBind, tkAddr,
+     tkProc, tkFunc, tkIterator, tkBind, tkAddr,
      tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr,
      tkTuple, tkObject, tkType, tkWhen, tkCase, tkOut:
     result = true
@@ -1038,21 +1064,29 @@ proc parseTypeDescKAux(p: var TParser, kind: TNodeKind,
   optInd(p, result)
   if not isOperator(p.tok) and isExprStart(p):
     addSon(result, primary(p, mode))
-  if kind == nkDistinctTy and p.tok.tokType in {tkWith, tkWithout}:
-    let nodeKind = if p.tok.tokType == tkWith: nkWith
-                   else: nkWithout
+  if kind == nkDistinctTy and p.tok.tokType == tkSymbol:
+    # XXX document this feature!
+    var nodeKind: TNodeKind
+    if p.tok.ident.s == "with":
+      nodeKind = nkWith
+    elif p.tok.ident.s == "without":
+      nodeKind = nkWithout
+    else:
+      return result
     getTok(p)
     let list = newNodeP(nodeKind, p)
     result.addSon list
     parseSymbolList(p, list)
 
 proc parseExpr(p: var TParser): PNode =
-  #| expr = (ifExpr
+  #| expr = (blockExpr
+  #|       | ifExpr
   #|       | whenExpr
   #|       | caseExpr
   #|       | tryExpr)
   #|       / simpleExpr
   case p.tok.tokType:
+  of tkBlock: result = parseBlock(p)
   of tkIf: result = parseIfExpr(p, nkIfExpr)
   of tkWhen: result = parseIfExpr(p, nkWhenExpr)
   of tkCase: result = parseCase(p)
@@ -1088,21 +1122,12 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
 
   case p.tok.tokType:
   of tkTuple: result = parseTuple(p, mode == pmTypeDef)
-  of tkProc: result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef})
+  of tkProc: result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}, nkLambda)
+  of tkFunc: result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}, nkFuncDef)
   of tkIterator:
-    when false:
-      if mode in {pmTypeDesc, pmTypeDef}:
-        result = parseProcExpr(p, false)
-        result.kind = nkIteratorTy
-      else:
-        # no anon iterators for now:
-        parMessage(p, errExprExpected, p.tok)
-        getTok(p)  # we must consume a token here to prevend endless loops!
-        result = ast.emptyNode
-    else:
-      result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef})
-      if result.kind == nkLambda: result.kind = nkIteratorDef
-      else: result.kind = nkIteratorTy
+    result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}, nkLambda)
+    if result.kind == nkLambda: result.kind = nkIteratorDef
+    else: result.kind = nkIteratorTy
   of tkEnum:
     if mode == pmTypeDef:
       result = parseEnum(p)
@@ -1115,13 +1140,9 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
     else:
       result = newNodeP(nkObjectTy, p)
       getTok(p)
-  of tkGeneric, tkConcept:
+  of tkConcept:
     if mode == pmTypeDef:
-      let wasGeneric = p.tok.tokType == tkGeneric
       result = parseTypeClass(p)
-      # hack so that it's remembered and can be marked as deprecated in
-      # sem'check:
-      if wasGeneric: result.flags.incl nfBase2
     else:
       parMessage(p, errInvalidToken, p.tok)
   of tkStatic:
@@ -1495,6 +1516,7 @@ proc parseFor(p: var TParser): PNode =
 
 proc parseBlock(p: var TParser): PNode =
   #| blockStmt = 'block' symbol? colcom stmt
+  #| blockExpr = 'block' symbol? colcom stmt
   result = newNodeP(nkBlockStmt, p)
   getTokNoInd(p)
   if p.tok.tokType == tkColon: addSon(result, ast.emptyNode)
@@ -1920,7 +1942,7 @@ proc parseVariable(p: var TParser): PNode =
   #| variable = (varTuple / identColonEquals) colonBody? indAndComment
   if p.tok.tokType == tkParLe: result = parseVarTuple(p)
   else: result = parseIdentColonEquals(p, {withPragma, withDot})
-  result{-1} = postExprBlocks(p, result{-1})
+  result[^1] = postExprBlocks(p, result[^1])
   indAndComment(p, result)
 
 proc parseBind(p: var TParser, k: TNodeKind): PNode =
@@ -2000,6 +2022,7 @@ proc complexOrSimpleStmt(p: var TParser): PNode =
   of tkDefer: result = parseStaticOrDefer(p, nkDefer)
   of tkAsm: result = parseAsm(p)
   of tkProc: result = parseRoutine(p, nkProcDef)
+  of tkFunc: result = parseRoutine(p, nkFuncDef)
   of tkMethod: result = parseRoutine(p, nkMethodDef)
   of tkIterator: result = parseRoutine(p, nkIteratorDef)
   of tkMacro: result = parseRoutine(p, nkMacroDef)
@@ -2050,14 +2073,19 @@ proc parseStmt(p: var TParser): PNode =
         if a.kind != nkEmpty:
           addSon(result, a)
         else:
-          parMessage(p, errExprExpected, p.tok)
-          getTok(p)
+          # This is done to make the new 'if' expressions work better.
+          # XXX Eventually we need to be more strict here.
+          if p.tok.tokType notin {tkElse, tkElif}:
+            parMessage(p, errExprExpected, p.tok)
+            getTok(p)
+          else:
+            break
         if not p.hasProgress and p.tok.tokType == tkEof: break
   else:
     # the case statement is only needed for better error messages:
     case p.tok.tokType
-    of tkIf, tkWhile, tkCase, tkTry, tkFor, tkBlock, tkAsm, tkProc, tkIterator,
-       tkMacro, tkType, tkConst, tkWhen, tkVar:
+    of tkIf, tkWhile, tkCase, tkTry, tkFor, tkBlock, tkAsm, tkProc, tkFunc,
+       tkIterator, tkMacro, tkType, tkConst, tkWhen, tkVar:
       parMessage(p, errComplexStmtRequiresInd)
       result = ast.emptyNode
     else:

compiler/passes.nim

@@ -7,17 +7,18 @@
 #    distribution, for details about the copyright.
 #
 
-# This module implements the passes functionality. A pass must implement the
-# `TPass` interface.
+## This module implements the passes functionality. A pass must implement the
+## `TPass` interface.
 
 import
   strutils, options, ast, astalgo, llstream, msgs, platform, os,
   condsyms, idents, renderer, types, extccomp, math, magicsys, nversion,
-  nimsets, syntaxes, times, rodread, idgen, modulegraphs, reorder
+  nimsets, syntaxes, times, rodread, idgen, modulegraphs, reorder, rod
+
 
 type
   TPassContext* = object of RootObj # the pass's context
-    fromCache*: bool  # true if created by "openCached"
+    rd*: PRodReader  # != nil if created by "openCached"
 
   PPassContext* = ref TPassContext
 
@@ -28,7 +29,8 @@ type
   TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}
 
   TPass* = tuple[open: TPassOpen, openCached: TPassOpenCached,
-                 process: TPassProcess, close: TPassClose]
+                 process: TPassProcess, close: TPassClose,
+                 isFrontend: bool]
 
   TPassData* = tuple[input: PNode, closeOutput: PNode]
   TPasses* = openArray[TPass]
@@ -40,11 +42,13 @@ type
 proc makePass*(open: TPassOpen = nil,
                openCached: TPassOpenCached = nil,
                process: TPassProcess = nil,
-               close: TPassClose = nil): TPass =
+               close: TPassClose = nil,
+               isFrontend = false): TPass =
   result.open = open
   result.openCached = openCached
   result.close = close
   result.process = process
+  result.isFrontend = isFrontend
 
 # the semantic checker needs these:
 var
@@ -64,7 +68,7 @@ proc astNeeded*(s: PSym): bool =
   # needs to be stored. The passes manager frees s.sons[codePos] when
   # appropriate to free the procedure body's memory. This is important
   # to keep memory usage down.
-  if (s.kind in {skMethod, skProc}) and
+  if (s.kind in {skMethod, skProc, skFunc}) and
       ({sfCompilerProc, sfCompileTime} * s.flags == {}) and
       (s.typ.callConv != ccInline) and
       (s.ast.sons[genericParamsPos].kind == nkEmpty):
@@ -117,7 +121,7 @@ proc openPassesCached(g: ModuleGraph; a: var TPassContextArray, module: PSym,
     if not isNil(gPasses[i].openCached):
       a[i] = gPasses[i].openCached(g, module, rd)
       if a[i] != nil:
-        a[i].fromCache = true
+        a[i].rd = rd
     else:
       a[i] = nil
 
@@ -177,7 +181,36 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
     a: TPassContextArray
     s: PLLStream
     fileIdx = module.fileIdx
-  if rd == nil:
+  if module.id < 0:
+    # new module caching mechanism:
+    for i in 0..<gPassesLen:
+      if not isNil(gPasses[i].open) and not gPasses[i].isFrontend:
+        a[i] = gPasses[i].open(graph, module, cache)
+      else:
+        a[i] = nil
+
+    var stmtIndex = 0
+    var doContinue = true
+    while doContinue:
+      let n = loadNode(module, stmtIndex)
+      if n == nil or graph.stopCompile(): break
+      #if n.kind == nkImportStmt:
+      #  echo "yes and it's ", n
+      inc stmtIndex
+      var m = n
+      for i in 0..<gPassesLen:
+        if not isNil(gPasses[i].process) and not gPasses[i].isFrontend:
+          m = gPasses[i].process(a[i], m)
+          if isNil(m):
+            doContinue = false
+            break
+
+    var m: PNode = nil
+    for i in 0..<gPassesLen:
+      if not isNil(gPasses[i].close) and not gPasses[i].isFrontend:
+        m = gPasses[i].close(graph, a[i], m)
+      a[i] = nil
+  elif rd == nil:
     openPasses(graph, a, module, cache)
     if stream == nil:
       let filename = fileIdx.toFullPathConsiderDirty
@@ -211,7 +244,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
             if n.kind == nkEmpty: break
             sl.add n
           if sfReorder in module.flags:
-            sl = reorder sl
+            sl = reorder(graph, sl, module, cache)
           discard processTopLevelStmt(sl, a)
           break
         elif not processTopLevelStmt(n, a): break

compiler/patterns.nim

@@ -63,7 +63,7 @@ proc sameTrees(a, b: PNode): bool =
 
 proc inSymChoice(sc, x: PNode): bool =
   if sc.kind == nkClosedSymChoice:
-    for i in 0.. <sc.len:
+    for i in 0..<sc.len:
       if sc.sons[i].sym == x.sym: return true
   elif sc.kind == nkOpenSymChoice:
     # same name suffices for open sym choices!
@@ -83,7 +83,7 @@ proc isPatternParam(c: PPatternContext, p: PNode): bool {.inline.} =
   result = p.kind == nkSym and p.sym.kind == skParam and p.sym.owner == c.owner
 
 proc matchChoice(c: PPatternContext, p, n: PNode): bool =
-  for i in 1 .. <p.len:
+  for i in 1 ..< p.len:
     if matches(c, p.sons[i], n): return true
 
 proc bindOrCheck(c: PPatternContext, param: PSym, n: PNode): bool =
@@ -115,7 +115,7 @@ proc matchNested(c: PPatternContext, p, n: PNode, rpn: bool): bool =
       if rpn: arglist.add(n.sons[0])
     elif n.kind == nkHiddenStdConv and n.sons[1].kind == nkBracket:
       let n = n.sons[1]
-      for i in 0.. <n.len:
+      for i in 0..<n.len:
         if not matchStarAux(c, op, n[i], arglist, rpn): return false
     elif checkTypes(c, p.sons[2].sym, n):
       add(arglist, n)
@@ -186,7 +186,7 @@ proc matches(c: PPatternContext, p, n: PNode): bool =
             # unpack varargs:
             let n = lastSon(n).sons[1]
             arglist = newNodeI(nkArgList, n.info, n.len)
-            for i in 0.. <n.len: arglist.sons[i] = n.sons[i]
+            for i in 0..<n.len: arglist.sons[i] = n.sons[i]
           else:
             arglist = newNodeI(nkArgList, n.info, sonsLen(n) - plen + 1)
             # f(1, 2, 3)
@@ -206,7 +206,7 @@ proc matches(c: PPatternContext, p, n: PNode): bool =
 
 proc matchStmtList(c: PPatternContext, p, n: PNode): PNode =
   proc matchRange(c: PPatternContext, p, n: PNode, i: int): bool =
-    for j in 0 .. <p.len:
+    for j in 0 ..< p.len:
       if not matches(c, p.sons[j], n.sons[i+j]):
         # we need to undo any bindings:
         if not isNil(c.mapping): c.mapping = nil
@@ -229,7 +229,7 @@ proc matchStmtList(c: PPatternContext, p, n: PNode): PNode =
 
 proc aliasAnalysisRequested(params: PNode): bool =
   if params.len >= 2:
-    for i in 1 .. < params.len:
+    for i in 1 ..< params.len:
       let param = params.sons[i].sym
       if whichAlias(param) != aqNone: return true
 
@@ -237,7 +237,7 @@ proc addToArgList(result, n: PNode) =
   if n.typ != nil and n.typ.kind != tyStmt:
     if n.kind != nkArgList: result.add(n)
     else:
-      for i in 0 .. <n.len: result.add(n.sons[i])
+      for i in 0 ..< n.len: result.add(n.sons[i])
 
 proc applyRule*(c: PContext, s: PSym, n: PNode): PNode =
   ## returns a tree to semcheck if the rule triggered; nil otherwise
@@ -256,7 +256,7 @@ proc applyRule*(c: PContext, s: PSym, n: PNode): PNode =
   var args: PNode
   if requiresAA:
     args = newNodeI(nkArgList, n.info)
-  for i in 1 .. < params.len:
+  for i in 1 ..< params.len:
     let param = params.sons[i].sym
     let x = getLazy(ctx, param)
     # couldn't bind parameter:
@@ -265,7 +265,7 @@ proc applyRule*(c: PContext, s: PSym, n: PNode): PNode =
     if requiresAA: addToArgList(args, x)
   # perform alias analysis here:
   if requiresAA:
-    for i in 1 .. < params.len:
+    for i in 1 ..< params.len:
       var rs = result.sons[i]
       let param = params.sons[i].sym
       case whichAlias(param)

compiler/pbraces.nim

@@ -906,9 +906,14 @@ proc parseTypeDescKAux(p: var TParser, kind: TNodeKind,
   optInd(p, result)
   if not isOperator(p.tok) and isExprStart(p):
     addSon(result, primary(p, mode))
-  if kind == nkDistinctTy and p.tok.tokType in {tkWith, tkWithout}:
-    let nodeKind = if p.tok.tokType == tkWith: nkWith
-                   else: nkWithout
+  if kind == nkDistinctTy and p.tok.tokType == tkSymbol:
+    var nodeKind: TNodeKind
+    if p.tok.ident.s == "with":
+      nodeKind = nkWith
+    elif p.tok.ident.s == "without":
+      nodeKind = nkWithout
+    else:
+      return result
     getTok(p)
     let list = newNodeP(nodeKind, p)
     result.addSon list

compiler/pluginsupport.nim

@@ -7,8 +7,9 @@
 #    distribution, for details about the copyright.
 #
 
-## Plugin support for the Nim compiler. Right now they
-## need to be build with the compiler, no DLL support.
+## Plugin support for the Nim compiler. Right now plugins
+## need to be built with the compiler only: plugins using 
+## DLLs or the FFI will not work.
 
 import ast, semdata, idents
 

compiler/pragmas.nim

@@ -21,17 +21,17 @@ const
 const
   procPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl,
     wMagic, wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader,
-    wCompilerproc, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge,
+    wCompilerProc, wCore, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge,
     wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC,
-    wAsmNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl,
+    wAsmNoStackFrame, wError, wDiscardable, wNoInit, wCodegenDecl,
     wGensym, wInject, wRaises, wTags, wLocks, wDelegator, wGcSafe,
-    wOverride, wConstructor, wExportNims, wUsed}
+    wOverride, wConstructor, wExportNims, wUsed, wLiftLocals}
   converterPragmas* = procPragmas
   methodPragmas* = procPragmas+{wBase}-{wImportCpp}
   templatePragmas* = {wImmediate, wDeprecated, wError, wGensym, wInject, wDirty,
-    wDelegator, wExportNims, wUsed}
+    wDelegator, wExportNims, wUsed, wPragma}
   macroPragmas* = {FirstCallConv..LastCallConv, wImmediate, wImportc, wExportc,
-    wNodecl, wMagic, wNosideeffect, wCompilerproc, wDeprecated, wExtern,
+    wNodecl, wMagic, wNosideeffect, wCompilerProc, wCore, wDeprecated, wExtern,
     wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wDelegator,
     wExportNims, wUsed}
   iteratorPragmas* = {FirstCallConv..LastCallConv, wNosideeffect, wSideeffect,
@@ -52,14 +52,14 @@ const
     wDeprecated, wExtern, wThread, wImportCpp, wImportObjC, wAsmNoStackFrame,
     wRaises, wLocks, wTags, wGcSafe}
   typePragmas* = {wImportc, wExportc, wDeprecated, wMagic, wAcyclic, wNodecl,
-    wPure, wHeader, wCompilerproc, wFinal, wSize, wExtern, wShallow,
+    wPure, wHeader, wCompilerProc, wCore, wFinal, wSize, wExtern, wShallow,
     wImportCpp, wImportObjC, wError, wIncompleteStruct, wByCopy, wByRef,
     wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked,
-    wBorrow, wGcSafe, wExportNims, wPartial, wUsed, wExplain}
+    wBorrow, wGcSafe, wExportNims, wPartial, wUsed, wExplain, wPackage}
   fieldPragmas* = {wImportc, wExportc, wDeprecated, wExtern,
     wImportCpp, wImportObjC, wError, wGuard, wBitsize, wUsed}
   varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl,
-    wMagic, wHeader, wDeprecated, wCompilerproc, wDynlib, wExtern,
+    wMagic, wHeader, wDeprecated, wCompilerProc, wCore, wDynlib, wExtern,
     wImportCpp, wImportObjC, wError, wNoInit, wCompileTime, wGlobal,
     wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims, wUsed}
   constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl,
@@ -70,6 +70,14 @@ const
                       wThread, wRaises, wLocks, wTags, wGcSafe}
   allRoutinePragmas* = methodPragmas + iteratorPragmas + lambdaPragmas
 
+proc getPragmaVal*(procAst: PNode; name: TSpecialWord): PNode =
+  let p = procAst[pragmasPos]
+  if p.kind == nkEmpty: return nil
+  for it in p:
+    if it.kind in nkPragmaCallKinds and it.len == 2 and it[0].kind == nkIdent and
+        it[0].ident.id == ord(name):
+      return it[1]
+
 proc pragma*(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords)
 # implementation
 
@@ -81,7 +89,7 @@ proc pragmaAsm*(c: PContext, n: PNode): char =
   if n != nil:
     for i in countup(0, sonsLen(n) - 1):
       let it = n.sons[i]
-      if it.kind == nkExprColonExpr and it.sons[0].kind == nkIdent:
+      if it.kind in nkPragmaCallKinds and it.len == 2 and it.sons[0].kind == nkIdent:
         case whichKeyword(it.sons[0].ident)
         of wSubsChar:
           if it.sons[1].kind == nkCharLit: result = chr(int(it.sons[1].intVal))
@@ -143,7 +151,7 @@ proc newEmptyStrNode(n: PNode): PNode {.noinline.} =
   result.strVal = ""
 
 proc getStrLitNode(c: PContext, n: PNode): PNode =
-  if n.kind != nkExprColonExpr:
+  if n.kind notin nkPragmaCallKinds or n.len != 2:
     localError(n.info, errStringLiteralExpected)
     # error correction:
     result = newEmptyStrNode(n)
@@ -160,7 +168,7 @@ proc expectStrLit(c: PContext, n: PNode): string =
   result = getStrLitNode(c, n).strVal
 
 proc expectIntLit(c: PContext, n: PNode): int =
-  if n.kind != nkExprColonExpr:
+  if n.kind notin nkPragmaCallKinds or n.len != 2:
     localError(n.info, errIntLiteralExpected)
   else:
     n.sons[1] = c.semConstExpr(c, n.sons[1])
@@ -169,7 +177,7 @@ proc expectIntLit(c: PContext, n: PNode): int =
     else: localError(n.info, errIntLiteralExpected)
 
 proc getOptionalStr(c: PContext, n: PNode, defaultStr: string): string =
-  if n.kind == nkExprColonExpr: result = expectStrLit(c, n)
+  if n.kind in nkPragmaCallKinds: result = expectStrLit(c, n)
   else: result = defaultStr
 
 proc processCodegenDecl(c: PContext, n: PNode, sym: PSym) =
@@ -178,7 +186,7 @@ proc processCodegenDecl(c: PContext, n: PNode, sym: PSym) =
 proc processMagic(c: PContext, n: PNode, s: PSym) =
   #if sfSystemModule notin c.module.flags:
   #  liMessage(n.info, errMagicOnlyInSystem)
-  if n.kind != nkExprColonExpr:
+  if n.kind notin nkPragmaCallKinds or n.len != 2:
     localError(n.info, errStringLiteralExpected)
     return
   var v: string
@@ -196,7 +204,7 @@ proc wordToCallConv(sw: TSpecialWord): TCallingConvention =
   result = TCallingConvention(ord(ccDefault) + ord(sw) - ord(wNimcall))
 
 proc isTurnedOn(c: PContext, n: PNode): bool =
-  if n.kind == nkExprColonExpr:
+  if n.kind in nkPragmaCallKinds and n.len == 2:
     let x = c.semConstBoolExpr(c, n.sons[1])
     n.sons[1] = x
     if x.kind == nkIntLit: return x.intVal != 0
@@ -215,7 +223,7 @@ proc pragmaNoForward(c: PContext, n: PNode; flag=sfNoForward) =
   else: excl(c.module.flags, flag)
 
 proc processCallConv(c: PContext, n: PNode) =
-  if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent):
+  if n.kind in nkPragmaCallKinds and n.len == 2 and n.sons[1].kind == nkIdent:
     var sw = whichKeyword(n.sons[1].ident)
     case sw
     of FirstCallConv..LastCallConv:
@@ -236,7 +244,7 @@ proc getLib(c: PContext, kind: TLibKind, path: PNode): PLib =
     result.isOverriden = options.isDynlibOverride(path.strVal)
 
 proc expectDynlibNode(c: PContext, n: PNode): PNode =
-  if n.kind != nkExprColonExpr:
+  if n.kind notin nkPragmaCallKinds or n.len != 2:
     localError(n.info, errStringLiteralExpected)
     # error correction:
     result = newEmptyStrNode(n)
@@ -256,7 +264,7 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym) =
     if not lib.isOverriden:
       c.optionStack[^1].dynlib = lib
   else:
-    if n.kind == nkExprColonExpr:
+    if n.kind in nkPragmaCallKinds:
       var lib = getLib(c, libDynamic, expectDynlibNode(c, n))
       if not lib.isOverriden:
         addToLib(lib, sym)
@@ -271,7 +279,7 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym) =
       sym.typ.callConv = ccCDecl
 
 proc processNote(c: PContext, n: PNode) =
-  if (n.kind == nkExprColonExpr) and (sonsLen(n) == 2) and
+  if (n.kind in nkPragmaCallKinds) and (sonsLen(n) == 2) and
       (n.sons[0].kind == nkBracketExpr) and
       (n.sons[0].sons.len == 2) and
       (n.sons[0].sons[1].kind == nkIdent) and
@@ -299,7 +307,7 @@ proc processNote(c: PContext, n: PNode) =
     invalidPragma(n)
 
 proc processOption(c: PContext, n: PNode): bool =
-  if n.kind != nkExprColonExpr: result = true
+  if n.kind notin nkPragmaCallKinds or n.len != 2: result = true
   elif n.sons[0].kind == nkBracketExpr: processNote(c, n)
   elif n.sons[0].kind != nkIdent: result = true
   else:
@@ -347,8 +355,8 @@ proc processOption(c: PContext, n: PNode): bool =
     else: result = true
 
 proc processPush(c: PContext, n: PNode, start: int) =
-  if n.sons[start-1].kind == nkExprColonExpr:
-    localError(n.info, errGenerated, "':' after 'push' not supported")
+  if n.sons[start-1].kind in nkPragmaCallKinds:
+    localError(n.info, errGenerated, "'push' can't have arguments")
   var x = newOptionEntry()
   var y = c.optionStack[^1]
   x.options = gOptions
@@ -373,14 +381,14 @@ proc processPop(c: PContext, n: PNode) =
     c.optionStack.setLen(c.optionStack.len - 1)
 
 proc processDefine(c: PContext, n: PNode) =
-  if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent):
+  if (n.kind in nkPragmaCallKinds and n.len == 2) and (n.sons[1].kind == nkIdent):
     defineSymbol(n.sons[1].ident.s)
     message(n.info, warnDeprecated, "define")
   else:
     invalidPragma(n)
 
 proc processUndef(c: PContext, n: PNode) =
-  if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent):
+  if (n.kind in nkPragmaCallKinds and n.len == 2) and (n.sons[1].kind == nkIdent):
     undefSymbol(n.sons[1].ident.s)
     message(n.info, warnDeprecated, "undef")
   else:
@@ -412,7 +420,7 @@ proc processCompile(c: PContext, n: PNode) =
       localError(n.info, errStringLiteralExpected)
       result = ""
 
-  let it = if n.kind == nkExprColonExpr: n.sons[1] else: n
+  let it = if n.kind in nkPragmaCallKinds and n.len == 2: n.sons[1] else: n
   if it.kind == nkPar and it.len == 2:
     let s = getStrLit(c, it, 0)
     let dest = getStrLit(c, it, 1)
@@ -445,7 +453,7 @@ proc pragmaBreakpoint(c: PContext, n: PNode) =
   discard getOptionalStr(c, n, "")
 
 proc pragmaWatchpoint(c: PContext, n: PNode) =
-  if n.kind == nkExprColonExpr:
+  if n.kind in nkPragmaCallKinds and n.len == 2:
     n.sons[1] = c.semExpr(c, n.sons[1])
   else:
     invalidPragma(n)
@@ -486,7 +494,7 @@ proc semAsmOrEmit*(con: PContext, n: PNode, marker: char): PNode =
     result = newNode(nkAsmStmt, n.info)
 
 proc pragmaEmit(c: PContext, n: PNode) =
-  if n.kind != nkExprColonExpr:
+  if n.kind notin nkPragmaCallKinds or n.len != 2:
     localError(n.info, errStringLiteralExpected)
   else:
     let n1 = n[1]
@@ -504,12 +512,12 @@ proc pragmaEmit(c: PContext, n: PNode) =
         localError(n.info, errStringLiteralExpected)
 
 proc noVal(n: PNode) =
-  if n.kind == nkExprColonExpr: invalidPragma(n)
+  if n.kind in nkPragmaCallKinds and n.len > 1: invalidPragma(n)
 
 proc pragmaUnroll(c: PContext, n: PNode) =
   if c.p.nestedLoopCounter <= 0:
     invalidPragma(n)
-  elif n.kind == nkExprColonExpr:
+  elif n.kind in nkPragmaCallKinds and n.len == 2:
     var unrollFactor = expectIntLit(c, n)
     if unrollFactor <% 32:
       n.sons[1] = newIntNode(nkIntLit, unrollFactor)
@@ -517,10 +525,11 @@ proc pragmaUnroll(c: PContext, n: PNode) =
       invalidPragma(n)
 
 proc pragmaLine(c: PContext, n: PNode) =
-  if n.kind == nkExprColonExpr:
+  if n.kind in nkPragmaCallKinds and n.len == 2:
     n.sons[1] = c.semConstExpr(c, n.sons[1])
     let a = n.sons[1]
     if a.kind == nkPar:
+      # unpack the tuple
       var x = a.sons[0]
       var y = a.sons[1]
       if x.kind == nkExprColonExpr: x = x.sons[1]
@@ -540,15 +549,13 @@ proc pragmaLine(c: PContext, n: PNode) =
     n.info = getInfoContext(-1)
 
 proc processPragma(c: PContext, n: PNode, i: int) =
-  var it = n.sons[i]
-  if it.kind != nkExprColonExpr: invalidPragma(n)
-  elif it.sons[0].kind != nkIdent: invalidPragma(n)
-  elif it.sons[1].kind != nkIdent: invalidPragma(n)
+  let it = n[i]
+  if it.kind notin nkPragmaCallKinds and it.len == 2: invalidPragma(n)
+  elif it[0].kind != nkIdent: invalidPragma(n)
+  elif it[1].kind != nkIdent: invalidPragma(n)
 
-  var userPragma = newSym(skTemplate, it.sons[1].ident, nil, it.info)
-  var body = newNodeI(nkPragma, n.info)
-  for j in i+1 .. sonsLen(n)-1: addSon(body, n.sons[j])
-  userPragma.ast = body
+  var userPragma = newSym(skTemplate, it[1].ident, nil, it.info)
+  userPragma.ast = newNode(nkPragma, n.info, n.sons[i+1..^1])
   strTableAdd(c.userPragmas, userPragma)
 
 proc pragmaRaisesOrTags(c: PContext, n: PNode) =
@@ -558,7 +565,7 @@ proc pragmaRaisesOrTags(c: PContext, n: PNode) =
       localError(x.info, errGenerated, "invalid type for raises/tags list")
     x.typ = t
 
-  if n.kind == nkExprColonExpr:
+  if n.kind in nkPragmaCallKinds and n.len == 2:
     let it = n.sons[1]
     if it.kind notin {nkCurly, nkBracket}:
       processExc(c, it)
@@ -568,18 +575,18 @@ proc pragmaRaisesOrTags(c: PContext, n: PNode) =
     invalidPragma(n)
 
 proc pragmaLockStmt(c: PContext; it: PNode) =
-  if it.kind != nkExprColonExpr:
+  if it.kind notin nkPragmaCallKinds or it.len != 2:
     invalidPragma(it)
   else:
     let n = it[1]
     if n.kind != nkBracket:
       localError(n.info, errGenerated, "locks pragma takes a list of expressions")
     else:
-      for i in 0 .. <n.len:
+      for i in 0 ..< n.len:
         n.sons[i] = c.semExpr(c, n.sons[i])
 
 proc pragmaLocks(c: PContext, it: PNode): TLockLevel =
-  if it.kind != nkExprColonExpr:
+  if it.kind notin nkPragmaCallKinds or it.len != 2:
     invalidPragma(it)
   else:
     case it[1].kind
@@ -596,7 +603,7 @@ proc pragmaLocks(c: PContext, it: PNode): TLockLevel =
         result = TLockLevel(x)
 
 proc typeBorrow(sym: PSym, n: PNode) =
-  if n.kind == nkExprColonExpr:
+  if n.kind in nkPragmaCallKinds and n.len == 2:
     let it = n.sons[1]
     if it.kind != nkAccQuoted:
       localError(n.info, "a type can only borrow `.` for now")
@@ -616,9 +623,10 @@ proc deprecatedStmt(c: PContext; pragma: PNode) =
   if pragma.kind != nkBracket:
     localError(pragma.info, "list of key:value pairs expected"); return
   for n in pragma:
-    if n.kind in {nkExprColonExpr, nkExprEqExpr}:
+    if n.kind in nkPragmaCallKinds and n.len == 2:
       let dest = qualifiedLookUp(c, n[1], {checkUndeclared})
-      assert dest != nil
+      if dest == nil or dest.kind in routineKinds:
+        localError(n.info, warnUser, "the .deprecated pragma is unreliable for routines")
       let src = considerQuotedIdent(n[0])
       let alias = newSym(skAlias, src, dest, n[0].info)
       incl(alias.flags, sfExported)
@@ -629,7 +637,7 @@ proc deprecatedStmt(c: PContext; pragma: PNode) =
       localError(n.info, "key:value pair expected")
 
 proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym =
-  if it.kind != nkExprColonExpr:
+  if it.kind notin nkPragmaCallKinds or it.len != 2:
     invalidPragma(it); return
   let n = it[1]
   if n.kind == nkSym:
@@ -646,23 +654,49 @@ proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym =
   else:
     result = qualifiedLookUp(c, n, {checkUndeclared})
 
-proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
+proc semCustomPragma(c: PContext, n: PNode): PNode =
+  if n.kind == nkIdent:
+    result = newTree(nkCall, n)
+  elif n.kind == nkExprColonExpr:
+    # pragma: arg -> pragma(arg)
+    result = newTree(nkCall, n[0], n[1])
+  elif n.kind in nkPragmaCallKinds + {nkIdent}:
+    result = n
+  else:
+    invalidPragma(n)
+    return n
+
+  let r = c.semOverloadedCall(c, result, n, {skTemplate}, {})
+  if r.isNil or sfCustomPragma notin r[0].sym.flags:
+    invalidPragma(n)
+  else:
+    result = r
+    if n.kind == nkIdent:
+      result = result[0]
+    elif n.kind == nkExprColonExpr:
+      result.kind = n.kind # pragma(arg) -> pragma: arg
+
+proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
                   validPragmas: TSpecialWords): bool =
   var it = n.sons[i]
-  var key = if it.kind == nkExprColonExpr: it.sons[0] else: it
+  var key = if it.kind in nkPragmaCallKinds and it.len > 1: it.sons[0] else: it
   if key.kind == nkBracketExpr:
     processNote(c, it)
     return
+  elif key.kind notin nkIdentKinds:
+    n.sons[i] = semCustomPragma(c, it)
+    return
   let ident = considerQuotedIdent(key)
   var userPragma = strTableGet(c.userPragmas, ident)
   if userPragma != nil:
+    # number of pragmas increase/decrease with user pragma expansion
     inc c.instCounter
     if c.instCounter > 100:
       globalError(it.info, errRecursiveDependencyX, userPragma.name.s)
+    
     pragma(c, sym, userPragma.ast, validPragmas)
-    # ensure the pragma is also remember for generic instantiations in other
-    # modules:
-    n.sons[i] = userPragma.ast
+    n.sons[i..i] = userPragma.ast.sons # expand user pragma with its content
+    i.inc(userPragma.ast.len - 1) # inc by -1 is ok, user pragmas was empty
     dec c.instCounter
   else:
     var k = whichKeyword(ident)
@@ -750,10 +784,6 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         incl(sym.loc.flags, lfNoDecl)
         # implies nodecl, because otherwise header would not make sense
         if sym.loc.r == nil: sym.loc.r = rope(sym.name.s)
-      of wDestructor:
-        sym.flags.incl sfOverriden
-        if sym.name.s.normalize != "destroy":
-          localError(n.info, errGenerated, "destructor has to be named 'destroy'")
       of wOverride:
         sym.flags.incl sfOverriden
       of wNosideeffect:
@@ -766,9 +796,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
       of wNoreturn:
         noVal(it)
         incl(sym.flags, sfNoReturn)
+        if sym.typ[0] != nil:
+          localError(sym.ast[paramsPos][0].info, errNoReturnWithReturnTypeNotAllowed)
       of wDynlib:
         processDynLib(c, it, sym)
-      of wCompilerproc:
+      of wCompilerProc, wCore:
         noVal(it)           # compilerproc may not get a string!
         cppDefine(c.graph.config, sym.name.s)
         if sfFromGeneric notin sym.flags: markCompilerProc(sym)
@@ -778,7 +810,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
       of wExplain:
         sym.flags.incl sfExplain
       of wDeprecated:
-        if it.kind == nkExprColonExpr: deprecatedStmt(c, it)
+        if sym != nil and sym.kind in routineKinds:
+          if it.kind in nkPragmaCallKinds: discard getStrLitNode(c, it)
+          incl(sym.flags, sfDeprecated)
+        elif it.kind in nkPragmaCallKinds: deprecatedStmt(c, it)
         elif sym != nil: incl(sym.flags, sfDeprecated)
         else: incl(c.module.flags, sfDeprecated)
       of wVarargs:
@@ -799,6 +834,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         noVal(it)
         if sym.typ == nil or tfFinal in sym.typ.flags: invalidPragma(it)
         else: incl(sym.typ.flags, tfInheritable)
+      of wPackage:
+        noVal(it)
+        if sym.typ == nil: invalidPragma(it)
+        else: incl(sym.flags, sfForward)
       of wAcyclic:
         noVal(it)
         if sym.typ == nil: invalidPragma(it)
@@ -853,8 +892,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         result = true
       of wPop: processPop(c, it)
       of wPragma:
-        processPragma(c, n, i)
-        result = true
+        if not sym.isNil and sym.kind == skTemplate:
+          sym.flags.incl sfCustomPragma
+        else:
+          processPragma(c, n, i)
+          result = true
       of wDiscardable:
         noVal(it)
         if sym != nil: incl(sym.flags, sfDiscardable)
@@ -928,7 +970,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         elif sym.typ == nil: invalidPragma(it)
         else: sym.typ.lockLevel = pragmaLocks(c, it)
       of wBitsize:
-        if sym == nil or sym.kind != skField or it.kind != nkExprColonExpr:
+        if sym == nil or sym.kind != skField:
           invalidPragma(it)
         else:
           sym.bitsize = expectIntLit(c, it)
@@ -946,7 +988,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         if sym == nil: invalidPragma(it)
         else: magicsys.registerNimScriptSymbol(sym)
       of wInjectStmt:
-        if it.kind != nkExprColonExpr:
+        if it.kind notin nkPragmaCallKinds or it.len != 2:
           localError(it.info, errExprExpected)
         else:
           it.sons[1] = c.semExpr(c, it.sons[1])
@@ -957,10 +999,12 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         else:
           localError(it.info, "'experimental' pragma only valid as toplevel statement")
       of wThis:
-        if it.kind == nkExprColonExpr:
+        if it.kind in nkPragmaCallKinds and it.len == 2:
           c.selfName = considerQuotedIdent(it[1])
-        else:
+        elif it.kind == nkIdent or it.len == 1:
           c.selfName = getIdent("self")
+        else:
+          localError(it.info, "'this' pragma is allowed to have zero or one arguments")
       of wNoRewrite:
         noVal(it)
       of wBase:
@@ -974,8 +1018,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         noVal(it)
         if sym == nil: invalidPragma(it)
         else: sym.flags.incl sfUsed
+      of wLiftLocals: discard
       else: invalidPragma(it)
-    else: invalidPragma(it)
+    else:
+      n.sons[i] = semCustomPragma(c, it)
+
 
 proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
                       validPragmas: TSpecialWords) =
@@ -984,9 +1031,11 @@ proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
       let o = it.otherPragmas
       if not o.isNil:
         pushInfoContext(n.info)
-        for i in countup(0, sonsLen(o) - 1):
+        var i = 0
+        while i < o.len():
           if singlePragma(c, sym, o, i, validPragmas):
             internalError(n.info, "implicitPragmas")
+          inc i
         popInfoContext()
 
     if lfExportLib in sym.loc.flags and sfExportc notin sym.flags:
@@ -1003,7 +1052,7 @@ proc hasPragma*(n: PNode, pragma: TSpecialWord): bool =
     return false
 
   for p in n.sons:
-    var key = if p.kind == nkExprColonExpr: p[0] else: p
+    var key = if p.kind in nkPragmaCallKinds and p.len > 1: p[0] else: p
     if key.kind == nkIdent and whichKeyword(key.ident) == pragma:
       return true
 
@@ -1011,9 +1060,10 @@ proc hasPragma*(n: PNode, pragma: TSpecialWord): bool =
 
 proc pragmaRec(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords) =
   if n == nil: return
-  for i in countup(0, sonsLen(n) - 1):
-    if n.sons[i].kind == nkPragma: pragmaRec(c, sym, n.sons[i], validPragmas)
-    elif singlePragma(c, sym, n, i, validPragmas): break
+  var i = 0
+  while i < n.len():
+    if singlePragma(c, sym, n, i, validPragmas): break
+    inc i
 
 proc pragma(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords) =
   if n == nil: return

compiler/renderer.nim

@@ -17,12 +17,12 @@ type
     renderNone, renderNoBody, renderNoComments, renderDocComments,
     renderNoPragmas, renderIds, renderNoProcDefs
   TRenderFlags* = set[TRenderFlag]
-  TRenderTok*{.final.} = object
+  TRenderTok* = object
     kind*: TTokType
     length*: int16
 
   TRenderTokSeq* = seq[TRenderTok]
-  TSrcGen*{.final.} = object
+  TSrcGen* = object
     indent*: int
     lineLen*: int
     pos*: int              # current position for iteration over the buffer
@@ -37,15 +37,11 @@ type
     inGenericParams: bool
     checkAnon: bool        # we're in a context that can contain sfAnon
     inPragma: int
+    when defined(nimpretty):
+      pendingNewlineCount: int
+      origContent: string
 
 
-proc renderModule*(n: PNode, filename: string, renderFlags: TRenderFlags = {})
-proc renderTree*(n: PNode, renderFlags: TRenderFlags = {}): string
-proc initTokRender*(r: var TSrcGen, n: PNode, renderFlags: TRenderFlags = {})
-proc getNextTok*(r: var TSrcGen, kind: var TTokType, literal: var string)
-
-proc `$`*(n: PNode): string = n.renderTree
-# implementation
 # We render the source code in a two phases: The first
 # determines how long the subtree will likely be, the second
 # phase appends to a buffer that will be the output.
@@ -67,9 +63,31 @@ proc renderDefinitionName*(s: PSym, noQuotes = false): string =
   else:
     result = '`' & x & '`'
 
+when not defined(nimpretty):
+  const
+    IndentWidth = 2
+    longIndentWid = IndentWidth * 2
+else:
+  template IndentWidth: untyped = lexer.gIndentationWidth
+  template longIndentWid: untyped = IndentWidth() * 2
+
+  proc minmaxLine(n: PNode): (int, int) =
+    case n.kind
+    of nkTripleStrLit:
+      result = (n.info.line.int, n.info.line.int + countLines(n.strVal))
+    of nkCommentStmt:
+      result = (n.info.line.int, n.info.line.int + countLines(n.comment))
+    else:
+      result = (n.info.line.int, n.info.line.int)
+    for i in 0 ..< safeLen(n):
+      let (currMin, currMax) = minmaxLine(n[i])
+      if currMin < result[0]: result[0] = currMin
+      if currMax > result[1]: result[1] = currMax
+
+  proc lineDiff(a, b: PNode): int =
+    result = minmaxLine(b)[0] - minmaxLine(a)[1]
+
 const
-  IndentWidth = 2
-  longIndentWid = 4
   MaxLineLen = 80
   LineCommentColumn = 30
 
@@ -95,7 +113,11 @@ proc addTok(g: var TSrcGen, kind: TTokType, s: string) =
 
 proc addPendingNL(g: var TSrcGen) =
   if g.pendingNL >= 0:
-    addTok(g, tkSpaces, "\n" & spaces(g.pendingNL))
+    when defined(nimpretty):
+      let newlines = repeat("\n", clamp(g.pendingNewlineCount, 1, 3))
+    else:
+      const newlines = "\n"
+    addTok(g, tkSpaces, newlines & spaces(g.pendingNL))
     g.lineLen = g.pendingNL
     g.pendingNL = - 1
     g.pendingWhitespace = -1
@@ -119,11 +141,17 @@ proc putNL(g: var TSrcGen) =
 
 proc optNL(g: var TSrcGen, indent: int) =
   g.pendingNL = indent
-  g.lineLen = indent          # BUGFIX
+  g.lineLen = indent
+  when defined(nimpretty): g.pendingNewlineCount = 0
 
 proc optNL(g: var TSrcGen) =
   optNL(g, g.indent)
 
+proc optNL(g: var TSrcGen; a, b: PNode) =
+  g.pendingNL = g.indent
+  g.lineLen = g.indent
+  when defined(nimpretty): g.pendingNewlineCount = lineDiff(a, b)
+
 proc indentNL(g: var TSrcGen) =
   inc(g.indent, IndentWidth)
   g.pendingNL = g.indent
@@ -147,8 +175,17 @@ proc put(g: var TSrcGen, kind: TTokType, s: string) =
 
 proc toNimChar(c: char): string =
   case c
-  of '\0': result = "\\0"
-  of '\x01'..'\x1F', '\x80'..'\xFF': result = "\\x" & strutils.toHex(ord(c), 2)
+  of '\0': result = "\\x00" # not "\\0" to avoid ambiguous cases like "\\012".
+  of '\a': result = "\\a" # \x07
+  of '\b': result = "\\b" # \x08
+  of '\t': result = "\\t" # \x09
+  of '\L': result = "\\L" # \x0A
+  of '\v': result = "\\v" # \x0B
+  of '\f': result = "\\f" # \x0C
+  of '\c': result = "\\c" # \x0D
+  of '\e': result = "\\e" # \x1B
+  of '\x01'..'\x06', '\x0E'..'\x1A', '\x1C'..'\x1F', '\x80'..'\xFF':
+    result = "\\x" & strutils.toHex(ord(c), 2)
   of '\'', '\"', '\\': result = '\\' & c
   else: result = c & ""
 
@@ -284,12 +321,12 @@ proc gcoms(g: var TSrcGen) =
   for i in countup(0, high(g.comStack)): gcom(g, g.comStack[i])
   popAllComs(g)
 
-proc lsub(n: PNode): int
-proc litAux(n: PNode, x: BiggestInt, size: int): string =
+proc lsub(g: TSrcGen; n: PNode): int
+proc litAux(g: TSrcGen; n: PNode, x: BiggestInt, size: int): string =
   proc skip(t: PType): PType =
     result = t
-    while result.kind in {tyGenericInst, tyRange, tyVar, tyDistinct,
-                          tyOrdinal, tyAlias}:
+    while result.kind in {tyGenericInst, tyRange, tyVar, tyLent, tyDistinct,
+                          tyOrdinal, tyAlias, tySink}:
       result = lastSon(result)
   if n.typ != nil and n.typ.skip.kind in {tyBool, tyEnum}:
     let enumfields = n.typ.skip.n
@@ -302,14 +339,23 @@ proc litAux(n: PNode, x: BiggestInt, size: int): string =
   elif nfBase16 in n.flags: result = "0x" & toHex(x, size * 2)
   else: result = $x
 
-proc ulitAux(n: PNode, x: BiggestInt, size: int): string =
+proc ulitAux(g: TSrcGen; n: PNode, x: BiggestInt, size: int): string =
   if nfBase2 in n.flags: result = "0b" & toBin(x, size * 8)
   elif nfBase8 in n.flags: result = "0o" & toOct(x, size * 3)
   elif nfBase16 in n.flags: result = "0x" & toHex(x, size * 2)
   else: result = $x
   # XXX proper unsigned output!
 
-proc atom(n: PNode): string =
+proc atom(g: TSrcGen; n: PNode): string =
+  when defined(nimpretty):
+    let comment = if n.info.commentOffsetA < n.info.commentOffsetB:
+                    " " & substr(g.origContent, n.info.commentOffsetA, n.info.commentOffsetB)
+                  else:
+                    ""
+    if n.info.offsetA <= n.info.offsetB:
+      # for some constructed tokens this can not be the case and we're better
+      # off to not mess with the offset then.
+      return substr(g.origContent, n.info.offsetA, n.info.offsetB) & comment
   var f: float32
   case n.kind
   of nkEmpty: result = ""
@@ -319,30 +365,30 @@ proc atom(n: PNode): string =
   of nkRStrLit: result = "r\"" & replace(n.strVal, "\"", "\"\"")  & '\"'
   of nkTripleStrLit: result = "\"\"\"" & n.strVal & "\"\"\""
   of nkCharLit: result = '\'' & toNimChar(chr(int(n.intVal))) & '\''
-  of nkIntLit: result = litAux(n, n.intVal, 4)
-  of nkInt8Lit: result = litAux(n, n.intVal, 1) & "\'i8"
-  of nkInt16Lit: result = litAux(n, n.intVal, 2) & "\'i16"
-  of nkInt32Lit: result = litAux(n, n.intVal, 4) & "\'i32"
-  of nkInt64Lit: result = litAux(n, n.intVal, 8) & "\'i64"
-  of nkUIntLit: result = ulitAux(n, n.intVal, 4) & "\'u"
-  of nkUInt8Lit: result = ulitAux(n, n.intVal, 1) & "\'u8"
-  of nkUInt16Lit: result = ulitAux(n, n.intVal, 2) & "\'u16"
-  of nkUInt32Lit: result = ulitAux(n, n.intVal, 4) & "\'u32"
-  of nkUInt64Lit: result = ulitAux(n, n.intVal, 8) & "\'u64"
+  of nkIntLit: result = litAux(g, n, n.intVal, 4)
+  of nkInt8Lit: result = litAux(g, n, n.intVal, 1) & "\'i8"
+  of nkInt16Lit: result = litAux(g, n, n.intVal, 2) & "\'i16"
+  of nkInt32Lit: result = litAux(g, n, n.intVal, 4) & "\'i32"
+  of nkInt64Lit: result = litAux(g, n, n.intVal, 8) & "\'i64"
+  of nkUIntLit: result = ulitAux(g, n, n.intVal, 4) & "\'u"
+  of nkUInt8Lit: result = ulitAux(g, n, n.intVal, 1) & "\'u8"
+  of nkUInt16Lit: result = ulitAux(g, n, n.intVal, 2) & "\'u16"
+  of nkUInt32Lit: result = ulitAux(g, n, n.intVal, 4) & "\'u32"
+  of nkUInt64Lit: result = ulitAux(g, n, n.intVal, 8) & "\'u64"
   of nkFloatLit:
     if n.flags * {nfBase2, nfBase8, nfBase16} == {}: result = $(n.floatVal)
-    else: result = litAux(n, (cast[PInt64](addr(n.floatVal)))[] , 8)
+    else: result = litAux(g, n, (cast[PInt64](addr(n.floatVal)))[] , 8)
   of nkFloat32Lit:
     if n.flags * {nfBase2, nfBase8, nfBase16} == {}:
       result = $n.floatVal & "\'f32"
     else:
       f = n.floatVal.float32
-      result = litAux(n, (cast[PInt32](addr(f)))[], 4) & "\'f32"
+      result = litAux(g, n, (cast[PInt32](addr(f)))[], 4) & "\'f32"
   of nkFloat64Lit:
     if n.flags * {nfBase2, nfBase8, nfBase16} == {}:
       result = $n.floatVal & "\'f64"
     else:
-      result = litAux(n, (cast[PInt64](addr(n.floatVal)))[], 8) & "\'f64"
+      result = litAux(g, n, (cast[PInt64](addr(n.floatVal)))[], 8) & "\'f64"
   of nkNilLit: result = "nil"
   of nkType:
     if (n.typ != nil) and (n.typ.sym != nil): result = n.typ.sym.name.s
@@ -351,21 +397,21 @@ proc atom(n: PNode): string =
     internalError("rnimsyn.atom " & $n.kind)
     result = ""
 
-proc lcomma(n: PNode, start: int = 0, theEnd: int = - 1): int =
+proc lcomma(g: TSrcGen; n: PNode, start: int = 0, theEnd: int = - 1): int =
   assert(theEnd < 0)
   result = 0
   for i in countup(start, sonsLen(n) + theEnd):
-    inc(result, lsub(n.sons[i]))
+    inc(result, lsub(g, n.sons[i]))
     inc(result, 2)            # for ``, ``
   if result > 0:
     dec(result, 2)            # last does not get a comma!
 
-proc lsons(n: PNode, start: int = 0, theEnd: int = - 1): int =
+proc lsons(g: TSrcGen; n: PNode, start: int = 0, theEnd: int = - 1): int =
   assert(theEnd < 0)
   result = 0
-  for i in countup(start, sonsLen(n) + theEnd): inc(result, lsub(n.sons[i]))
+  for i in countup(start, sonsLen(n) + theEnd): inc(result, lsub(g, n.sons[i]))
 
-proc lsub(n: PNode): int =
+proc lsub(g: TSrcGen; n: PNode): int =
   # computes the length of a tree
   if isNil(n): return 0
   if n.comment != nil: return MaxLineLen + 1
@@ -373,108 +419,108 @@ proc lsub(n: PNode): int =
   of nkEmpty: result = 0
   of nkTripleStrLit:
     if containsNL(n.strVal): result = MaxLineLen + 1
-    else: result = len(atom(n))
+    else: result = len(atom(g, n))
   of succ(nkEmpty)..pred(nkTripleStrLit), succ(nkTripleStrLit)..nkNilLit:
-    result = len(atom(n))
+    result = len(atom(g, n))
   of nkCall, nkBracketExpr, nkCurlyExpr, nkConv, nkPattern, nkObjConstr:
-    result = lsub(n.sons[0]) + lcomma(n, 1) + 2
-  of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv: result = lsub(n[1])
-  of nkCast: result = lsub(n.sons[0]) + lsub(n.sons[1]) + len("cast[]()")
-  of nkAddr: result = (if n.len>0: lsub(n.sons[0]) + len("addr()") else: 4)
-  of nkStaticExpr: result = lsub(n.sons[0]) + len("static_")
-  of nkHiddenAddr, nkHiddenDeref: result = lsub(n.sons[0])
-  of nkCommand: result = lsub(n.sons[0]) + lcomma(n, 1) + 1
-  of nkExprEqExpr, nkAsgn, nkFastAsgn: result = lsons(n) + 3
-  of nkPar, nkCurly, nkBracket, nkClosure: result = lcomma(n) + 2
-  of nkArgList: result = lcomma(n)
+    result = lsub(g, n.sons[0]) + lcomma(g, n, 1) + 2
+  of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv: result = lsub(g, n[1])
+  of nkCast: result = lsub(g, n.sons[0]) + lsub(g, n.sons[1]) + len("cast[]()")
+  of nkAddr: result = (if n.len>0: lsub(g, n.sons[0]) + len("addr()") else: 4)
+  of nkStaticExpr: result = lsub(g, n.sons[0]) + len("static_")
+  of nkHiddenAddr, nkHiddenDeref: result = lsub(g, n.sons[0])
+  of nkCommand: result = lsub(g, n.sons[0]) + lcomma(g, n, 1) + 1
+  of nkExprEqExpr, nkAsgn, nkFastAsgn: result = lsons(g, n) + 3
+  of nkPar, nkCurly, nkBracket, nkClosure: result = lcomma(g, n) + 2
+  of nkArgList: result = lcomma(g, n)
   of nkTableConstr:
-    result = if n.len > 0: lcomma(n) + 2 else: len("{:}")
+    result = if n.len > 0: lcomma(g, n) + 2 else: len("{:}")
   of nkClosedSymChoice, nkOpenSymChoice:
-    result = lsons(n) + len("()") + sonsLen(n) - 1
-  of nkTupleTy: result = lcomma(n) + len("tuple[]")
+    result = lsons(g, n) + len("()") + sonsLen(n) - 1
+  of nkTupleTy: result = lcomma(g, n) + len("tuple[]")
   of nkTupleClassTy: result = len("tuple")
-  of nkDotExpr: result = lsons(n) + 1
-  of nkBind: result = lsons(n) + len("bind_")
-  of nkBindStmt: result = lcomma(n) + len("bind_")
-  of nkMixinStmt: result = lcomma(n) + len("mixin_")
-  of nkCheckedFieldExpr: result = lsub(n.sons[0])
-  of nkLambda: result = lsons(n) + len("proc__=_")
-  of nkDo: result = lsons(n) + len("do__:_")
+  of nkDotExpr: result = lsons(g, n) + 1
+  of nkBind: result = lsons(g, n) + len("bind_")
+  of nkBindStmt: result = lcomma(g, n) + len("bind_")
+  of nkMixinStmt: result = lcomma(g, n) + len("mixin_")
+  of nkCheckedFieldExpr: result = lsub(g, n.sons[0])
+  of nkLambda: result = lsons(g, n) + len("proc__=_")
+  of nkDo: result = lsons(g, n) + len("do__:_")
   of nkConstDef, nkIdentDefs:
-    result = lcomma(n, 0, - 3)
+    result = lcomma(g, n, 0, - 3)
     var L = sonsLen(n)
-    if n.sons[L - 2].kind != nkEmpty: result = result + lsub(n.sons[L - 2]) + 2
-    if n.sons[L - 1].kind != nkEmpty: result = result + lsub(n.sons[L - 1]) + 3
-  of nkVarTuple: result = lcomma(n, 0, - 3) + len("() = ") + lsub(lastSon(n))
-  of nkChckRangeF: result = len("chckRangeF") + 2 + lcomma(n)
-  of nkChckRange64: result = len("chckRange64") + 2 + lcomma(n)
-  of nkChckRange: result = len("chckRange") + 2 + lcomma(n)
+    if n.sons[L - 2].kind != nkEmpty: result = result + lsub(g, n.sons[L - 2]) + 2
+    if n.sons[L - 1].kind != nkEmpty: result = result + lsub(g, n.sons[L - 1]) + 3
+  of nkVarTuple: result = lcomma(g, n, 0, - 3) + len("() = ") + lsub(g, lastSon(n))
+  of nkChckRangeF: result = len("chckRangeF") + 2 + lcomma(g, n)
+  of nkChckRange64: result = len("chckRange64") + 2 + lcomma(g, n)
+  of nkChckRange: result = len("chckRange") + 2 + lcomma(g, n)
   of nkObjDownConv, nkObjUpConv, nkStringToCString, nkCStringToString:
     result = 2
-    if sonsLen(n) >= 1: result = result + lsub(n.sons[0])
-    result = result + lcomma(n, 1)
-  of nkExprColonExpr: result = lsons(n) + 2
-  of nkInfix: result = lsons(n) + 2
+    if sonsLen(n) >= 1: result = result + lsub(g, n.sons[0])
+    result = result + lcomma(g, n, 1)
+  of nkExprColonExpr: result = lsons(g, n) + 2
+  of nkInfix: result = lsons(g, n) + 2
   of nkPrefix:
-    result = lsons(n)+1+(if n.len > 0 and n.sons[1].kind == nkInfix: 2 else: 0)
-  of nkPostfix: result = lsons(n)
-  of nkCallStrLit: result = lsons(n)
-  of nkPragmaExpr: result = lsub(n.sons[0]) + lcomma(n, 1)
-  of nkRange: result = lsons(n) + 2
-  of nkDerefExpr: result = lsub(n.sons[0]) + 2
-  of nkAccQuoted: result = lsons(n) + 2
+    result = lsons(g, n)+1+(if n.len > 0 and n.sons[1].kind == nkInfix: 2 else: 0)
+  of nkPostfix: result = lsons(g, n)
+  of nkCallStrLit: result = lsons(g, n)
+  of nkPragmaExpr: result = lsub(g, n.sons[0]) + lcomma(g, n, 1)
+  of nkRange: result = lsons(g, n) + 2
+  of nkDerefExpr: result = lsub(g, n.sons[0]) + 2
+  of nkAccQuoted: result = lsons(g, n) + 2
   of nkIfExpr:
-    result = lsub(n.sons[0].sons[0]) + lsub(n.sons[0].sons[1]) + lsons(n, 1) +
+    result = lsub(g, n.sons[0].sons[0]) + lsub(g, n.sons[0].sons[1]) + lsons(g, n, 1) +
         len("if_:_")
-  of nkElifExpr: result = lsons(n) + len("_elif_:_")
-  of nkElseExpr: result = lsub(n.sons[0]) + len("_else:_") # type descriptions
-  of nkTypeOfExpr: result = (if n.len > 0: lsub(n.sons[0]) else: 0)+len("type()")
-  of nkRefTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) + len("ref")
-  of nkPtrTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) + len("ptr")
-  of nkVarTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) + len("var")
+  of nkElifExpr: result = lsons(g, n) + len("_elif_:_")
+  of nkElseExpr: result = lsub(g, n.sons[0]) + len("_else:_") # type descriptions
+  of nkTypeOfExpr: result = (if n.len > 0: lsub(g, n.sons[0]) else: 0)+len("type()")
+  of nkRefTy: result = (if n.len > 0: lsub(g, n.sons[0])+1 else: 0) + len("ref")
+  of nkPtrTy: result = (if n.len > 0: lsub(g, n.sons[0])+1 else: 0) + len("ptr")
+  of nkVarTy: result = (if n.len > 0: lsub(g, n.sons[0])+1 else: 0) + len("var")
   of nkDistinctTy:
-    result = len("distinct") + (if n.len > 0: lsub(n.sons[0])+1 else: 0)
+    result = len("distinct") + (if n.len > 0: lsub(g, n.sons[0])+1 else: 0)
     if n.len > 1:
       result += (if n[1].kind == nkWith: len("_with_") else: len("_without_"))
-      result += lcomma(n[1])
-  of nkStaticTy: result = (if n.len > 0: lsub(n.sons[0]) else: 0) +
+      result += lcomma(g, n[1])
+  of nkStaticTy: result = (if n.len > 0: lsub(g, n.sons[0]) else: 0) +
                                                          len("static[]")
-  of nkTypeDef: result = lsons(n) + 3
-  of nkOfInherit: result = lsub(n.sons[0]) + len("of_")
-  of nkProcTy: result = lsons(n) + len("proc_")
-  of nkIteratorTy: result = lsons(n) + len("iterator_")
-  of nkSharedTy: result = lsons(n) + len("shared_")
+  of nkTypeDef: result = lsons(g, n) + 3
+  of nkOfInherit: result = lsub(g, n.sons[0]) + len("of_")
+  of nkProcTy: result = lsons(g, n) + len("proc_")
+  of nkIteratorTy: result = lsons(g, n) + len("iterator_")
+  of nkSharedTy: result = lsons(g, n) + len("shared_")
   of nkEnumTy:
     if sonsLen(n) > 0:
-      result = lsub(n.sons[0]) + lcomma(n, 1) + len("enum_")
+      result = lsub(g, n.sons[0]) + lcomma(g, n, 1) + len("enum_")
     else:
       result = len("enum")
-  of nkEnumFieldDef: result = lsons(n) + 3
+  of nkEnumFieldDef: result = lsons(g, n) + 3
   of nkVarSection, nkLetSection:
     if sonsLen(n) > 1: result = MaxLineLen + 1
-    else: result = lsons(n) + len("var_")
+    else: result = lsons(g, n) + len("var_")
   of nkUsingStmt:
     if sonsLen(n) > 1: result = MaxLineLen + 1
-    else: result = lsons(n) + len("using_")
-  of nkReturnStmt: result = lsub(n.sons[0]) + len("return_")
-  of nkRaiseStmt: result = lsub(n.sons[0]) + len("raise_")
-  of nkYieldStmt: result = lsub(n.sons[0]) + len("yield_")
-  of nkDiscardStmt: result = lsub(n.sons[0]) + len("discard_")
-  of nkBreakStmt: result = lsub(n.sons[0]) + len("break_")
-  of nkContinueStmt: result = lsub(n.sons[0]) + len("continue_")
-  of nkPragma: result = lcomma(n) + 4
+    else: result = lsons(g, n) + len("using_")
+  of nkReturnStmt: result = lsub(g, n.sons[0]) + len("return_")
+  of nkRaiseStmt: result = lsub(g, n.sons[0]) + len("raise_")
+  of nkYieldStmt: result = lsub(g, n.sons[0]) + len("yield_")
+  of nkDiscardStmt: result = lsub(g, n.sons[0]) + len("discard_")
+  of nkBreakStmt: result = lsub(g, n.sons[0]) + len("break_")
+  of nkContinueStmt: result = lsub(g, n.sons[0]) + len("continue_")
+  of nkPragma: result = lcomma(g, n) + 4
   of nkCommentStmt: result = if n.comment.isNil: 0 else: len(n.comment)
-  of nkOfBranch: result = lcomma(n, 0, - 2) + lsub(lastSon(n)) + len("of_:_")
-  of nkImportAs: result = lsub(n.sons[0]) + len("_as_") + lsub(n.sons[1])
-  of nkElifBranch: result = lsons(n) + len("elif_:_")
-  of nkElse: result = lsub(n.sons[0]) + len("else:_")
-  of nkFinally: result = lsub(n.sons[0]) + len("finally:_")
-  of nkGenericParams: result = lcomma(n) + 2
+  of nkOfBranch: result = lcomma(g, n, 0, - 2) + lsub(g, lastSon(n)) + len("of_:_")
+  of nkImportAs: result = lsub(g, n.sons[0]) + len("_as_") + lsub(g, n.sons[1])
+  of nkElifBranch: result = lsons(g, n) + len("elif_:_")
+  of nkElse: result = lsub(g, n.sons[0]) + len("else:_")
+  of nkFinally: result = lsub(g, n.sons[0]) + len("finally:_")
+  of nkGenericParams: result = lcomma(g, n) + 2
   of nkFormalParams:
-    result = lcomma(n, 1) + 2
-    if n.sons[0].kind != nkEmpty: result = result + lsub(n.sons[0]) + 2
+    result = lcomma(g, n, 1) + 2
+    if n.sons[0].kind != nkEmpty: result = result + lsub(g, n.sons[0]) + 2
   of nkExceptBranch:
-    result = lcomma(n, 0, -2) + lsub(lastSon(n)) + len("except_:_")
+    result = lcomma(g, n, 0, -2) + lsub(g, lastSon(n)) + len("except_:_")
   else: result = MaxLineLen + 1
 
 proc fits(g: TSrcGen, x: int): bool =
@@ -517,7 +563,7 @@ proc gcommaAux(g: var TSrcGen, n: PNode, ind: int, start: int = 0,
                theEnd: int = - 1, separator = tkComma) =
   for i in countup(start, sonsLen(n) + theEnd):
     var c = i < sonsLen(n) + theEnd
-    var sublen = lsub(n.sons[i]) + ord(c)
+    var sublen = lsub(g, n.sons[i]) + ord(c)
     if not fits(g, sublen) and (ind + sublen < MaxLineLen): optNL(g, ind)
     let oldLen = g.tokens.len
     gsub(g, n.sons[i])
@@ -564,12 +610,12 @@ proc gsection(g: var TSrcGen, n: PNode, c: TContext, kind: TTokType,
     gcoms(g)
   dedent(g)
 
-proc longMode(n: PNode, start: int = 0, theEnd: int = - 1): bool =
+proc longMode(g: TSrcGen; n: PNode, start: int = 0, theEnd: int = - 1): bool =
   result = n.comment != nil
   if not result:
     # check further
     for i in countup(start, sonsLen(n) + theEnd):
-      if (lsub(n.sons[i]) > MaxLineLen):
+      if (lsub(g, n.sons[i]) > MaxLineLen):
         result = true
         break
 
@@ -577,12 +623,16 @@ proc gstmts(g: var TSrcGen, n: PNode, c: TContext, doIndent=true) =
   if n.kind == nkEmpty: return
   if n.kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
     if doIndent: indentNL(g)
-    for i in countup(0, sonsLen(n) - 1):
-      optNL(g)
-      if n.sons[i].kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
-        gstmts(g, n.sons[i], c, doIndent=false)
+    let L = n.len
+    for i in 0 .. L-1:
+      if i > 0:
+        optNL(g, n[i-1], n[i])
       else:
-        gsub(g, n.sons[i])
+        optNL(g)
+      if n[i].kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
+        gstmts(g, n[i], c, doIndent=false)
+      else:
+        gsub(g, n[i])
       gcoms(g)
     if doIndent: dedent(g)
   else:
@@ -597,7 +647,7 @@ proc gif(g: var TSrcGen, n: PNode) =
   gsub(g, n.sons[0].sons[0])
   initContext(c)
   putWithSpace(g, tkColon, ":")
-  if longMode(n) or (lsub(n.sons[0].sons[1]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[0].sons[1]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n.sons[0].sons[1], c)
@@ -612,7 +662,7 @@ proc gwhile(g: var TSrcGen, n: PNode) =
   gsub(g, n.sons[0])
   putWithSpace(g, tkColon, ":")
   initContext(c)
-  if longMode(n) or (lsub(n.sons[1]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[1]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n.sons[1], c)
@@ -621,7 +671,7 @@ proc gpattern(g: var TSrcGen, n: PNode) =
   var c: TContext
   put(g, tkCurlyLe, "{")
   initContext(c)
-  if longMode(n) or (lsub(n.sons[0]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[0]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n, c)
@@ -632,7 +682,7 @@ proc gpragmaBlock(g: var TSrcGen, n: PNode) =
   gsub(g, n.sons[0])
   putWithSpace(g, tkColon, ":")
   initContext(c)
-  if longMode(n) or (lsub(n.sons[1]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[1]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n.sons[1], c)
@@ -642,7 +692,7 @@ proc gtry(g: var TSrcGen, n: PNode) =
   put(g, tkTry, "try")
   putWithSpace(g, tkColon, ":")
   initContext(c)
-  if longMode(n) or (lsub(n.sons[0]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[0]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n.sons[0], c)
@@ -653,8 +703,8 @@ proc gfor(g: var TSrcGen, n: PNode) =
   var length = sonsLen(n)
   putWithSpace(g, tkFor, "for")
   initContext(c)
-  if longMode(n) or
-      (lsub(n.sons[length - 1]) + lsub(n.sons[length - 2]) + 6 + g.lineLen >
+  if longMode(g, n) or
+      (lsub(g, n.sons[length - 1]) + lsub(g, n.sons[length - 2]) + 6 + g.lineLen >
       MaxLineLen):
     incl(c.flags, rfLongMode)
   gcomma(g, n, c, 0, - 3)
@@ -669,8 +719,9 @@ proc gcase(g: var TSrcGen, n: PNode) =
   var c: TContext
   initContext(c)
   var length = sonsLen(n)
+  if length == 0: return
   var last = if n.sons[length-1].kind == nkElse: -2 else: -1
-  if longMode(n, 0, last): incl(c.flags, rfLongMode)
+  if longMode(g, n, 0, last): incl(c.flags, rfLongMode)
   putWithSpace(g, tkCase, "case")
   gsub(g, n.sons[0])
   gcoms(g)
@@ -678,7 +729,7 @@ proc gcase(g: var TSrcGen, n: PNode) =
   gsons(g, n, c, 1, last)
   if last == - 2:
     initContext(c)
-    if longMode(n.sons[length - 1]): incl(c.flags, rfLongMode)
+    if longMode(g, n.sons[length - 1]): incl(c.flags, rfLongMode)
     gsub(g, n.sons[length - 1], c)
 
 proc gproc(g: var TSrcGen, n: PNode) =
@@ -740,7 +791,7 @@ proc gblock(g: var TSrcGen, n: PNode) =
   else:
     put(g, tkBlock, "block")
   putWithSpace(g, tkColon, ":")
-  if longMode(n) or (lsub(n.sons[1]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[1]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)
   # XXX I don't get why this is needed here! gstmts should already handle this!
@@ -753,7 +804,7 @@ proc gstaticStmt(g: var TSrcGen, n: PNode) =
   putWithSpace(g, tkStatic, "static")
   putWithSpace(g, tkColon, ":")
   initContext(c)
-  if longMode(n) or (lsub(n.sons[0]) + g.lineLen > MaxLineLen):
+  if longMode(g, n) or (lsub(g, n.sons[0]) + g.lineLen > MaxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n.sons[0], c)
@@ -771,7 +822,7 @@ proc gident(g: var TSrcGen, n: PNode) =
       (n.typ != nil and tfImplicitTypeParam in n.typ.flags): return
 
   var t: TTokType
-  var s = atom(n)
+  var s = atom(g, n)
   if (s[0] in lexer.SymChars):
     if (n.kind == nkIdent):
       if (n.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
@@ -785,7 +836,10 @@ proc gident(g: var TSrcGen, n: PNode) =
     t = tkOpr
   put(g, t, s)
   if n.kind == nkSym and (renderIds in g.flags or sfGenSym in n.sym.flags):
-    put(g, tkIntLit, $n.sym.id)
+    when defined(debugMagics):
+      put(g, tkIntLit, $n.sym.id & $n.sym.magic)
+    else:
+      put(g, tkIntLit, $n.sym.id)
 
 proc doParamsAux(g: var TSrcGen, params: PNode) =
   if params.len > 1:
@@ -816,34 +870,42 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     a: TContext
   if n.comment != nil: pushCom(g, n)
   case n.kind                 # atoms:
-  of nkTripleStrLit: putRawStr(g, tkTripleStrLit, n.strVal)
+  of nkTripleStrLit: put(g, tkTripleStrLit, atom(g, n))
   of nkEmpty: discard
-  of nkType: put(g, tkInvalid, atom(n))
+  of nkType: put(g, tkInvalid, atom(g, n))
   of nkSym, nkIdent: gident(g, n)
-  of nkIntLit: put(g, tkIntLit, atom(n))
-  of nkInt8Lit: put(g, tkInt8Lit, atom(n))
-  of nkInt16Lit: put(g, tkInt16Lit, atom(n))
-  of nkInt32Lit: put(g, tkInt32Lit, atom(n))
-  of nkInt64Lit: put(g, tkInt64Lit, atom(n))
-  of nkUIntLit: put(g, tkUIntLit, atom(n))
-  of nkUInt8Lit: put(g, tkUInt8Lit, atom(n))
-  of nkUInt16Lit: put(g, tkUInt16Lit, atom(n))
-  of nkUInt32Lit: put(g, tkUInt32Lit, atom(n))
-  of nkUInt64Lit: put(g, tkUInt64Lit, atom(n))
-  of nkFloatLit: put(g, tkFloatLit, atom(n))
-  of nkFloat32Lit: put(g, tkFloat32Lit, atom(n))
-  of nkFloat64Lit: put(g, tkFloat64Lit, atom(n))
-  of nkFloat128Lit: put(g, tkFloat128Lit, atom(n))
-  of nkStrLit: put(g, tkStrLit, atom(n))
-  of nkRStrLit: put(g, tkRStrLit, atom(n))
-  of nkCharLit: put(g, tkCharLit, atom(n))
-  of nkNilLit: put(g, tkNil, atom(n))    # complex expressions
+  of nkIntLit: put(g, tkIntLit, atom(g, n))
+  of nkInt8Lit: put(g, tkInt8Lit, atom(g, n))
+  of nkInt16Lit: put(g, tkInt16Lit, atom(g, n))
+  of nkInt32Lit: put(g, tkInt32Lit, atom(g, n))
+  of nkInt64Lit: put(g, tkInt64Lit, atom(g, n))
+  of nkUIntLit: put(g, tkUIntLit, atom(g, n))
+  of nkUInt8Lit: put(g, tkUInt8Lit, atom(g, n))
+  of nkUInt16Lit: put(g, tkUInt16Lit, atom(g, n))
+  of nkUInt32Lit: put(g, tkUInt32Lit, atom(g, n))
+  of nkUInt64Lit: put(g, tkUInt64Lit, atom(g, n))
+  of nkFloatLit: put(g, tkFloatLit, atom(g, n))
+  of nkFloat32Lit: put(g, tkFloat32Lit, atom(g, n))
+  of nkFloat64Lit: put(g, tkFloat64Lit, atom(g, n))
+  of nkFloat128Lit: put(g, tkFloat128Lit, atom(g, n))
+  of nkStrLit: put(g, tkStrLit, atom(g, n))
+  of nkRStrLit: put(g, tkRStrLit, atom(g, n))
+  of nkCharLit: put(g, tkCharLit, atom(g, n))
+  of nkNilLit: put(g, tkNil, atom(g, n))    # complex expressions
   of nkCall, nkConv, nkDotCall, nkPattern, nkObjConstr:
     if n.len > 0 and isBracket(n[0]):
       gsub(g, n, 1)
       put(g, tkBracketLe, "[")
       gcomma(g, n, 2)
       put(g, tkBracketRi, "]")
+    elif n.len > 1 and n.lastSon.kind == nkStmtList:
+      gsub(g, n[0])
+      if n.len > 2:
+        put(g, tkParLe, "(")
+        gcomma(g, n, 1, -2)
+        put(g, tkParRi, ")")
+      put(g, tkColon, ":")
+      gsub(g, n, n.len-1)
     else:
       if sonsLen(n) >= 1: gsub(g, n.sons[0])
       put(g, tkParLe, "(")
@@ -1003,7 +1065,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     gsub(g, n, 1)
     put(g, tkSpaces, Space)
     gsub(g, n, 0)        # binary operator
-    if not fits(g, lsub(n.sons[2]) + lsub(n.sons[0]) + 1):
+    if not fits(g, lsub(g, n.sons[2]) + lsub(g, n.sons[0]) + 1):
       optNL(g, g.indent + longIndentWid)
     else:
       put(g, tkSpaces, Space)
@@ -1011,7 +1073,11 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
   of nkPrefix:
     gsub(g, n, 0)
     if n.len > 1:
-      put(g, tkSpaces, Space)
+      let opr = if n[0].kind == nkIdent: n[0].ident
+                elif n[0].kind == nkSym: n[0].sym.name
+                else: nil
+      if n[1].kind == nkPrefix or (opr != nil and renderer.isKeyword(opr)):
+        put(g, tkSpaces, Space)
       if n.sons[1].kind == nkInfix:
         put(g, tkParLe, "(")
         gsub(g, n.sons[1])
@@ -1031,7 +1097,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
   of nkAccQuoted:
     put(g, tkAccent, "`")
     if n.len > 0: gsub(g, n.sons[0])
-    for i in 1 .. <n.len:
+    for i in 1 ..< n.len:
       put(g, tkSpaces, Space)
       gsub(g, n.sons[i])
     put(g, tkAccent, "`")
@@ -1079,9 +1145,9 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
       gsub(g, n.sons[0])
       if n.len > 1:
         if n[1].kind == nkWith:
-          putWithSpace(g, tkWith, " with")
+          putWithSpace(g, tkSymbol, " with")
         else:
-          putWithSpace(g, tkWithout, " without")
+          putWithSpace(g, tkSymbol, " without")
         gcomma(g, n[1])
     else:
       put(g, tkDistinct, "distinct")
@@ -1166,6 +1232,9 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
   of nkProcDef:
     if renderNoProcDefs notin g.flags: putWithSpace(g, tkProc, "proc")
     gproc(g, n)
+  of nkFuncDef:
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkFunc, "func")
+    gproc(g, n)
   of nkConverterDef:
     if renderNoProcDefs notin g.flags: putWithSpace(g, tkConverter, "converter")
     gproc(g, n)
@@ -1313,8 +1382,11 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     gstmts(g, n.sons[0], c)
   of nkExceptBranch:
     optNL(g)
-    putWithSpace(g, tkExcept, "except")
-    gcomma(g, n, 0, - 2)
+    if n.len != 1:
+      putWithSpace(g, tkExcept, "except")
+    else:
+      put(g, tkExcept, "except")
+    gcomma(g, n, 0, -2)
     putWithSpace(g, tkColon, ":")
     gcoms(g)
     gstmts(g, lastSon(n), c)
@@ -1324,7 +1396,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
         if p.typ == nil or tfImplicitTypeParam notin p.typ.flags:
           return true
       return false
-    
+
     if n.hasExplicitParams:
       put(g, tkBracketLe, "[")
       gsemicolon(g, n)
@@ -1343,8 +1415,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     put(g, tkBracketRi, "]")
   of nkTupleClassTy:
     put(g, tkTuple, "tuple")
-  of nkMetaNode_Obsolete:
-    put(g, tkParLe, "(META|")
+  of nkComesFrom:
+    put(g, tkParLe, "(ComesFrom|")
     gsub(g, n, 0)
     put(g, tkParRi, ")")
   of nkGotoState, nkState:
@@ -1360,18 +1432,32 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     #nkNone, nkExplicitTypeListCall:
     internalError(n.info, "rnimsyn.gsub(" & $n.kind & ')')
 
-proc renderTree(n: PNode, renderFlags: TRenderFlags = {}): string =
+proc renderTree*(n: PNode, renderFlags: TRenderFlags = {}): string =
   var g: TSrcGen
   initSrcGen(g, renderFlags)
-  gsub(g, n)
+  # do not indent the initial statement list so that
+  # writeFile("file.nim", repr n)
+  # produces working Nim code:
+  if n.kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
+    gstmts(g, n, emptyContext, doIndent = false)
+  else:
+    gsub(g, n)
   result = g.buf
 
-proc renderModule(n: PNode, filename: string,
-                  renderFlags: TRenderFlags = {}) =
+proc `$`*(n: PNode): string = n.renderTree
+
+proc renderModule*(n: PNode, infile, outfile: string,
+                   renderFlags: TRenderFlags = {}) =
   var
     f: File
     g: TSrcGen
   initSrcGen(g, renderFlags)
+  when defined(nimpretty):
+    try:
+      g.origContent = readFile(infile)
+    except IOError:
+      rawMessage(errCannotOpenFile, infile)
+
   for i in countup(0, sonsLen(n) - 1):
     gsub(g, n.sons[i])
     optNL(g)
@@ -1382,17 +1468,17 @@ proc renderModule(n: PNode, filename: string,
   gcoms(g)
   if optStdout in gGlobalOptions:
     write(stdout, g.buf)
-  elif open(f, filename, fmWrite):
+  elif open(f, outfile, fmWrite):
     write(f, g.buf)
     close(f)
   else:
-    rawMessage(errCannotOpenFile, filename)
+    rawMessage(errCannotOpenFile, outfile)
 
-proc initTokRender(r: var TSrcGen, n: PNode, renderFlags: TRenderFlags = {}) =
+proc initTokRender*(r: var TSrcGen, n: PNode, renderFlags: TRenderFlags = {}) =
   initSrcGen(r, renderFlags)
   gsub(r, n)
 
-proc getNextTok(r: var TSrcGen, kind: var TTokType, literal: var string) =
+proc getNextTok*(r: var TSrcGen, kind: var TTokType, literal: var string) =
   if r.idx < len(r.tokens):
     kind = r.tokens[r.idx].kind
     var length = r.tokens[r.idx].length.int

compiler/reorder.nim

@@ -1,13 +1,40 @@
 
-import intsets, tables, ast, idents, renderer
+import 
+  intsets, ast, idents, algorithm, renderer, parser, ospaths, strutils, 
+  sequtils, msgs, modulegraphs, syntaxes, options, modulepaths, tables
 
-const
-  nfTempMark = nfTransf
-  nfPermMark = nfNoRewrite
+type
+  DepN = ref object
+    pnode: PNode
+    id, idx, lowLink: int
+    onStack: bool
+    kids: seq[DepN]
+    hAQ, hIS, hB, hCmd: int
+    when not defined(release):
+      expls: seq[string]
+  DepG = seq[DepN]
+
+when not defined(release):
+  var idNames = newTable[int, string]()
+
+proc newDepN(id: int, pnode: PNode): DepN =
+  new(result)
+  result.id = id
+  result.pnode = pnode
+  result.idx = -1
+  result.lowLink = -1
+  result.onStack = false
+  result.kids = @[]
+  result.hAQ = -1
+  result.hIS = -1
+  result.hB = -1
+  result.hCmd = -1
+  when not defined(release):
+    result.expls = @[]
 
 proc accQuoted(n: PNode): PIdent =
   var id = ""
-  for i in 0 .. <n.len:
+  for i in 0 ..< n.len:
     let x = n[i]
     case x.kind
     of nkIdent: id.add(x.ident.s)
@@ -21,10 +48,19 @@ proc addDecl(n: PNode; declares: var IntSet) =
   of nkPragmaExpr: addDecl(n[0], declares)
   of nkIdent:
     declares.incl n.ident.id
+    when not defined(release):
+      idNames[n.ident.id] = n.ident.s
   of nkSym:
     declares.incl n.sym.name.id
+    when not defined(release):
+      idNames[n.sym.name.id] = n.sym.name.s
   of nkAccQuoted:
-    declares.incl accQuoted(n).id
+    let a = accQuoted(n)
+    declares.incl a.id
+    when not defined(release):
+      idNames[a.id] = a.s
+  of nkEnumFieldDef:
+    addDecl(n[0], declares)
   else: discard
 
 proc computeDeps(n: PNode, declares, uses: var IntSet; topLevel: bool) =
@@ -32,7 +68,7 @@ proc computeDeps(n: PNode, declares, uses: var IntSet; topLevel: bool) =
   template decl(n) =
     if topLevel: addDecl(n, declares)
   case n.kind
-  of procDefs:
+  of procDefs, nkMacroDef, nkTemplateDef:
     decl(n[0])
     for i in 1..bodyPos: deps(n[i])
   of nkLetSection, nkVarSection, nkUsingStmt:
@@ -44,43 +80,358 @@ proc computeDeps(n: PNode, declares, uses: var IntSet; topLevel: bool) =
     for a in n:
       if a.len >= 3:
         decl(a[0])
-        for i in 1..<a.len: deps(a[i])
+        for i in 1..<a.len:
+          if a[i].kind == nkEnumTy:
+            # declare enum members
+            for b in a[i]:
+              decl(b)
+          else:
+            deps(a[i])
+  of nkIdentDefs:
+    for i in 1..<n.len: # avoid members identifiers in object definition
+      deps(n[i])
   of nkIdent: uses.incl n.ident.id
   of nkSym: uses.incl n.sym.name.id
   of nkAccQuoted: uses.incl accQuoted(n).id
   of nkOpenSymChoice, nkClosedSymChoice:
     uses.incl n.sons[0].sym.name.id
-  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse:
+  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse, nkStaticStmt:
     for i in 0..<len(n): computeDeps(n[i], declares, uses, topLevel)
+  of nkPragma:
+    let a = n.sons[0]
+    if a.kind == nkExprColonExpr and a.sons[0].kind == nkIdent and
+       a.sons[0].ident.s == "pragma":
+        # user defined pragma
+        decl(a.sons[1])
+    else:
+      for i in 0..<safeLen(n): deps(n[i])
   else:
     for i in 0..<safeLen(n): deps(n[i])
 
-proc visit(i: int; all, res: PNode; deps: var seq[(IntSet, IntSet)]): bool =
-  let n = all[i]
-  if nfTempMark in n.flags:
-    # not a DAG!
+proc cleanPath(s: string): string =
+  # Here paths may have the form A / B or "A/B"
+  result = ""
+  for c in s:
+    if c != ' ' and c != '\"':
+      result.add c
+
+proc joinPath(parts: seq[string]): string =
+  let nb = parts.len
+  assert nb > 0
+  if nb == 1:
+    return parts[0]
+  result = parts[0] / parts[1]
+  for i in 2..<parts.len:
+    result = result / parts[i]
+
+proc getIncludePath(n: PNode, modulePath: string): string =
+  let istr = n.renderTree.cleanPath
+  let (pdir, _) = modulePath.splitPath
+  let p = istr.split('/').joinPath.addFileExt("nim")
+  result = pdir / p
+
+proc hasIncludes(n:PNode): bool =
+  for a in n:
+    if a.kind == nkIncludeStmt:
+      return true
+
+proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: int32;
+                    cache: IdentCache): PNode {.procvar.} =
+  result = syntaxes.parseFile(fileIdx, cache)
+  graph.addDep(s, fileIdx)
+  graph.addIncludeDep(s.position.int32, fileIdx)
+
+proc expandIncludes(graph: ModuleGraph, module: PSym, n: PNode, 
+                    modulePath: string, includedFiles: var IntSet,
+                    cache: IdentCache): PNode =
+  # Parses includes and injects them in the current tree
+  if not n.hasIncludes:
+    return n
+  result = newNodeI(nkStmtList, n.info)
+  for a in n:
+    if a.kind == nkIncludeStmt:
+      for i in 0..<a.len:
+        var f = checkModuleName(a.sons[i])
+        if f != InvalidFileIDX:
+          if containsOrIncl(includedFiles, f):
+            localError(a.info, errRecursiveDependencyX, f.toFilename)
+          else:
+            let nn = includeModule(graph, module, f, cache)
+            let nnn = expandIncludes(graph, module, nn, modulePath, 
+                                      includedFiles, cache)
+            excl(includedFiles, f)
+            for b in nnn:
+              result.add b
+    else:
+      result.add a
+
+proc splitSections(n: PNode): PNode =
+  # Split typeSections and ConstSections into
+  # sections that contain only one definition
+  assert n.kind == nkStmtList
+  result = newNodeI(nkStmtList, n.info)
+  for a in n:
+    if a.kind in {nkTypeSection, nkConstSection} and a.len > 1:
+      for b in a:
+        var s = newNode(a.kind)
+        s.info = b.info
+        s.add b
+        result.add s
+    else:
+      result.add a
+
+proc haveSameKind(dns: seq[DepN]): bool =
+  # Check if all the nodes in a strongly connected
+  # component have the same kind
+  result = true
+  let kind = dns[0].pnode.kind
+  for dn in dns:
+    if dn.pnode.kind != kind:
+      return false
+
+proc mergeSections(comps: seq[seq[DepN]], res: PNode) =
+  # Merges typeSections and ConstSections when they form
+  # a strong component (ex: circular type definition)
+  for c in comps:
+    assert c.len > 0
+    if c.len == 1:
+      res.add c[0].pnode
+    else:
+      let fstn = c[0].pnode
+      let kind = fstn.kind
+      # always return to the original order when we got circular dependencies
+      let cs = c.sortedByIt(it.id)
+      if kind in {nkTypeSection, nkConstSection} and haveSameKind(cs):
+        # Circular dependency between type or const sections, we just
+        # need to merge them
+        var sn = newNode(kind)
+        for dn in cs:
+          sn.add dn.pnode.sons[0]
+        res.add sn
+      else:
+          # Problematic circular dependency, we arrange the nodes into
+          # their original relative order and make sure to re-merge
+          # consecutive type and const sections
+          var wmsg = "Circular dependency detected. reorder pragma may not be able to" &
+            " reorder some nodes properely"
+          when not defined(release):
+            wmsg &= ":\n"
+            for i in 0..<cs.len-1:
+                for j in i..<cs.len:
+                  for ci in 0..<cs[i].kids.len:
+                    if cs[i].kids[ci].id == cs[j].id:
+                      wmsg &= "line " & $cs[i].pnode.info.line &
+                        " depends on line " & $cs[j].pnode.info.line &
+                        ": " & cs[i].expls[ci] & "\n"
+            for j in 0..<cs.len-1:
+                for ci in 0..<cs[^1].kids.len:
+                  if cs[^1].kids[ci].id == cs[j].id:
+                    wmsg &= "line " & $cs[^1].pnode.info.line &
+                      " depends on line " & $cs[j].pnode.info.line &
+                      ": " & cs[^1].expls[ci] & "\n"
+          message(cs[0].pnode.info, warnUser, wmsg)
+
+          var i = 0
+          while i < cs.len:
+            if cs[i].pnode.kind in {nkTypeSection, nkConstSection}:
+              let ckind = cs[i].pnode.kind
+              var sn = newNode(ckind)
+              sn.add cs[i].pnode[0]
+              inc i
+              while i < cs.len and cs[i].pnode.kind == ckind :
+                sn.add cs[i].pnode[0]
+                inc i
+              res.add sn
+            else:
+              res.add cs[i].pnode
+              inc i
+
+proc hasImportStmt(n: PNode): bool =
+  # Checks if the node is an import statement or
+  # i it contains one
+  case n.kind
+  of nkImportStmt, nkFromStmt, nkImportExceptStmt:
     return true
-  if nfPermMark notin n.flags:
-    incl n.flags, nfTempMark
-    var uses = deps[i][1]
-    for j in 0..<all.len:
-      if j != i:
-        let declares = deps[j][0]
+  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse, nkStaticStmt:
+    for a in n:
+      if a.hasImportStmt:
+        return true
+  else:
+    result = false
+
+proc hasImportStmt(n: DepN): bool =
+  if n.hIS < 0:
+    n.hIS = ord(n.pnode.hasImportStmt)
+  result = bool(n.hIS)
+
+proc hasCommand(n: PNode): bool =
+  # Checks if the node is a command or a call
+  # or if it contains one
+  case n.kind
+  of nkCommand, nkCall:
+    result = true
+  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse,
+      nkStaticStmt, nkLetSection, nkConstSection, nkVarSection,
+      nkIdentDefs:
+        for a in n:
+          if a.hasCommand:
+            return true
+  else:
+    return false
+
+proc hasCommand(n: DepN): bool =
+  if n.hCmd < 0:
+    n.hCmd = ord(n.pnode.hasCommand)
+  result = bool(n.hCmd)
+
+proc hasAccQuoted(n: PNode): bool =
+  if n.kind == nkAccQuoted:
+    return true
+  for a in n:
+    if hasAccQuoted(a):
+      return true
+
+const extandedProcDefs = procDefs + {nkMacroDef,  nkTemplateDef}
+
+proc hasAccQuotedDef(n: PNode): bool =
+  # Checks if the node is a function, macro, template ...
+  # with a quoted name or if it contains one
+  case n.kind
+  of extandedProcDefs:
+    result = n[0].hasAccQuoted
+  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse, nkStaticStmt:
+    for a in n:
+      if a.hasAccQuotedDef:
+        return true
+  else:
+    result = false
+
+proc hasAccQuotedDef(n: DepN): bool =
+  if n.hAQ < 0:
+    n.hAQ = ord(n.pnode.hasAccQuotedDef)
+  result = bool(n.hAQ)
+
+proc hasBody(n: PNode): bool =
+  # Checks if the node is a function, macro, template ...
+  # with a body or if it contains one
+  case n.kind
+  of nkCommand, nkCall:
+    result = true
+  of extandedProcDefs:
+    result = n[^1].kind == nkStmtList
+  of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse, nkStaticStmt:
+    for a in n:
+      if a.hasBody:
+        return true
+  else:
+    result = false
+
+proc hasBody(n: DepN): bool =
+  if n.hB < 0:
+    n.hB = ord(n.pnode.hasBody)
+  result = bool(n.hB)
+
+proc intersects(s1, s2: IntSet): bool =
+  for a in s1:
+    if s2.contains(a):
+      return true
+
+proc buildGraph(n: PNode, deps: seq[(IntSet, IntSet)]): DepG =
+  # Build a dependency graph
+  result = newSeqOfCap[DepN](deps.len)
+  for i in 0..<deps.len:
+    result.add newDepN(i, n.sons[i])
+  for i in 0..<deps.len:
+    var ni = result[i]
+    let uses = deps[i][1]
+    let niHasBody = ni.hasBody
+    let niHasCmd = ni.hasCommand
+    for j in 0..<deps.len:
+      if i == j: continue
+      var nj = result[j]
+      let declares = deps[j][0]
+      if j < i and nj.hasCommand and niHasCmd:
+        # Preserve order for commands and calls
+        ni.kids.add nj
+        when not defined(release):
+          ni.expls.add "both have commands and one comes after the other"
+      elif j < i and nj.hasImportStmt:
+        # Every node that comes after an import statement must
+        # depend on that import
+        ni.kids.add nj
+        when not defined(release):
+          ni.expls.add "parent is, or contains, an import statement and child comes after it"
+      elif j < i and niHasBody and nj.hasAccQuotedDef:
+        # Every function, macro, template... with a body depends
+        # on precedent function declarations that have quoted names.
+        # That's because it is hard to detect the use of functions
+        # like "[]=", "[]", "or" ... in their bodies.
+        ni.kids.add nj
+        when not defined(release):
+          ni.expls.add "one declares a quoted identifier and the other has a body and comes after it"
+      elif j < i and niHasBody and not nj.hasBody and
+        intersects(deps[i][0], declares):
+          # Keep function declaration before function definition
+          ni.kids.add nj
+          when not defined(release):
+            for dep in deps[i][0]:
+              if dep in declares:
+                ni.expls.add "one declares \"" & idNames[dep] & "\" and the other defines it"
+      else:
         for d in declares:
           if uses.contains(d):
-            let oldLen = res.len
-            if visit(j, all, res, deps):
-              result = true
-              # rollback what we did, it turned out to be a dependency that caused
-              # trouble:
-              for k in oldLen..<res.len:
-                res.sons[k].flags = res.sons[k].flags - {nfPermMark, nfTempMark}
-              if oldLen != res.len: res.sons.setLen oldLen
-            break
-    n.flags = n.flags + {nfPermMark} - {nfTempMark}
-    res.add n
+            ni.kids.add nj
+            when not defined(release):
+              ni.expls.add "one declares \"" & idNames[d] & "\" and the other uses it"
 
-proc reorder*(n: PNode): PNode =
+proc strongConnect(v: var DepN, idx: var int, s: var seq[DepN],
+                   res: var seq[seq[DepN]]) =
+  # Recursive part of trajan's algorithm
+  v.idx = idx
+  v.lowLink = idx
+  inc idx
+  s.add v
+  v.onStack = true
+  for w in v.kids.mitems:
+    if w.idx < 0:
+      strongConnect(w, idx, s, res)
+      v.lowLink = min(v.lowLink, w.lowLink)
+    elif w.onStack:
+      v.lowLink = min(v.lowLink, w.idx)
+  if v.lowLink == v.idx:
+    var comp = newSeq[DepN]()
+    while true:
+      var w = s.pop
+      w.onStack = false
+      comp.add w
+      if w.id == v.id: break
+    res.add comp
+
+proc getStrongComponents(g: var DepG): seq[seq[DepN]] =
+  ## Tarjan's algorithm. Performs a topological sort
+  ## and detects strongly connected components.
+  result = newSeq[seq[DepN]]()
+  var s = newSeq[DepN]()
+  var idx = 0
+  for v in g.mitems:
+    if v.idx < 0:
+      strongConnect(v, idx, s, result)
+
+proc hasForbiddenPragma(n: PNode): bool =
+  # Checks if the tree node has some pragmas that do not
+  # play well with reordering, like the push/pop pragma
+  for a in n:
+    if a.kind == nkPragma and a[0].kind == nkIdent and
+        a[0].ident.s == "push":
+          return true
+
+proc reorder*(graph: ModuleGraph, n: PNode, module: PSym, cache: IdentCache): PNode =
+  if n.hasForbiddenPragma:
+    return n
+  var includedFiles = initIntSet()
+  let mpath = module.fileIdx.toFullPath
+  let n = expandIncludes(graph, module, n, mpath, 
+                          includedFiles, cache).splitSections
   result = newNodeI(nkStmtList, n.info)
   var deps = newSeq[(IntSet, IntSet)](n.len)
   for i in 0..<n.len:
@@ -88,15 +439,6 @@ proc reorder*(n: PNode): PNode =
     deps[i][1] = initIntSet()
     computeDeps(n[i], deps[i][0], deps[i][1], true)
 
-  for i in 0 .. n.len-1:
-    discard visit(i, n, result, deps)
-  for i in 0..<result.len:
-    result.sons[i].flags = result.sons[i].flags - {nfTempMark, nfPermMark}
-  when false:
-    # reverse the result:
-    let L = result.len-1
-    for i in 0 .. result.len div 2:
-      result.sons[i].flags = result.sons[i].flags - {nfTempMark, nfPermMark}
-      result.sons[L - i].flags = result.sons[L - i].flags - {nfTempMark, nfPermMark}
-      swap(result.sons[i], result.sons[L - i])
-  #echo result
+  var g = buildGraph(n, deps)
+  let comps = getStrongComponents(g)
+  mergeSections(comps, result)

compiler/rod.nim

@@ -0,0 +1,132 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements the canonalization for the various caching mechanisms.
+
+import ast, idgen
+
+when not defined(nimSymbolfiles):
+  template setupModuleCache* = discard
+  template storeNode*(module: PSym; n: PNode) = discard
+  template loadNode*(module: PSym; index: var int): PNode = PNode(nil)
+
+  template getModuleId*(fileIdx: int32; fullpath: string): int = getID()
+
+  template addModuleDep*(module, fileIdx: int32; isIncludeFile: bool) = discard
+
+  template storeRemaining*(module: PSym) = discard
+
+else:
+  include rodimpl
+
+when false:
+  type
+    BlobWriter* = object
+      buf: string
+      pos: int
+
+    SerializationAction = enum acRead, acWrite
+
+  # Varint implementation inspired by SQLite.
+  proc rdVaruint64(z: ptr UncheckedArray[byte]; n: int; pResult: var uint64): int =
+    if z[0] <= 240:
+      pResult = z[0]
+      return 1
+    if z[0] <= 248:
+      if n < 2: return 0
+      pResult = (z[0] - 241) * 256 + z[1] + 240
+      return 2
+    if n < z[0]-246: return 0
+    if z[0] == 249:
+      pResult = 2288 + 256*z[1] + z[2]
+      return 3
+    if z[0] == 250:
+      pResult = (z[1] shl 16u64) + (z[2] shl 8u64) + z[3]
+      return 4
+    let x = (z[1] shl 24) + (z[2] shl 16) + (z[3] shl 8) + z[4]
+    if z[0] == 251:
+      pResult = x
+      return 5
+    if z[0] == 252:
+      pResult = (((uint64)x) shl 8) + z[5]
+      return 6
+    if z[0] == 253:
+      pResult = (((uint64)x) shl 16) + (z[5] shl 8) + z[6]
+      return 7
+    if z[0] == 254:
+      pResult = (((uint64)x) shl 24) + (z[5] shl 16) + (z[6] shl 8) + z[7]
+      return 8
+    pResult = (((uint64)x) shl 32) +
+                (0xffffffff & ((z[5] shl 24) + (z[6] shl 16) + (z[7] shl 8) + z[8]))
+    return 9
+
+  proc varintWrite32(z: ptr UncheckedArray[byte]; y: uint32) =
+    z[0] = uint8(y shr 24)
+    z[1] = uint8(y shr 16)
+    z[2] = uint8(y shr 8)
+    z[3] = uint8(y)
+
+  proc sqlite4PutVarint64(z: ptr UncheckedArray[byte], x: uint64): int =
+    ## Write a varint into z. The buffer z must be at least 9 characters
+    ## long to accommodate the largest possible varint. Returns the number of
+    ## bytes used.
+    if x <= 240:
+      z[0] = uint8 x
+      return 1
+    if x <= 2287:
+      y = uint32(x - 240)
+      z[0] = uint8(y shr 8 + 241)
+      z[1] = uint8(y and 255)
+      return 2
+    if x <= 67823:
+      y = uint32(x - 2288)
+      z[0] = 249
+      z[1] = uint8(y shr 8)
+      z[2] = uint8(y and 255)
+      return 3
+    let y = uint32 x
+    let w = uint32(x shr 32)
+    if w == 0:
+      if y <= 16777215:
+        z[0] = 250
+        z[1] = uint8(y shr 16)
+        z[2] = uint8(y shr 8)
+        z[3] = uint8(y)
+        return 4
+      z[0] = 251
+      varintWrite32(z+1, y)
+      return 5
+    if w <= 255:
+      z[0] = 252
+      z[1] = uint8 w
+      varintWrite32(z+2, y)
+      return 6
+    if w <= 65535:
+      z[0] = 253
+      z[1] = uint8(w shr 8)
+      z[2] = uint8 w
+      varintWrite32(z+3, y)
+      return 7
+    if w <= 16777215:
+      z[0] = 254
+      z[1] = uint8(w shr 16)
+      z[2] = uint8(w shr 8)
+      z[3] = uint8 w
+      varintWrite32(z+4, y)
+      return 8
+    z[0] = 255
+    varintWrite32(z+1, w)
+    varintWrite32(z+5, y)
+    return 9
+
+  template field(x: BiggestInt; action: SerializationAction) =
+    when action == acRead:
+      readBiggestInt(x)
+    else:
+      writeBiggestInt()

compiler/rodimpl.nim

@@ -0,0 +1,947 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2018 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements the new compilation cache.
+
+import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
+  renderer, rodutils, std / sha1, idents, astalgo, magicsys
+
+## Todo:
+## - Implement the 'import' replay logic so that the codegen runs over
+##   dependent modules.
+## - Make conditional symbols and the configuration part of a module's
+##   dependencies.
+## - Test multi methods.
+## - Implement the limited VM support based on sets.
+## - Depencency computation should use signature hashes in order to
+##   avoid recompiling dependent modules.
+
+var db: DbConn
+
+proc hashFileCached(fileIdx: int32; fullpath: string): string =
+  result = msgs.getHash(fileIdx)
+  if result.len == 0:
+    result = $secureHashFile(fullpath)
+    msgs.setHash(fileIdx, result)
+
+proc needsRecompile(fileIdx: int32; fullpath: string; cycleCheck: var IntSet): bool =
+  let root = db.getRow(sql"select id, fullhash from filenames where fullpath = ?",
+    fullpath)
+  if root[0].len == 0: return true
+  if root[1] != hashFileCached(fileIdx, fullpath):
+    return true
+  # cycle detection: assume "not changed" is correct.
+  if cycleCheck.containsOrIncl(fileIdx):
+    return false
+  # check dependencies (recursively):
+  for row in db.fastRows(sql"select fullpath from filenames where id in (select dependency from deps where module = ?)",
+                         root[0]):
+    let dep = row[0]
+    if needsRecompile(dep.fileInfoIdx, dep, cycleCheck):
+      return true
+  return false
+
+proc getModuleId*(fileIdx: int32; fullpath: string): int =
+  if gSymbolFiles != v2Sf: return getID()
+  let module = db.getRow(
+    sql"select id, fullHash from modules where fullpath = ?", fullpath)
+  let currentFullhash = hashFileCached(fileIdx, fullpath)
+  if module[0].len == 0:
+    result = int db.insertID(sql"insert into modules(fullpath, interfHash, fullHash) values (?, ?, ?)",
+      fullpath, "", currentFullhash)
+  else:
+    result = parseInt(module[0])
+    if currentFullhash == module[1]:
+      # not changed, so use the cached AST (even if it might be wrong
+      # due to its dependencies):
+      doAssert(result != 0)
+      var cycleCheck = initIntSet()
+      if not needsRecompile(fileIdx, fullpath, cycleCheck):
+        return -result
+    db.exec(sql"update modules set fullHash = ? where id = ?", currentFullhash, module[0])
+    db.exec(sql"delete from deps where module = ?", module[0])
+    db.exec(sql"delete from types where module = ?", module[0])
+    db.exec(sql"delete from syms where module = ?", module[0])
+    db.exec(sql"delete from toplevelstmts where module = ?", module[0])
+    db.exec(sql"delete from statics where module = ?", module[0])
+
+type
+  TRodWriter = object
+    module: PSym
+    sstack: seq[PSym]          # a stack of symbols to process
+    tstack: seq[PType]         # a stack of types to process
+    tmarks, smarks: IntSet
+    forwardedSyms: seq[PSym]
+
+  PRodWriter = var TRodWriter
+
+proc initRodWriter(module: PSym): TRodWriter =
+  result = TRodWriter(module: module, sstack: @[], tstack: @[],
+    tmarks: initIntSet(), smarks: initIntSet(), forwardedSyms: @[])
+
+when false:
+  proc getDefines(): string =
+    result = ""
+    for d in definedSymbolNames():
+      if result.len != 0: add(result, " ")
+      add(result, d)
+
+const
+  rodNL = "\L"
+
+proc pushType(w: PRodWriter, t: PType) =
+  if not containsOrIncl(w.tmarks, t.id):
+    w.tstack.add(t)
+
+proc pushSym(w: PRodWriter, s: PSym) =
+  if not containsOrIncl(w.smarks, s.id):
+    w.sstack.add(s)
+
+proc toDbFileId(fileIdx: int32): int =
+  if fileIdx == -1: return -1
+  let fullpath = fileIdx.toFullPath
+  let row = db.getRow(sql"select id, fullhash from filenames where fullpath = ?",
+    fullpath)
+  let id = row[0]
+  let fullhash = hashFileCached(fileIdx, fullpath)
+  if id.len == 0:
+    result = int db.insertID(sql"insert into filenames(fullpath, fullhash) values (?, ?)",
+      fullpath, fullhash)
+  else:
+    if row[1] != fullhash:
+      db.exec(sql"update filenames set fullhash = ? where fullpath = ?", fullhash, fullpath)
+    result = parseInt(id)
+
+proc fromDbFileId(dbId: int): int32 =
+  if dbId == -1: return -1
+  let fullpath = db.getValue(sql"select fullpath from filenames where id = ?", dbId)
+  doAssert fullpath.len > 0, "cannot find file name for DB ID " & $dbId
+  result = fileInfoIdx(fullpath)
+
+proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
+                result: var string) =
+  if n == nil:
+    # nil nodes have to be stored too:
+    result.add("()")
+    return
+  result.add('(')
+  encodeVInt(ord(n.kind), result)
+  # we do not write comments for now
+  # Line information takes easily 20% or more of the filesize! Therefore we
+  # omit line information if it is the same as the parent's line information:
+  if fInfo.fileIndex != n.info.fileIndex:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+    result.add(',')
+    encodeVInt(n.info.line, result)
+    result.add(',')
+    encodeVInt(toDbFileId(n.info.fileIndex), result)
+  elif fInfo.line != n.info.line:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+    result.add(',')
+    encodeVInt(n.info.line, result)
+  elif fInfo.col != n.info.col:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+  # No need to output the file index, as this is the serialization of one
+  # file.
+  let f = n.flags * PersistentNodeFlags
+  if f != {}:
+    result.add('$')
+    encodeVInt(cast[int32](f), result)
+  if n.typ != nil:
+    result.add('^')
+    encodeVInt(n.typ.id, result)
+    pushType(w, n.typ)
+  case n.kind
+  of nkCharLit..nkUInt64Lit:
+    if n.intVal != 0:
+      result.add('!')
+      encodeVBiggestInt(n.intVal, result)
+  of nkFloatLit..nkFloat64Lit:
+    if n.floatVal != 0.0:
+      result.add('!')
+      encodeStr($n.floatVal, result)
+  of nkStrLit..nkTripleStrLit:
+    if n.strVal != "":
+      result.add('!')
+      encodeStr(n.strVal, result)
+  of nkIdent:
+    result.add('!')
+    encodeStr(n.ident.s, result)
+  of nkSym:
+    result.add('!')
+    encodeVInt(n.sym.id, result)
+    pushSym(w, n.sym)
+  else:
+    for i in countup(0, sonsLen(n) - 1):
+      encodeNode(w, n.info, n.sons[i], result)
+  add(result, ')')
+
+proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
+  var oldLen = result.len
+  result.add('<')
+  if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
+  if loc.storage != low(loc.storage):
+    add(result, '*')
+    encodeVInt(ord(loc.storage), result)
+  if loc.flags != {}:
+    add(result, '$')
+    encodeVInt(cast[int32](loc.flags), result)
+  if loc.lode != nil:
+    add(result, '^')
+    encodeNode(w, unknownLineInfo(), loc.lode, result)
+    #encodeVInt(cast[int32](loc.t.id), result)
+    #pushType(w, loc.t)
+  if loc.r != nil:
+    add(result, '!')
+    encodeStr($loc.r, result)
+  if oldLen + 1 == result.len:
+    # no data was necessary, so remove the '<' again:
+    setLen(result, oldLen)
+  else:
+    add(result, '>')
+
+proc encodeType(w: PRodWriter, t: PType, result: var string) =
+  if t == nil:
+    # nil nodes have to be stored too:
+    result.add("[]")
+    return
+  # we need no surrounding [] here because the type is in a line of its own
+  if t.kind == tyForward: internalError("encodeType: tyForward")
+  # for the new rodfile viewer we use a preceding [ so that the data section
+  # can easily be disambiguated:
+  add(result, '[')
+  encodeVInt(ord(t.kind), result)
+  add(result, '+')
+  encodeVInt(t.id, result)
+  if t.n != nil:
+    encodeNode(w, w.module.info, t.n, result)
+  if t.flags != {}:
+    add(result, '$')
+    encodeVInt(cast[int32](t.flags), result)
+  if t.callConv != low(t.callConv):
+    add(result, '?')
+    encodeVInt(ord(t.callConv), result)
+  if t.owner != nil:
+    add(result, '*')
+    encodeVInt(t.owner.id, result)
+    pushSym(w, t.owner)
+  if t.sym != nil:
+    add(result, '&')
+    encodeVInt(t.sym.id, result)
+    pushSym(w, t.sym)
+  if t.size != - 1:
+    add(result, '/')
+    encodeVBiggestInt(t.size, result)
+  if t.align != 2:
+    add(result, '=')
+    encodeVInt(t.align, result)
+  if t.lockLevel.ord != UnspecifiedLockLevel.ord:
+    add(result, '\14')
+    encodeVInt(t.lockLevel.int16, result)
+  if t.destructor != nil and t.destructor.id != 0:
+    add(result, '\15')
+    encodeVInt(t.destructor.id, result)
+    pushSym(w, t.destructor)
+  if t.deepCopy != nil:
+    add(result, '\16')
+    encodeVInt(t.deepcopy.id, result)
+    pushSym(w, t.deepcopy)
+  if t.assignment != nil:
+    add(result, '\17')
+    encodeVInt(t.assignment.id, result)
+    pushSym(w, t.assignment)
+  if t.sink != nil:
+    add(result, '\18')
+    encodeVInt(t.sink.id, result)
+    pushSym(w, t.sink)
+  for i, s in items(t.methods):
+    add(result, '\19')
+    encodeVInt(i, result)
+    add(result, '\20')
+    encodeVInt(s.id, result)
+    pushSym(w, s)
+  encodeLoc(w, t.loc, result)
+  for i in countup(0, sonsLen(t) - 1):
+    if t.sons[i] == nil:
+      add(result, "^()")
+    else:
+      add(result, '^')
+      encodeVInt(t.sons[i].id, result)
+      pushType(w, t.sons[i])
+
+proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) =
+  add(result, '|')
+  encodeVInt(ord(lib.kind), result)
+  add(result, '|')
+  encodeStr($lib.name, result)
+  add(result, '|')
+  encodeNode(w, info, lib.path, result)
+
+proc encodeInstantiations(w: PRodWriter; s: seq[PInstantiation];
+                          result: var string) =
+  for t in s:
+    result.add('\15')
+    encodeVInt(t.sym.id, result)
+    pushSym(w, t.sym)
+    for tt in t.concreteTypes:
+      result.add('\17')
+      encodeVInt(tt.id, result)
+      pushType(w, tt)
+    result.add('\20')
+    encodeVInt(t.compilesId, result)
+
+proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
+  if s == nil:
+    # nil nodes have to be stored too:
+    result.add("{}")
+    return
+  # we need no surrounding {} here because the symbol is in a line of its own
+  encodeVInt(ord(s.kind), result)
+  result.add('+')
+  encodeVInt(s.id, result)
+  result.add('&')
+  encodeStr(s.name.s, result)
+  if s.typ != nil:
+    result.add('^')
+    encodeVInt(s.typ.id, result)
+    pushType(w, s.typ)
+  result.add('?')
+  if s.info.col != -1'i16: encodeVInt(s.info.col, result)
+  result.add(',')
+  if s.info.line != -1'i16: encodeVInt(s.info.line, result)
+  result.add(',')
+  encodeVInt(toDbFileId(s.info.fileIndex), result)
+  if s.owner != nil:
+    result.add('*')
+    encodeVInt(s.owner.id, result)
+    pushSym(w, s.owner)
+  if s.flags != {}:
+    result.add('$')
+    encodeVInt(cast[int32](s.flags), result)
+  if s.magic != mNone:
+    result.add('@')
+    encodeVInt(ord(s.magic), result)
+  result.add('!')
+  encodeVInt(cast[int32](s.options), result)
+  if s.position != 0:
+    result.add('%')
+    encodeVInt(s.position, result)
+  if s.offset != - 1:
+    result.add('`')
+    encodeVInt(s.offset, result)
+  encodeLoc(w, s.loc, result)
+  if s.annex != nil: encodeLib(w, s.annex, s.info, result)
+  if s.constraint != nil:
+    add(result, '#')
+    encodeNode(w, unknownLineInfo(), s.constraint, result)
+  case s.kind
+  of skType, skGenericParam:
+    for t in s.typeInstCache:
+      result.add('\14')
+      encodeVInt(t.id, result)
+      pushType(w, t)
+  of routineKinds:
+    encodeInstantiations(w, s.procInstCache, result)
+    if s.gcUnsafetyReason != nil:
+      result.add('\16')
+      encodeVInt(s.gcUnsafetyReason.id, result)
+      pushSym(w, s.gcUnsafetyReason)
+  of skModule, skPackage:
+    encodeInstantiations(w, s.usedGenerics, result)
+    # we don't serialize:
+    #tab*: TStrTable         # interface table for modules
+  of skLet, skVar, skField, skForVar:
+    if s.guard != nil:
+      result.add('\18')
+      encodeVInt(s.guard.id, result)
+      pushSym(w, s.guard)
+    if s.bitsize != 0:
+      result.add('\19')
+      encodeVInt(s.bitsize, result)
+  else: discard
+  # lazy loading will soon reload the ast lazily, so the ast needs to be
+  # the last entry of a symbol:
+  if s.ast != nil:
+    # we used to attempt to save space here by only storing a dummy AST if
+    # it is not necessary, but Nim's heavy compile-time evaluation features
+    # make that unfeasible nowadays:
+    encodeNode(w, s.info, s.ast, result)
+
+proc storeSym(w: PRodWriter; s: PSym) =
+  if sfForward in s.flags and s.kind != skModule:
+    w.forwardedSyms.add s
+    return
+  var buf = newStringOfCap(160)
+  encodeSym(w, s, buf)
+  # XXX only store the name for exported symbols in order to speed up lookup
+  # times once we enable the skStub logic.
+  db.exec(sql"insert into syms(nimid, module, name, data, exported) values (?, ?, ?, ?, ?)",
+    s.id, abs(w.module.id), s.name.s, buf, ord(sfExported in s.flags))
+
+proc storeType(w: PRodWriter; t: PType) =
+  var buf = newStringOfCap(160)
+  encodeType(w, t, buf)
+  db.exec(sql"insert into types(nimid, module, data) values (?, ?, ?)",
+    t.id, abs(w.module.id), buf)
+
+var w = initRodWriter(nil)
+
+proc storeNode*(module: PSym; n: PNode) =
+  if gSymbolFiles != v2Sf: return
+  w.module = module
+  var buf = newStringOfCap(160)
+  encodeNode(w, module.info, n, buf)
+  db.exec(sql"insert into toplevelstmts(module, position, data) values (?, ?, ?)",
+    abs(module.id), module.offset, buf)
+  inc module.offset
+  var i = 0
+  while true:
+    if i > 10_000:
+      quit "loop never ends!"
+    if w.sstack.len > 0:
+      let s = w.sstack.pop()
+      when false:
+        echo "popped ", s.name.s, " ", s.id
+      storeSym(w, s)
+    elif w.tstack.len > 0:
+      let t = w.tstack.pop()
+      storeType(w, t)
+      when false:
+        echo "popped type ", typeToString(t), " ", t.id
+    else:
+      break
+    inc i
+
+proc storeRemaining*(module: PSym) =
+  if gSymbolFiles != v2Sf: return
+  w.module = module
+  for s in w.forwardedSyms:
+    assert sfForward notin s.flags
+    storeSym(w, s)
+  w.forwardedSyms.setLen 0
+
+# ---------------- decoder -----------------------------------
+type
+  TRodReader = object
+    module: PSym
+    #sstack: seq[(PSym, ptr PSym)]       # a stack of symbols to process
+    #tstack: seq[(PType, ptr PType)]     # a stack of types to process
+
+    #tmarks, smarks: IntSet
+    syms: Table[int, PSym] ## XXX make this more efficients
+    types: Table[int, PType]
+    cache: IdentCache
+
+  BlobReader = object
+    s: string
+    pos: int
+
+  PRodReader = var TRodReader
+
+proc initRodReader(cache: IdentCache): TRodReader =
+  TRodReader(module: nil,
+    syms: initTable[int, PSym](), types: initTable[int, PType](),
+    cache: cache)
+
+var gr = initRodReader(newIdentCache())
+
+using
+  r: PRodReader
+  b: var BlobReader
+
+proc loadSym(r; id: int, info: TLineInfo): PSym
+proc loadType(r; id: int, info: TLineInfo): PType
+
+proc decodeLineInfo(r; b; info: var TLineInfo) =
+  if b.s[b.pos] == '?':
+    inc(b.pos)
+    if b.s[b.pos] == ',': info.col = -1'i16
+    else: info.col = int16(decodeVInt(b.s, b.pos))
+    if b.s[b.pos] == ',':
+      inc(b.pos)
+      if b.s[b.pos] == ',': info.line = -1'i16
+      else: info.line = int16(decodeVInt(b.s, b.pos))
+      if b.s[b.pos] == ',':
+        inc(b.pos)
+        info.fileIndex = fromDbFileId(decodeVInt(b.s, b.pos))
+
+proc skipNode(b) =
+  assert b.s[b.pos] == '('
+  var par = 0
+  var pos = b.pos+1
+  while true:
+    case b.s[pos]
+    of ')':
+      if par == 0: break
+      dec par
+    of '(': inc par
+    else: discard
+    inc pos
+  b.pos = pos+1 # skip ')'
+
+proc decodeNodeLazyBody(r; b; fInfo: TLineInfo,
+                        belongsTo: PSym): PNode =
+  result = nil
+  if b.s[b.pos] == '(':
+    inc(b.pos)
+    if b.s[b.pos] == ')':
+      inc(b.pos)
+      return                  # nil node
+    result = newNodeI(TNodeKind(decodeVInt(b.s, b.pos)), fInfo)
+    decodeLineInfo(r, b, result.info)
+    if b.s[b.pos] == '$':
+      inc(b.pos)
+      result.flags = cast[TNodeFlags](int32(decodeVInt(b.s, b.pos)))
+    if b.s[b.pos] == '^':
+      inc(b.pos)
+      var id = decodeVInt(b.s, b.pos)
+      result.typ = loadType(r, id, result.info)
+    case result.kind
+    of nkCharLit..nkUInt64Lit:
+      if b.s[b.pos] == '!':
+        inc(b.pos)
+        result.intVal = decodeVBiggestInt(b.s, b.pos)
+    of nkFloatLit..nkFloat64Lit:
+      if b.s[b.pos] == '!':
+        inc(b.pos)
+        var fl = decodeStr(b.s, b.pos)
+        result.floatVal = parseFloat(fl)
+    of nkStrLit..nkTripleStrLit:
+      if b.s[b.pos] == '!':
+        inc(b.pos)
+        result.strVal = decodeStr(b.s, b.pos)
+      else:
+        result.strVal = ""
+    of nkIdent:
+      if b.s[b.pos] == '!':
+        inc(b.pos)
+        var fl = decodeStr(b.s, b.pos)
+        result.ident = r.cache.getIdent(fl)
+      else:
+        internalError(result.info, "decodeNode: nkIdent")
+    of nkSym:
+      if b.s[b.pos] == '!':
+        inc(b.pos)
+        var id = decodeVInt(b.s, b.pos)
+        result.sym = loadSym(r, id, result.info)
+      else:
+        internalError(result.info, "decodeNode: nkSym")
+    else:
+      var i = 0
+      while b.s[b.pos] != ')':
+        when false:
+          if belongsTo != nil and i == bodyPos:
+            addSonNilAllowed(result, nil)
+            belongsTo.offset = b.pos
+            skipNode(b)
+          else:
+            discard
+        addSonNilAllowed(result, decodeNodeLazyBody(r, b, result.info, nil))
+        inc i
+    if b.s[b.pos] == ')': inc(b.pos)
+    else: internalError(result.info, "decodeNode: ')' missing")
+  else:
+    internalError(fInfo, "decodeNode: '(' missing " & $b.pos)
+
+proc decodeNode(r; b; fInfo: TLineInfo): PNode =
+  result = decodeNodeLazyBody(r, b, fInfo, nil)
+
+proc decodeLoc(r; b; loc: var TLoc, info: TLineInfo) =
+  if b.s[b.pos] == '<':
+    inc(b.pos)
+    if b.s[b.pos] in {'0'..'9', 'a'..'z', 'A'..'Z'}:
+      loc.k = TLocKind(decodeVInt(b.s, b.pos))
+    else:
+      loc.k = low(loc.k)
+    if b.s[b.pos] == '*':
+      inc(b.pos)
+      loc.storage = TStorageLoc(decodeVInt(b.s, b.pos))
+    else:
+      loc.storage = low(loc.storage)
+    if b.s[b.pos] == '$':
+      inc(b.pos)
+      loc.flags = cast[TLocFlags](int32(decodeVInt(b.s, b.pos)))
+    else:
+      loc.flags = {}
+    if b.s[b.pos] == '^':
+      inc(b.pos)
+      loc.lode = decodeNode(r, b, info)
+      # rrGetType(b, decodeVInt(b.s, b.pos), info)
+    else:
+      loc.lode = nil
+    if b.s[b.pos] == '!':
+      inc(b.pos)
+      loc.r = rope(decodeStr(b.s, b.pos))
+    else:
+      loc.r = nil
+    if b.s[b.pos] == '>': inc(b.pos)
+    else: internalError(info, "decodeLoc " & b.s[b.pos])
+
+proc loadBlob(query: SqlQuery; id: int): BlobReader =
+  let blob = db.getValue(query, id)
+  if blob.len == 0:
+    internalError("symbolfiles: cannot find ID " & $ id)
+  result = BlobReader(pos: 0)
+  shallowCopy(result.s, blob)
+
+proc loadType(r; id: int; info: TLineInfo): PType =
+  result = r.types.getOrDefault(id)
+  if result != nil: return result
+  var b = loadBlob(sql"select data from types where nimid = ?", id)
+
+  if b.s[b.pos] == '[':
+    inc(b.pos)
+    if b.s[b.pos] == ']':
+      inc(b.pos)
+      return                  # nil type
+  new(result)
+  result.kind = TTypeKind(decodeVInt(b.s, b.pos))
+  if b.s[b.pos] == '+':
+    inc(b.pos)
+    result.id = decodeVInt(b.s, b.pos)
+    setId(result.id)
+    #if debugIds: registerID(result)
+  else:
+    internalError(info, "decodeType: no id")
+  # here this also avoids endless recursion for recursive type
+  r.types[result.id] = result
+  if b.s[b.pos] == '(': result.n = decodeNode(r, b, unknownLineInfo())
+  if b.s[b.pos] == '$':
+    inc(b.pos)
+    result.flags = cast[TTypeFlags](int32(decodeVInt(b.s, b.pos)))
+  if b.s[b.pos] == '?':
+    inc(b.pos)
+    result.callConv = TCallingConvention(decodeVInt(b.s, b.pos))
+  if b.s[b.pos] == '*':
+    inc(b.pos)
+    result.owner = loadSym(r, decodeVInt(b.s, b.pos), info)
+  if b.s[b.pos] == '&':
+    inc(b.pos)
+    result.sym = loadSym(r, decodeVInt(b.s, b.pos), info)
+  if b.s[b.pos] == '/':
+    inc(b.pos)
+    result.size = decodeVInt(b.s, b.pos)
+  else:
+    result.size = -1
+  if b.s[b.pos] == '=':
+    inc(b.pos)
+    result.align = decodeVInt(b.s, b.pos).int16
+  else:
+    result.align = 2
+
+  if b.s[b.pos] == '\14':
+    inc(b.pos)
+    result.lockLevel = decodeVInt(b.s, b.pos).TLockLevel
+  else:
+    result.lockLevel = UnspecifiedLockLevel
+
+  if b.s[b.pos] == '\15':
+    inc(b.pos)
+    result.destructor = loadSym(r, decodeVInt(b.s, b.pos), info)
+  if b.s[b.pos] == '\16':
+    inc(b.pos)
+    result.deepCopy = loadSym(r, decodeVInt(b.s, b.pos), info)
+  if b.s[b.pos] == '\17':
+    inc(b.pos)
+    result.assignment = loadSym(r, decodeVInt(b.s, b.pos), info)
+  if b.s[b.pos] == '\18':
+    inc(b.pos)
+    result.sink = loadSym(r, decodeVInt(b.s, b.pos), info)
+  while b.s[b.pos] == '\19':
+    inc(b.pos)
+    let x = decodeVInt(b.s, b.pos)
+    doAssert b.s[b.pos] == '\20'
+    inc(b.pos)
+    let y = loadSym(r, decodeVInt(b.s, b.pos), info)
+    result.methods.safeAdd((x, y))
+  decodeLoc(r, b, result.loc, info)
+  while b.s[b.pos] == '^':
+    inc(b.pos)
+    if b.s[b.pos] == '(':
+      inc(b.pos)
+      if b.s[b.pos] == ')': inc(b.pos)
+      else: internalError(info, "decodeType ^(" & b.s[b.pos])
+      rawAddSon(result, nil)
+    else:
+      var d = decodeVInt(b.s, b.pos)
+      rawAddSon(result, loadType(r, d, info))
+
+proc decodeLib(r; b; info: TLineInfo): PLib =
+  result = nil
+  if b.s[b.pos] == '|':
+    new(result)
+    inc(b.pos)
+    result.kind = TLibKind(decodeVInt(b.s, b.pos))
+    if b.s[b.pos] != '|': internalError("decodeLib: 1")
+    inc(b.pos)
+    result.name = rope(decodeStr(b.s, b.pos))
+    if b.s[b.pos] != '|': internalError("decodeLib: 2")
+    inc(b.pos)
+    result.path = decodeNode(r, b, info)
+
+proc decodeInstantiations(r; b; info: TLineInfo;
+                          s: var seq[PInstantiation]) =
+  while b.s[b.pos] == '\15':
+    inc(b.pos)
+    var ii: PInstantiation
+    new ii
+    ii.sym = loadSym(r, decodeVInt(b.s, b.pos), info)
+    ii.concreteTypes = @[]
+    while b.s[b.pos] == '\17':
+      inc(b.pos)
+      ii.concreteTypes.add loadType(r, decodeVInt(b.s, b.pos), info)
+    if b.s[b.pos] == '\20':
+      inc(b.pos)
+      ii.compilesId = decodeVInt(b.s, b.pos)
+    s.safeAdd ii
+
+proc loadSymFromBlob(r; b; info: TLineInfo): PSym =
+  if b.s[b.pos] == '{':
+    inc(b.pos)
+    if b.s[b.pos] == '}':
+      inc(b.pos)
+      return                  # nil sym
+  var k = TSymKind(decodeVInt(b.s, b.pos))
+  var id: int
+  if b.s[b.pos] == '+':
+    inc(b.pos)
+    id = decodeVInt(b.s, b.pos)
+    setId(id)
+  else:
+    internalError(info, "decodeSym: no id")
+  var ident: PIdent
+  if b.s[b.pos] == '&':
+    inc(b.pos)
+    ident = r.cache.getIdent(decodeStr(b.s, b.pos))
+  else:
+    internalError(info, "decodeSym: no ident")
+  #echo "decoding: {", ident.s
+  new(result)
+  result.id = id
+  result.kind = k
+  result.name = ident         # read the rest of the symbol description:
+  r.syms[result.id] = result
+  if b.s[b.pos] == '^':
+    inc(b.pos)
+    result.typ = loadType(r, decodeVInt(b.s, b.pos), info)
+  decodeLineInfo(r, b, result.info)
+  if b.s[b.pos] == '*':
+    inc(b.pos)
+    result.owner = loadSym(r, decodeVInt(b.s, b.pos), result.info)
+  if b.s[b.pos] == '$':
+    inc(b.pos)
+    result.flags = cast[TSymFlags](int32(decodeVInt(b.s, b.pos)))
+  if b.s[b.pos] == '@':
+    inc(b.pos)
+    result.magic = TMagic(decodeVInt(b.s, b.pos))
+  if b.s[b.pos] == '!':
+    inc(b.pos)
+    result.options = cast[TOptions](int32(decodeVInt(b.s, b.pos)))
+  else:
+    result.options = r.module.options
+  if b.s[b.pos] == '%':
+    inc(b.pos)
+    result.position = decodeVInt(b.s, b.pos)
+  if b.s[b.pos] == '`':
+    inc(b.pos)
+    result.offset = decodeVInt(b.s, b.pos)
+  else:
+    result.offset = - 1
+  decodeLoc(r, b, result.loc, result.info)
+  result.annex = decodeLib(r, b, info)
+  if b.s[b.pos] == '#':
+    inc(b.pos)
+    result.constraint = decodeNode(r, b, unknownLineInfo())
+  case result.kind
+  of skType, skGenericParam:
+    while b.s[b.pos] == '\14':
+      inc(b.pos)
+      result.typeInstCache.safeAdd loadType(r, decodeVInt(b.s, b.pos), result.info)
+  of routineKinds:
+    decodeInstantiations(r, b, result.info, result.procInstCache)
+    if b.s[b.pos] == '\16':
+      inc(b.pos)
+      result.gcUnsafetyReason = loadSym(r, decodeVInt(b.s, b.pos), result.info)
+  of skModule, skPackage:
+    decodeInstantiations(r, b, result.info, result.usedGenerics)
+  of skLet, skVar, skField, skForVar:
+    if b.s[b.pos] == '\18':
+      inc(b.pos)
+      result.guard = loadSym(r, decodeVInt(b.s, b.pos), result.info)
+    if b.s[b.pos] == '\19':
+      inc(b.pos)
+      result.bitsize = decodeVInt(b.s, b.pos).int16
+  else: discard
+
+  if b.s[b.pos] == '(':
+    #if result.kind in routineKinds:
+    #  result.ast = decodeNodeLazyBody(b, result.info, result)
+    #else:
+    result.ast = decodeNode(r, b, result.info)
+  if sfCompilerProc in result.flags:
+    registerCompilerProc(result)
+    #echo "loading ", result.name.s
+
+proc loadSym(r; id: int; info: TLineInfo): PSym =
+  result = r.syms.getOrDefault(id)
+  if result != nil: return result
+  var b = loadBlob(sql"select data from syms where nimid = ?", id)
+  result = loadSymFromBlob(r, b, info)
+  doAssert id == result.id, "symbol ID is not consistent!"
+
+proc loadModuleSymTab(r; module: PSym) =
+  ## goal: fill  module.tab
+  gr.syms[module.id] = module
+  for row in db.fastRows(sql"select nimid, data from syms where module = ? and exported = 1", abs(module.id)):
+    let id = parseInt(row[0])
+    var s = r.syms.getOrDefault(id)
+    if s == nil:
+      var b = BlobReader(pos: 0)
+      shallowCopy(b.s, row[1])
+      s = loadSymFromBlob(r, b, module.info)
+    assert s != nil
+    strTableAdd(module.tab, s)
+  if sfSystemModule in module.flags:
+    magicsys.systemModule = module
+
+proc loadNode*(module: PSym; index: int): PNode =
+  assert gSymbolFiles == v2Sf
+  if index == 0:
+    loadModuleSymTab(gr, module)
+    #index = parseInt db.getValue(
+    #  sql"select min(id) from toplevelstmts where module = ?", abs module.id)
+  var b = BlobReader(pos: 0)
+  b.s = db.getValue(sql"select data from toplevelstmts where position = ? and module = ?",
+                    index, abs module.id)
+  if b.s.len == 0:
+    db.exec(sql"insert into controlblock(idgen) values (?)", gFrontEndId)
+    return nil # end marker
+  gr.module = module
+  result = decodeNode(gr, b, module.info)
+
+proc addModuleDep*(module, fileIdx: int32; isIncludeFile: bool) =
+  if gSymbolFiles != v2Sf: return
+
+  let a = toDbFileId(module)
+  let b = toDbFileId(fileIdx)
+
+  db.exec(sql"insert into deps(module, dependency, isIncludeFile) values (?, ?, ?)",
+    a, b, ord(isIncludeFile))
+
+# --------------- Database model ---------------------------------------------
+
+proc createDb() =
+  db.exec(sql"""
+    create table if not exists controlblock(
+      idgen integer not null
+    );
+  """)
+
+  db.exec(sql"""
+    create table if not exists filenames(
+      id integer primary key,
+      fullpath varchar(8000) not null,
+      fullHash varchar(256) not null
+    );
+  """)
+  db.exec sql"create index if not exists FilenameIx on filenames(fullpath);"
+
+  db.exec(sql"""
+    create table if not exists modules(
+      id integer primary key,
+      fullpath varchar(8000) not null,
+      interfHash varchar(256) not null,
+      fullHash varchar(256) not null,
+
+      created timestamp not null default (DATETIME('now'))
+    );""")
+  db.exec(sql"""create unique index if not exists SymNameIx on modules(fullpath);""")
+
+  db.exec(sql"""
+    create table if not exists deps(
+      id integer primary key,
+      module integer not null,
+      dependency integer not null,
+      isIncludeFile integer not null,
+      foreign key (module) references filenames(id),
+      foreign key (dependency) references filenames(id)
+    );""")
+  db.exec(sql"""create index if not exists DepsIx on deps(module);""")
+
+  db.exec(sql"""
+    create table if not exists types(
+      id integer primary key,
+      nimid integer not null,
+      module integer not null,
+      data blob not null,
+      foreign key (module) references module(id)
+    );
+  """)
+  db.exec sql"create index TypeByModuleIdx on types(module);"
+  db.exec sql"create index TypeByNimIdIdx on types(nimid);"
+
+  db.exec(sql"""
+    create table if not exists syms(
+      id integer primary key,
+      nimid integer not null,
+      module integer not null,
+      name varchar(256) not null,
+      data blob not null,
+      exported int not null,
+      foreign key (module) references module(id)
+    );
+  """)
+  db.exec sql"create index if not exists SymNameIx on syms(name);"
+  db.exec sql"create index SymByNameAndModuleIdx on syms(name, module);"
+  db.exec sql"create index SymByModuleIdx on syms(module);"
+  db.exec sql"create index SymByNimIdIdx on syms(nimid);"
+
+
+  db.exec(sql"""
+    create table if not exists toplevelstmts(
+      id integer primary key,
+      position integer not null,
+      module integer not null,
+      data blob not null,
+      foreign key (module) references module(id)
+    );
+  """)
+  db.exec sql"create index TopLevelStmtByModuleIdx on toplevelstmts(module);"
+  db.exec sql"create index TopLevelStmtByPositionIdx on toplevelstmts(position);"
+
+  db.exec(sql"""
+    create table if not exists statics(
+      id integer primary key,
+      module integer not null,
+      data blob not null,
+      foreign key (module) references module(id)
+    );
+  """)
+  db.exec sql"create index StaticsByModuleIdx on toplevelstmts(module);"
+  db.exec sql"insert into controlblock(idgen) values (0)"
+
+proc setupModuleCache* =
+  if gSymbolFiles != v2Sf: return
+  let dbfile = getNimcacheDir() / "rodfiles.db"
+  if not fileExists(dbfile):
+    db = open(connection=dbfile, user="nim", password="",
+              database="nim")
+    createDb()
+  else:
+    db = open(connection=dbfile, user="nim", password="",
+              database="nim")
+  db.exec(sql"pragma journal_mode=off")
+  db.exec(sql"pragma SYNCHRONOUS=off")
+  db.exec(sql"pragma LOCKING_MODE=exclusive")
+  let lastId = db.getValue(sql"select max(idgen) from controlblock")
+  if lastId.len > 0:
+    idgen.setId(parseInt lastId)

compiler/rodread.nim

@@ -90,7 +90,7 @@
 
 import
   os, options, strutils, nversion, ast, astalgo, msgs, platform, condsyms,
-  ropes, idents, securehash, idgen, types, rodutils, memfiles, tables
+  ropes, idents, std / sha1, idgen, types, rodutils, memfiles, tables
 
 type
   TReasonForRecompile* = enum ## all the reasons that can trigger recompilation
@@ -257,9 +257,9 @@ proc decodeLoc(r: PRodReader, loc: var TLoc, info: TLineInfo) =
       loc.k = low(loc.k)
     if r.s[r.pos] == '*':
       inc(r.pos)
-      loc.s = TStorageLoc(decodeVInt(r.s, r.pos))
+      loc.storage = TStorageLoc(decodeVInt(r.s, r.pos))
     else:
-      loc.s = low(loc.s)
+      loc.storage = low(loc.storage)
     if r.s[r.pos] == '$':
       inc(r.pos)
       loc.flags = cast[TLocFlags](int32(decodeVInt(r.s, r.pos)))
@@ -267,9 +267,10 @@ proc decodeLoc(r: PRodReader, loc: var TLoc, info: TLineInfo) =
       loc.flags = {}
     if r.s[r.pos] == '^':
       inc(r.pos)
-      loc.t = rrGetType(r, decodeVInt(r.s, r.pos), info)
+      loc.lode = decodeNode(r, info)
+      # rrGetType(r, decodeVInt(r.s, r.pos), info)
     else:
-      loc.t = nil
+      loc.lode = nil
     if r.s[r.pos] == '!':
       inc(r.pos)
       loc.r = rope(decodeStr(r.s, r.pos))
@@ -335,10 +336,13 @@ proc decodeType(r: PRodReader, info: TLineInfo): PType =
   if r.s[r.pos] == '\17':
     inc(r.pos)
     result.assignment = rrGetSym(r, decodeVInt(r.s, r.pos), info)
-  while r.s[r.pos] == '\18':
+  if r.s[r.pos] == '\18':
+    inc(r.pos)
+    result.sink = rrGetSym(r, decodeVInt(r.s, r.pos), info)
+  while r.s[r.pos] == '\19':
     inc(r.pos)
     let x = decodeVInt(r.s, r.pos)
-    doAssert r.s[r.pos] == '\19'
+    doAssert r.s[r.pos] == '\20'
     inc(r.pos)
     let y = rrGetSym(r, decodeVInt(r.s, r.pos), info)
     result.methods.safeAdd((x, y))
@@ -588,7 +592,7 @@ proc cmdChangeTriggersRecompilation(old, new: TCommands): bool =
       return false
   of cmdNone, cmdDoc, cmdInterpret, cmdPretty, cmdGenDepend, cmdDump,
       cmdCheck, cmdParse, cmdScan, cmdIdeTools, cmdDef,
-      cmdRst2html, cmdRst2tex, cmdInteractive, cmdRun:
+      cmdRst2html, cmdRst2tex, cmdInteractive, cmdRun, cmdJsonScript:
     discard
   # else: trigger recompilation:
   result = true
@@ -791,7 +795,7 @@ proc getReader(moduleId: int): PRodReader =
   # the module ID! We could introduce a mapping ID->PRodReader but I'll leave
   # this for later versions if benchmarking shows the linear search causes
   # problems:
-  for i in 0 .. <gMods.len:
+  for i in 0 ..< gMods.len:
     result = gMods[i].rd
     if result != nil and result.moduleID == moduleId: return result
   return nil
@@ -809,6 +813,7 @@ proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym =
         encodeVInt(id, x)
         internalError(info, "missing from both indexes: +" & x)
       var rd = getReader(moduleID)
+      doAssert rd != nil
       d = iiTableGet(rd.index.tab, id)
       if d != InvalidKey:
         result = decodeSymSafePos(rd, d, info)
@@ -857,12 +862,11 @@ proc loadMethods(r: PRodReader) =
     if r.s[r.pos] == ' ': inc(r.pos)
 
 proc getHash*(fileIdx: int32): SecureHash =
-  internalAssert fileIdx >= 0 and fileIdx < gMods.len
-
-  if gMods[fileIdx].hashDone:
+  if fileIdx <% gMods.len and gMods[fileIdx].hashDone:
     return gMods[fileIdx].hash
 
   result = secureHashFile(fileIdx.toFullPath)
+  if fileIdx >= gMods.len: setLen(gMods, fileIdx+1)
   gMods[fileIdx].hash = result
 
 template growCache*(cache, pos) =
@@ -908,7 +912,7 @@ proc checkDep(fileIdx: int32; cache: IdentCache): TReasonForRecompile =
 
 proc handleSymbolFile*(module: PSym; cache: IdentCache): PRodReader =
   let fileIdx = module.fileIdx
-  if optSymbolFiles notin gGlobalOptions:
+  if gSymbolFiles in {disabledSf, writeOnlySf, v2Sf}:
     module.id = getID()
     return nil
   idgen.loadMaxIds(options.gProjectPath / options.gProjectName)
@@ -1233,4 +1237,4 @@ proc viewFile(rodfile: string) =
   outf.close
 
 when isMainModule:
-  viewFile(paramStr(1).addFileExt(rodExt))
+  viewFile(paramStr(1).addFileExt(RodExt))

compiler/rodutils.nim

@@ -8,22 +8,31 @@
 #
 
 ## Serialization utilities for the compiler.
-import strutils
+import strutils, math
 
-proc c_sprintf(buf, frmt: cstring) {.importc: "sprintf", header: "<stdio.h>", nodecl, varargs.}
+proc c_snprintf(s: cstring; n:uint; frmt: cstring): cint {.importc: "snprintf", header: "<stdio.h>", nodecl, varargs.}
 
 proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string =
-  if f != f:
+  case classify(f)
+  of fcNaN:
     result = "NAN"
-  elif f == 0.0:
+  of fcNegZero:
+    result = "-0.0" & literalPostfix
+  of fcZero:
     result = "0.0" & literalPostfix
-  elif f == 0.5 * f:
-    if f > 0.0: result = "INF"
-    else: result = "-INF"
+  of fcInf:
+    result = "INF"
+  of fcNegInf:
+    result = "-INF"
   else:
-    var buf: array[0..80, char]
-    c_sprintf(buf, "%#.16e" & literalPostfix, f)
-    result = $buf
+    when defined(nimNoArrayToCstringConversion):
+      result = newString(81)
+      let n = c_snprintf(result.cstring, result.len.uint, "%#.16e%s", f, literalPostfix.cstring)
+      setLen(result, n)
+    else:
+      var buf: array[0..80, char]
+      discard c_snprintf(buf.cstring, buf.len.uint, "%#.16e%s", f, literalPostfix.cstring)
+      result = $buf.cstring
 
 proc encodeStr*(s: string, result: var string) =
   for i in countup(0, len(s) - 1):
@@ -58,6 +67,8 @@ proc decodeStr*(s: cstring, pos: var int): string =
 const
   chars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 
+{.push overflowChecks: off.}
+
 # since negative numbers require a leading '-' they use up 1 byte. Thus we
 # subtract/add `vintDelta` here to save space for small negative numbers
 # which are common in ROD files:
@@ -122,6 +133,8 @@ proc decodeVInt*(s: cstring, pos: var int): int =
 proc decodeVBiggestInt*(s: cstring, pos: var int): BiggestInt =
   decodeIntImpl()
 
+{.pop.}
+
 iterator decodeVIntArray*(s: cstring): int =
   var i = 0
   while s[i] != '\0':
@@ -133,4 +146,3 @@ iterator decodeStrArray*(s: cstring): string =
   while s[i] != '\0':
     yield decodeStr(s, i)
     if s[i] == ' ': inc i
-

compiler/rodwrite.nim

@@ -13,8 +13,8 @@
 
 import
   intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform,
-  condsyms, ropes, idents, securehash, rodread, passes, importer, idgen,
-  rodutils
+  condsyms, ropes, idents, std / sha1, rodread, passes, idgen,
+  rodutils, modulepaths
 
 from modulegraphs import ModuleGraph
 
@@ -175,16 +175,17 @@ proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
   var oldLen = result.len
   result.add('<')
   if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
-  if loc.s != low(loc.s):
+  if loc.storage != low(loc.storage):
     add(result, '*')
-    encodeVInt(ord(loc.s), result)
+    encodeVInt(ord(loc.storage), result)
   if loc.flags != {}:
     add(result, '$')
     encodeVInt(cast[int32](loc.flags), result)
-  if loc.t != nil:
+  if loc.lode != nil:
     add(result, '^')
-    encodeVInt(cast[int32](loc.t.id), result)
-    pushType(w, loc.t)
+    encodeNode(w, unknownLineInfo(), loc.lode, result)
+    #encodeVInt(cast[int32](loc.t.id), result)
+    #pushType(w, loc.t)
   if loc.r != nil:
     add(result, '!')
     encodeStr($loc.r, result)
@@ -244,10 +245,14 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
     add(result, '\17')
     encodeVInt(t.assignment.id, result)
     pushSym(w, t.assignment)
-  for i, s in items(t.methods):
+  if t.sink != nil:
     add(result, '\18')
-    encodeVInt(i, result)
+    encodeVInt(t.sink.id, result)
+    pushSym(w, t.sink)
+  for i, s in items(t.methods):
     add(result, '\19')
+    encodeVInt(i, result)
+    add(result, '\20')
     encodeVInt(s.id, result)
     pushSym(w, s)
   encodeLoc(w, t.loc, result)
@@ -385,9 +390,9 @@ proc symStack(w: PRodWriter): int =
       inc result
     elif iiTableGet(w.index.tab, s.id) == InvalidKey:
       var m = getModule(s)
-      if m == nil and s.kind != skPackage and sfGenSym notin s.flags:
-        internalError("symStack: module nil: " & s.name.s)
-      if s.kind == skPackage or {sfFromGeneric, sfGenSym} * s.flags != {} or m.id == w.module.id:
+      #if m == nil and s.kind != skPackage and sfGenSym notin s.flags:
+      #  internalError("symStack: module nil: " & s.name.s & " " & $s.kind & " ID " & $s.id)
+      if m == nil or s.kind == skPackage or {sfFromGeneric, sfGenSym} * s.flags != {} or m.id == w.module.id:
         # put definition in here
         var L = w.data.len
         addToIndex(w.index, s.id, L)
@@ -406,7 +411,7 @@ proc symStack(w: PRodWriter): int =
           add(w.compilerProcs, ' ')
           encodeVInt(s.id, w.compilerProcs)
           add(w.compilerProcs, rodNL)
-        if s.kind == skConverter or hasPattern(s):
+        if s.kind == skConverter or (s.ast != nil and hasPattern(s)):
           if w.converters.len != 0: add(w.converters, ' ')
           encodeVInt(s.id, w.converters)
         if s.kind == skMethod and sfDispatcher notin s.flags:
@@ -579,7 +584,7 @@ proc process(c: PPassContext, n: PNode): PNode =
     for i in countup(0, sonsLen(n) - 1): discard process(c, n.sons[i])
     #var s = n.sons[namePos].sym
     #addInterfaceSym(w, s)
-  of nkProcDef, nkIteratorDef, nkConverterDef,
+  of nkProcDef, nkFuncDef, nkIteratorDef, nkConverterDef,
       nkTemplateDef, nkMacroDef:
     let s = n.sons[namePos].sym
     if s == nil: internalError(n.info, "rodwrite.process")
@@ -637,7 +642,7 @@ proc process(c: PPassContext, n: PNode): PNode =
 
 proc myOpen(g: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
   if module.id < 0: internalError("rodwrite: module ID not set")
-  var w = newRodWriter(module.fileIdx.getHash, module, cache)
+  var w = newRodWriter(rodread.getHash module.fileIdx, module, cache)
   rawAddInterfaceSym(w, module)
   result = w
 

compiler/scriptconfig.nim

@@ -73,13 +73,14 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
   cbos copyFile:
     os.copyFile(getString(a, 0), getString(a, 1))
   cbos getLastModificationTime:
-    setResult(a, toSeconds(getLastModificationTime(getString(a, 0))))
+    # depends on Time's implementation!
+    setResult(a, int64(getLastModificationTime(getString(a, 0))))
 
   cbos rawExec:
     setResult(a, osproc.execCmd getString(a, 0))
 
   cbconf getEnv:
-    setResult(a, os.getEnv(a.getString 0))
+    setResult(a, os.getEnv(a.getString 0, a.getString 1))
   cbconf existsEnv:
     setResult(a, os.existsEnv(a.getString 0))
   cbconf dirExists:
@@ -143,6 +144,7 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
 
 proc runNimScript*(cache: IdentCache; scriptName: string;
                    freshDefines=true; config: ConfigRef=nil) =
+  rawMessage(hintConf, scriptName)
   passes.gIncludeFile = includeModule
   passes.gImportModule = importModule
   let graph = newModuleGraph(config)

compiler/sem.nim

@@ -12,11 +12,11 @@
 import
   ast, strutils, hashes, options, lexer, astalgo, trees, treetab,
   wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math,
-  magicsys, parser, nversion, nimsets, semfold, importer,
+  magicsys, parser, nversion, nimsets, semfold, modulepaths, importer,
   procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch,
   intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2, nimfix.pretty, semmacrosanity,
-  semparallel, lowerings, pluginsupport, plugins.active
+  semparallel, lowerings, pluginsupport, plugins.active, rod
 
 from modulegraphs import ModuleGraph
 
@@ -74,7 +74,7 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
     localError(arg.info, errExprXHasNoType,
                renderTree(arg, {renderNoComments}))
     # error correction:
-    result = copyNode(arg)
+    result = copyTree(arg)
     result.typ = formal
   else:
     result = indexTypesMatch(c, formal, arg.typ, arg)
@@ -102,8 +102,8 @@ proc commonType*(x, y: PType): PType =
   # if expressions, etc.:
   if x == nil: return x
   if y == nil: return y
-  var a = skipTypes(x, {tyGenericInst, tyAlias})
-  var b = skipTypes(y, {tyGenericInst, tyAlias})
+  var a = skipTypes(x, {tyGenericInst, tyAlias, tySink})
+  var b = skipTypes(y, {tyGenericInst, tyAlias, tySink})
   result = x
   if a.kind in {tyExpr, tyNil}: result = y
   elif b.kind in {tyExpr, tyNil}: result = x
@@ -122,7 +122,7 @@ proc commonType*(x, y: PType): PType =
     if a.sons[idx].kind == tyEmpty: return y
   elif a.kind == tyTuple and b.kind == tyTuple and a.len == b.len:
     var nt: PType
-    for i in 0.. <a.len:
+    for i in 0..<a.len:
       let aEmpty = isEmptyContainer(a.sons[i])
       let bEmpty = isEmptyContainer(b.sons[i])
       if aEmpty != bEmpty:
@@ -165,6 +165,19 @@ proc commonType*(x, y: PType): PType =
         result = newType(k, r.owner)
         result.addSonSkipIntLit(r)
 
+proc endsInNoReturn(n: PNode): bool =
+  # check if expr ends in raise exception or call of noreturn proc
+  var it = n
+  while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0:
+    it = it.lastSon
+  result = it.kind == nkRaiseStmt or
+    it.kind in nkCallKinds and it[0].kind == nkSym and sfNoReturn in it[0].sym.flags
+
+proc commonType*(x: PType, y: PNode): PType =
+  # ignore exception raising branches in case/if expressions
+  if endsInNoReturn(y): return x
+  commonType(x, y.typ)
+
 proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym =
   result = newSym(kind, considerQuotedIdent(n), getCurrOwner(c), n.info)
   when defined(nimsuggest):
@@ -202,12 +215,17 @@ proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
 proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode,
                         allowed: TSymFlags): PSym
 
-proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind) =
-  let t = typeAllowed(typ, kind)
+proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind;
+                      flags: TTypeAllowedFlags = {}) =
+  let t = typeAllowed(typ, kind, flags)
   if t != nil:
-    if t == typ: localError(info, "invalid type: '" & typeToString(typ) & "'")
-    else: localError(info, "invalid type: '" & typeToString(t) &
-                           "' in this context: '" & typeToString(typ) & "'")
+    if t == typ:
+      localError(info, "invalid type: '" & typeToString(typ) &
+        "' for " & substr($kind, 2).toLowerAscii)
+    else:
+      localError(info, "invalid type: '" & typeToString(t) &
+        "' in this context: '" & typeToString(typ) &
+        "' for " & substr($kind, 2).toLowerAscii)
 
 proc paramsTypeCheck(c: PContext, typ: PType) {.inline.} =
   typeAllowedCheck(typ.n.info, typ, skProc)
@@ -423,7 +441,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
   result = evalMacroCall(c.module, c.cache, n, nOrig, sym)
   if efNoSemCheck notin flags:
     result = semAfterMacroCall(c, n, result, sym, flags)
-  result = wrapInComesFrom(nOrig.info, result)
+  result = wrapInComesFrom(nOrig.info, sym, result)
   popInfoContext()
 
 proc forceBool(c: PContext, n: PNode): PNode =
@@ -449,7 +467,7 @@ include semtypes, semtempl, semgnrc, semstmts, semexprs
 proc addCodeForGenerics(c: PContext, n: PNode) =
   for i in countup(c.lastGenericIdx, c.generics.len - 1):
     var prc = c.generics[i].inst.sym
-    if prc.kind in {skProc, skMethod, skConverter} and prc.magic == mNone:
+    if prc.kind in {skProc, skFunc, skMethod, skConverter} and prc.magic == mNone:
       if prc.ast == nil or prc.ast.sons[bodyPos] == nil:
         internalError(prc.info, "no code for " & prc.name.s)
       else:
@@ -488,6 +506,8 @@ proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
 
 proc myOpenCached(graph: ModuleGraph; module: PSym; rd: PRodReader): PPassContext =
   result = myOpen(graph, module, rd.cache)
+
+proc replayMethodDefs(graph: ModuleGraph; rd: PRodReader) =
   for m in items(rd.methods): methodDef(graph, m, true)
 
 proc isImportSystemStmt(n: PNode): bool =
@@ -495,13 +515,15 @@ proc isImportSystemStmt(n: PNode): bool =
   case n.kind
   of nkImportStmt:
     for x in n:
-      let f = checkModuleName(x, false)
+      if x.kind == nkIdent:
+        let f = checkModuleName(x, false)
+        if f == magicsys.systemModule.info.fileIndex:
+          return true
+  of nkImportExceptStmt, nkFromStmt:
+    if n[0].kind == nkIdent:
+      let f = checkModuleName(n[0], false)
       if f == magicsys.systemModule.info.fileIndex:
         return true
-  of nkImportExceptStmt, nkFromStmt:
-    let f = checkModuleName(n[0], false)
-    if f == magicsys.systemModule.info.fileIndex:
-      return true
   else: discard
 
 proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =
@@ -520,14 +542,18 @@ proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =
   else:
     result = n
   result = semStmt(c, result)
-  # BUGFIX: process newly generated generics here, not at the end!
-  if c.lastGenericIdx < c.generics.len:
-    var a = newNodeI(nkStmtList, n.info)
-    addCodeForGenerics(c, a)
-    if sonsLen(a) > 0:
-      # a generic has been added to `a`:
-      if result.kind != nkEmpty: addSon(a, result)
-      result = a
+  when false:
+    # Code generators are lazy now and can deal with undeclared procs, so these
+    # steps are not required anymore and actually harmful for the upcoming
+    # destructor support.
+    # BUGFIX: process newly generated generics here, not at the end!
+    if c.lastGenericIdx < c.generics.len:
+      var a = newNodeI(nkStmtList, n.info)
+      addCodeForGenerics(c, a)
+      if sonsLen(a) > 0:
+        # a generic has been added to `a`:
+        if result.kind != nkEmpty: addSon(a, result)
+        result = a
   result = hloStmt(c, result)
   if gCmd == cmdInteractive and not isEmptyType(result.typ):
     result = buildEchoStmt(c, result)
@@ -563,6 +589,19 @@ proc myProcess(context: PPassContext, n: PNode): PNode =
       else:
         result = ast.emptyNode
       #if gCmd == cmdIdeTools: findSuggest(c, n)
+  rod.storeNode(c.module, result)
+
+proc testExamples(c: PContext) =
+  let inp = toFullPath(c.module.info)
+  let outp = inp.changeFileExt"" & "_examples.nim"
+  renderModule(c.runnableExamples, inp, outp)
+  let backend = if isDefined("js"): "js"
+                elif isDefined("cpp"): "cpp"
+                elif isDefined("objc"): "objc"
+                else: "c"
+  if os.execShellCmd("nim " & backend & " -r " & outp) != 0:
+    quit "[Examples] failed"
+  removeFile(outp)
 
 proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
   var c = PContext(context)
@@ -576,7 +615,12 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
   addCodeForGenerics(c, result)
   if c.module.ast != nil:
     result.add(c.module.ast)
+  if c.rd != nil:
+    replayMethodDefs(graph, c.rd)
   popOwner(c)
   popProcCon(c)
+  storeRemaining(c.module)
+  if c.runnableExamples != nil: testExamples(c)
 
-const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
+const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose,
+                          isFrontend = true)

compiler/semasgn.nim

@@ -7,8 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
-## This module implements lifting for assignments. Later versions of this code
-## will be able to also lift ``=deepCopy`` and ``=destroy``.
+## This module implements lifting for type-bound operations
+## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``).
 
 # included from sem.nim
 
@@ -22,7 +22,8 @@ type
     recurse: bool
 
 proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode)
-proc liftBody(c: PContext; typ: PType; info: TLineInfo): PSym
+proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
+              info: TLineInfo): PSym {.discardable.}
 
 proc at(a, i: PNode, elemType: PType): PNode =
   result = newNodeI(nkBracketExpr, a.info, 2)
@@ -31,7 +32,7 @@ proc at(a, i: PNode, elemType: PType): PNode =
   result.typ = elemType
 
 proc liftBodyTup(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  for i in 0 .. <t.len:
+  for i in 0 ..< t.len:
     let lit = lowerings.newIntLit(i)
     liftBodyAux(c, t.sons[i], body, x.at(lit, t.sons[i]), y.at(lit, t.sons[i]))
 
@@ -57,7 +58,7 @@ proc liftBodyObj(c: var TLiftCtx; n, body, x, y: PNode) =
     var access = dotField(x, n[0].sym)
     caseStmt.add(access)
     # copy the branches over, but replace the fields with the for loop body:
-    for i in 1 .. <n.len:
+    for i in 1 ..< n.len:
       var branch = copyTree(n[i])
       let L = branch.len
       branch.sons[L-1] = newNodeI(nkStmtList, c.info)
@@ -92,13 +93,41 @@ proc newAsgnStmt(le, ri: PNode): PNode =
   result.sons[0] = le
   result.sons[1] = ri
 
-proc newDestructorCall(op: PSym; x: PNode): PNode =
+proc newOpCall(op: PSym; x: PNode): PNode =
   result = newNodeIT(nkCall, x.info, op.typ.sons[0])
   result.add(newSymNode(op))
   result.add x
 
+proc destructorCall(c: PContext; op: PSym; x: PNode): PNode =
+  result = newNodeIT(nkCall, x.info, op.typ.sons[0])
+  result.add(newSymNode(op))
+  if newDestructors:
+    result.add genAddr(c, x)
+  else:
+    result.add x
+
 proc newDeepCopyCall(op: PSym; x, y: PNode): PNode =
-  result = newAsgnStmt(x, newDestructorCall(op, y))
+  result = newAsgnStmt(x, newOpCall(op, y))
+
+proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
+                        field: PSym): bool =
+  if tfHasAsgn in t.flags:
+    var op: PSym
+    if sameType(t, c.asgnForType):
+      # generate recursive call:
+      if c.recurse:
+        op = c.fn
+      else:
+        c.recurse = true
+        return false
+    else:
+      op = field
+      if op == nil:
+        op = liftBody(c.c, t, c.kind, c.info)
+    markUsed(c.info, op, c.c.graph.usageSym)
+    styleCheckUse(c.info, op)
+    body.add newAsgnCall(c.c, op, x, y)
+    result = true
 
 proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
   case c.kind
@@ -107,26 +136,12 @@ proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
     if op != nil:
       markUsed(c.info, op, c.c.graph.usageSym)
       styleCheckUse(c.info, op)
-      body.add newDestructorCall(op, x)
+      body.add destructorCall(c.c, op, x)
       result = true
   of attachedAsgn:
-    if tfHasAsgn in t.flags:
-      var op: PSym
-      if sameType(t, c.asgnForType):
-        # generate recursive call:
-        if c.recurse:
-          op = c.fn
-        else:
-          c.recurse = true
-          return false
-      else:
-        op = t.assignment
-        if op == nil:
-          op = liftBody(c.c, t, c.info)
-      markUsed(c.info, op, c.c.graph.usageSym)
-      styleCheckUse(c.info, op)
-      body.add newAsgnCall(c.c, op, x, y)
-      result = true
+    result = considerAsgnOrSink(c, t, body, x, y, t.assignment)
+  of attachedSink:
+    result = considerAsgnOrSink(c, t, body, x, y, t.sink)
   of attachedDeepCopy:
     let op = t.deepCopy
     if op != nil:
@@ -185,10 +200,10 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case t.kind
   of tyNone, tyEmpty, tyVoid: discard
   of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString,
-      tyPtr, tyString, tyRef:
+      tyPtr, tyString, tyRef, tyOpt:
     defaultOp(c, t, body, x, y)
   of tyArray, tySequence:
-    if tfHasAsgn in t.flags:
+    if {tfHasAsgn, tfUncheckedArray} * t.flags == {tfHasAsgn}:
       if t.kind == tySequence:
         # XXX add 'nil' handling here
         body.add newSeqCall(c.c, x, y)
@@ -227,9 +242,9 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) =
      tyTypeDesc, tyGenericInvocation, tyForward:
     internalError(c.info, "assignment requested for type: " & typeToString(t))
   of tyOrdinal, tyRange, tyInferred,
-     tyGenericInst, tyFieldAccessor, tyStatic, tyVar, tyAlias:
+     tyGenericInst, tyStatic, tyVar, tyLent, tyAlias, tySink:
     liftBodyAux(c, lastSon(t), body, x, y)
-  of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("liftBodyAux")
+  of tyUnused, tyOptAsRef: internalError("liftBodyAux")
 
 proc newProcType(info: TLineInfo; owner: PSym): PType =
   result = newType(tyProc, owner)
@@ -245,12 +260,20 @@ proc addParam(procType: PType; param: PSym) =
   addSon(procType.n, newSymNode(param))
   rawAddSon(procType, param.typ)
 
-proc liftBody(c: PContext; typ: PType; info: TLineInfo): PSym =
+proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
+              info: TLineInfo): PSym =
   var a: TLiftCtx
   a.info = info
   a.c = c
+  a.kind = kind
   let body = newNodeI(nkStmtList, info)
-  result = newSym(skProc, getIdent":lifted=", typ.owner, info)
+  let procname = case kind
+                 of attachedAsgn: getIdent"="
+                 of attachedSink: getIdent"=sink"
+                 of attachedDeepCopy: getIdent"=deepcopy"
+                 of attachedDestructor: getIdent"=destroy"
+
+  result = newSym(skProc, procname, typ.owner, info)
   a.fn = result
   a.asgnForType = typ
 
@@ -261,27 +284,54 @@ proc liftBody(c: PContext; typ: PType; info: TLineInfo): PSym =
 
   result.typ = newProcType(info, typ.owner)
   result.typ.addParam dest
-  result.typ.addParam src
+  if kind != attachedDestructor:
+    result.typ.addParam src
 
   liftBodyAux(a, typ, body, newSymNode(dest).newDeref, newSymNode(src))
+  # recursion is handled explicitly, do not register the type based operation
+  # before 'liftBodyAux':
+  case kind
+  of attachedAsgn: typ.assignment = result
+  of attachedSink: typ.sink = result
+  of attachedDeepCopy: typ.deepCopy = result
+  of attachedDestructor: typ.destructor = result
 
   var n = newNodeI(nkProcDef, info, bodyPos+1)
-  for i in 0 .. < n.len: n.sons[i] = emptyNode
+  for i in 0 ..< n.len: n.sons[i] = emptyNode
   n.sons[namePos] = newSymNode(result)
   n.sons[paramsPos] = result.typ.n
   n.sons[bodyPos] = body
   result.ast = n
+  incl result.flags, sfFromGeneric
 
-  # register late as recursion is handled differently
-  typ.assignment = result
-  #echo "Produced this ", n
 
 proc getAsgnOrLiftBody(c: PContext; typ: PType; info: TLineInfo): PSym =
-  let t = typ.skipTypes({tyGenericInst, tyVar, tyAlias})
+  let t = typ.skipTypes({tyGenericInst, tyVar, tyLent, tyAlias, tySink})
   result = t.assignment
   if result.isNil:
-    result = liftBody(c, t, info)
+    result = liftBody(c, t, attachedAsgn, info)
 
 proc overloadedAsgn(c: PContext; dest, src: PNode): PNode =
   let a = getAsgnOrLiftBody(c, dest.typ, dest.info)
   result = newAsgnCall(c, a, dest, src)
+
+proc liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
+  ## In the semantic pass this is called in strategic places
+  ## to ensure we lift assignment, destructors and moves properly.
+  ## The later 'destroyer' pass depends on it.
+  if not newDestructors or not hasDestructor(typ): return
+  when false:
+    # do not produce wrong liftings while we're still instantiating generics:
+    # now disabled; breaks topttree.nim!
+    if c.typesWithOps.len > 0: return
+  let typ = typ.skipTypes({tyGenericInst, tyAlias})
+  # we generate the destructor first so that other operators can depend on it:
+  if typ.destructor == nil:
+    liftBody(c, typ, attachedDestructor, info)
+  if typ.assignment == nil:
+    liftBody(c, typ, attachedAsgn, info)
+  if typ.sink == nil:
+    liftBody(c, typ, attachedSink, info)
+
+#proc patchResolvedTypeBoundOp*(c: PContext; n: PNode): PNode =
+#  if n.kind == nkCall and

compiler/semcall.nim

@@ -27,7 +27,7 @@ proc sameMethodDispatcher(a, b: PSym): bool =
       # method collide[T](a: TThing, b: TUnit[T]) is instantiated and not
       # method collide[T](a: TUnit[T], b: TThing)! This means we need to
       # *instantiate* every candidate! However, we don't keep more than 2-3
-      # candidated around so we cannot implement that for now. So in order
+      # candidates around so we cannot implement that for now. So in order
       # to avoid subtle problems, the call remains ambiguous and needs to
       # be disambiguated by the programmer; this way the right generic is
       # instantiated.
@@ -90,6 +90,10 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
     if c.currentScope.symbols.counter == counterInitial or syms != nil:
       matches(c, n, orig, z)
       if z.state == csMatch:
+        #if sym.name.s == "==" and (n.info ?? "temp3"):
+        #  echo typeToString(sym.typ)
+        #  writeMatches(z)
+
         # little hack so that iterators are preferred over everything else:
         if sym.kind == skIterator: inc(z.exactMatches, 200)
         case best.state
@@ -102,6 +106,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
         errors.safeAdd(CandidateError(
           sym: sym,
           unmatchedVarParam: int z.mutabilityProblem,
+          firstMismatch: z.firstMismatch,
           diagnostics: z.diagnostics))
     else:
       # Symbol table has been modified. Restart and pre-calculate all syms
@@ -119,6 +124,20 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
     else:
       break
 
+proc effectProblem(f, a: PType; result: var string) =
+  if f.kind == tyProc and a.kind == tyProc:
+    if tfThread in f.flags and tfThread notin a.flags:
+      result.add "\n  This expression is not GC-safe. Annotate the " &
+          "proc with {.gcsafe.} to get extended error information."
+    elif tfNoSideEffect in f.flags and tfNoSideEffect notin a.flags:
+      result.add "\n  This expression can have side effects. Annotate the " &
+          "proc with {.noSideEffect.} to get extended error information."
+
+proc renderNotLValue(n: PNode): string =
+  result = $n
+  if n.kind in {nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv} and n.len == 2:
+    result = typeToString(n.typ.skipTypes(abstractVar)) & "(" & result & ")"
+
 proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
                             (TPreferedDesc, string) =
   var prefer = preferName
@@ -150,9 +169,31 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
     else:
       add(candidates, err.sym.getProcHeader(prefer))
     add(candidates, "\n")
-    if err.unmatchedVarParam != 0 and err.unmatchedVarParam < n.len:
+    if err.firstMismatch != 0 and n.len > 1:
+      let cond = n.len > 2
+      if cond:
+        candidates.add("  first type mismatch at position: " & $err.firstMismatch &
+          "\n  required type: ")
+      var wanted, got: PType = nil
+      if err.firstMismatch < err.sym.typ.len:
+        wanted = err.sym.typ.sons[err.firstMismatch]
+        if cond: candidates.add typeToString(wanted)
+      else:
+        if cond: candidates.add "none"
+      if err.firstMismatch < n.len:
+        if cond:
+          candidates.add "\n  but expression '"
+          candidates.add renderTree(n[err.firstMismatch])
+          candidates.add "' is of type: "
+        got = n[err.firstMismatch].typ
+        if cond: candidates.add typeToString(got)
+      if wanted != nil and got != nil:
+        effectProblem(wanted, got, candidates)
+      if cond: candidates.add "\n"
+    elif err.unmatchedVarParam != 0 and err.unmatchedVarParam < n.len:
       add(candidates, "for a 'var' type a variable needs to be passed, but '" &
-                      renderTree(n[err.unmatchedVarParam]) & "' is immutable\n")
+                      renderNotLValue(n[err.unmatchedVarParam]) &
+                      "' is immutable\n")
     for diag in err.diagnostics:
       add(candidates, diag & "\n")
 
@@ -172,10 +213,10 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) =
   let (prefer, candidates) = presentFailedCandidates(c, n, errors)
   var result = msgKindToString(errTypeMismatch)
   add(result, describeArgs(c, n, 1, prefer))
-  add(result, ')')
+  add(result, '>')
   if candidates != "":
     add(result, "\n" & msgKindToString(errButExpected) & "\n" & candidates)
-  localError(n.info, errGenerated, result)
+  localError(n.info, errGenerated, result & "\nexpression: " & $n)
 
 proc bracketNotFoundError(c: PContext; n: PNode) =
   var errors: CandidateErrors = @[]
@@ -185,7 +226,7 @@ proc bracketNotFoundError(c: PContext; n: PNode) =
   while symx != nil:
     if symx.kind in routineKinds:
       errors.add(CandidateError(sym: symx,
-                                unmatchedVarParam: 0,
+                                unmatchedVarParam: 0, firstMismatch: 0,
                                 diagnostics: nil))
     symx = nextOverloadIter(o, c, headSymbol)
   if errors.len == 0:
@@ -231,12 +272,11 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
 
     if nfDotField in n.flags:
       internalAssert f.kind == nkIdent and n.sonsLen >= 2
-      let calleeName = newStrNode(nkStrLit, f.ident.s).withInfo(n.info)
 
       # leave the op head symbol empty,
       # we are going to try multiple variants
-      n.sons[0..1] = [nil, n[1], calleeName]
-      orig.sons[0..1] = [nil, orig[1], calleeName]
+      n.sons[0..1] = [nil, n[1], f]
+      orig.sons[0..1] = [nil, orig[1], f]
 
       template tryOp(x) =
         let op = newIdentNode(getIdent(x), n.info)
@@ -251,8 +291,8 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
         tryOp "."
 
     elif nfDotSetter in n.flags and f.kind == nkIdent and n.len == 3:
-      let calleeName = newStrNode(nkStrLit,
-        f.ident.s[0..f.ident.s.len-2]).withInfo(n.info)
+      # we need to strip away the trailing '=' here:
+      let calleeName = newIdentNode(getIdent(f.ident.s[0..f.ident.s.len-2]), n.info)
       let callOp = newIdentNode(getIdent".=", n.info)
       n.sons[0..1] = [callOp, n[1], calleeName]
       orig.sons[0..1] = [callOp, orig[1], calleeName]
@@ -260,9 +300,9 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
 
     if overloadsState == csEmpty and result.state == csEmpty:
       if nfDotField in n.flags and nfExplicitCall notin n.flags:
-        localError(n.info, errUndeclaredField, considerQuotedIdent(f).s)
+        localError(n.info, errUndeclaredField, considerQuotedIdent(f, n).s)
       else:
-        localError(n.info, errUndeclaredRoutine, considerQuotedIdent(f).s)
+        localError(n.info, errUndeclaredRoutine, considerQuotedIdent(f, n).s)
       return
     elif result.state != csMatch:
       if nfExprCall in n.flags:
@@ -306,7 +346,7 @@ proc instGenericConvertersArg*(c: PContext, a: PNode, x: TCandidate) =
 proc instGenericConvertersSons*(c: PContext, n: PNode, x: TCandidate) =
   assert n.kind in nkCallKinds
   if x.genericConverter:
-    for i in 1 .. <n.len:
+    for i in 1 ..< n.len:
       instGenericConvertersArg(c, n.sons[i], x)
 
 proc indexTypesMatch(c: PContext, f, a: PType, arg: PNode): PNode =
@@ -374,7 +414,7 @@ proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode =
 
 proc canDeref(n: PNode): bool {.inline.} =
   result = n.len >= 2 and (let t = n[1].typ;
-    t != nil and t.skipTypes({tyGenericInst, tyAlias}).kind in {tyPtr, tyRef})
+    t != nil and t.skipTypes({tyGenericInst, tyAlias, tySink}).kind in {tyPtr, tyRef})
 
 proc tryDeref(n: PNode): PNode =
   result = newNodeI(nkHiddenDeref, n.info)
@@ -468,7 +508,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
     for i in countup(0, len(a)-1):
       var candidate = a.sons[i].sym
       if candidate.kind in {skProc, skMethod, skConverter,
-                            skIterator}:
+                            skFunc, skIterator}:
         # it suffices that the candidate has the proper number of generic
         # type parameters:
         if safeLen(candidate.ast.sons[genericParamsPos]) == n.len-1:
@@ -490,7 +530,7 @@ proc searchForBorrowProc(c: PContext, startScope: PScope, fn: PSym): PSym =
   var call = newNodeI(nkCall, fn.info)
   var hasDistinct = false
   call.add(newIdentNode(fn.name, fn.info))
-  for i in 1.. <fn.typ.n.len:
+  for i in 1..<fn.typ.n.len:
     let param = fn.typ.n.sons[i]
     let t = skipTypes(param.typ, abstractVar-{tyTypeDesc, tyDistinct})
     if t.kind == tyDistinct or param.typ.kind == tyDistinct: hasDistinct = true

compiler/semdata.nim

@@ -37,7 +37,6 @@ type
                               # in standalone ``except`` and ``finally``
     next*: PProcCon           # used for stacking procedure contexts
     wasForwarded*: bool       # whether the current proc has a separate header
-    bracketExpr*: PNode       # current bracket expression (for ^ support)
     mapping*: TIdTable
 
   TMatchedConcept* = object
@@ -65,11 +64,12 @@ type
     efWantStmt, efAllowStmt, efDetermineType, efExplain,
     efAllowDestructor, efWantValue, efOperand, efNoSemCheck,
     efNoProcvarCheck, efNoEvaluateGeneric, efInCall, efFromHlo,
-  
+
   TExprFlags* = set[TExprFlag]
 
   TTypeAttachedOp* = enum
     attachedAsgn,
+    attachedSink,
     attachedDeepCopy,
     attachedDestructor
 
@@ -112,6 +112,7 @@ type
     semGenerateInstance*: proc (c: PContext, fn: PSym, pt: TIdTable,
                                 info: TLineInfo): PSym
     includedFiles*: IntSet    # used to detect recursive include files
+    pureEnumFields*: TStrTable   # pure enum fields that can be used unambiguously
     userPragmas*: TStrTable
     evalContext*: PEvalContext
     unknownIdents*: IntSet     # ids of all unknown identifiers to prevent
@@ -130,6 +131,12 @@ type
     recursiveDep*: string
     suggestionsMade*: bool
     inTypeContext*: int
+    typesWithOps*: seq[(PType, PType)] #\
+      # We need to instantiate the type bound ops lazily after
+      # the generic type has been constructed completely. See
+      # tests/destructor/topttree.nim for an example that
+      # would otherwise fail.
+    runnableExamples*: PNode
 
 proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair =
   result.genericSym = s
@@ -210,12 +217,14 @@ proc newContext*(graph: ModuleGraph; module: PSym; cache: IdentCache): PContext
   result.converters = @[]
   result.patterns = @[]
   result.includedFiles = initIntSet()
+  initStrTable(result.pureEnumFields)
   initStrTable(result.userPragmas)
   result.generics = @[]
   result.unknownIdents = initIntSet()
   result.cache = cache
   result.graph = graph
   initStrTable(result.signatures)
+  result.typesWithOps = @[]
 
 
 proc inclSym(sq: var TSymSeq, s: PSym) =
@@ -250,19 +259,19 @@ proc makePtrType*(c: PContext, baseType: PType): PType =
 proc makeTypeWithModifier*(c: PContext,
                            modifier: TTypeKind,
                            baseType: PType): PType =
-  assert modifier in {tyVar, tyPtr, tyRef, tyStatic, tyTypeDesc}
+  assert modifier in {tyVar, tyLent, tyPtr, tyRef, tyStatic, tyTypeDesc}
 
-  if modifier in {tyVar, tyTypeDesc} and baseType.kind == modifier:
+  if modifier in {tyVar, tyLent, tyTypeDesc} and baseType.kind == modifier:
     result = baseType
   else:
     result = newTypeS(modifier, c)
     addSonSkipIntLit(result, baseType.assertNotNil)
 
-proc makeVarType*(c: PContext, baseType: PType): PType =
-  if baseType.kind == tyVar:
+proc makeVarType*(c: PContext, baseType: PType; kind = tyVar): PType =
+  if baseType.kind == kind:
     result = baseType
   else:
-    result = newTypeS(tyVar, c)
+    result = newTypeS(kind, c)
     addSonSkipIntLit(result, baseType.assertNotNil)
 
 proc makeTypeDesc*(c: PContext, typ: PType): PType =
@@ -331,7 +340,7 @@ proc makeNotType*(c: PContext, t1: PType): PType =
 
 proc nMinusOne*(n: PNode): PNode =
   result = newNode(nkCall, n.info, @[
-    newSymNode(getSysMagic("<", mUnaryLt)),
+    newSymNode(getSysMagic("pred", mPred)),
     n])
 
 # Remember to fix the procs below this one when you make changes!
@@ -371,7 +380,7 @@ proc makeRangeType*(c: PContext; first, last: BiggestInt;
   addSonSkipIntLit(result, intType) # basetype of range
 
 proc markIndirect*(c: PContext, s: PSym) {.inline.} =
-  if s.kind in {skProc, skConverter, skMethod, skIterator}:
+  if s.kind in {skProc, skFunc, skConverter, skMethod, skIterator}:
     incl(s.flags, sfAddrTaken)
     # XXX add to 'c' for global analysis
 

compiler/semdestruct.nim

@@ -1,245 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2013 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements destructors.
-
-# included from sem.nim
-
-# special marker values that indicates that we are
-# 1) AnalyzingDestructor: currently analyzing the type for destructor
-# generation (needed for recursive types)
-# 2) DestructorIsTrivial: completed the analysis before and determined
-# that the type has a trivial destructor
-var analyzingDestructor, destructorIsTrivial: PSym
-new(analyzingDestructor)
-new(destructorIsTrivial)
-
-var
-  destructorName = getIdent"destroy_"
-  destructorParam = getIdent"this_"
-  destructorPragma = newIdentNode(getIdent"destructor", unknownLineInfo())
-
-proc instantiateDestructor(c: PContext, typ: PType): PType
-
-proc doDestructorStuff(c: PContext, s: PSym, n: PNode) =
-  var t = s.typ.sons[1].skipTypes({tyVar})
-  if t.kind == tyGenericInvocation:
-    for i in 1 .. <t.sonsLen:
-      if t.sons[i].kind != tyGenericParam:
-        localError(n.info, errDestructorNotGenericEnough)
-        return
-    t = t.base
-  elif t.kind == tyCompositeTypeClass:
-    t = t.base
-    if t.kind != tyGenericBody:
-      localError(n.info, errDestructorNotGenericEnough)
-      return
-
-  t.destructor = s
-  # automatically insert calls to base classes' destructors
-  if n.sons[bodyPos].kind != nkEmpty:
-    for i in countup(0, t.sonsLen - 1):
-      # when inheriting directly from object
-      # there will be a single nil son
-      if t.sons[i] == nil: continue
-      let destructableT = instantiateDestructor(c, t.sons[i])
-      if destructableT != nil:
-        n.sons[bodyPos].addSon(newNode(nkCall, t.sym.info, @[
-            useSym(destructableT.destructor, c.graph.usageSym),
-            n.sons[paramsPos][1][0]]))
-
-proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode
-
-proc destroySym(c: PContext, field: PSym, holder: PNode): PNode =
-  let destructableT = instantiateDestructor(c, field.typ)
-  if destructableT != nil:
-    result = newNode(nkCall, field.info, @[
-      useSym(destructableT.destructor, c.graph.usageSym),
-      newNode(nkDotExpr, field.info, @[holder, useSym(field, c.graph.usageSym)])])
-
-proc destroyCase(c: PContext, n: PNode, holder: PNode): PNode =
-  var nonTrivialFields = 0
-  result = newNode(nkCaseStmt, n.info, @[])
-  # case x.kind
-  result.addSon(newNode(nkDotExpr, n.info, @[holder, n.sons[0]]))
-  for i in countup(1, n.len - 1):
-    # of A, B:
-    let ni = n[i]
-    var caseBranch = newNode(ni.kind, ni.info, ni.sons[0..ni.len-2])
-
-    let stmt = destroyFieldOrFields(c, ni.lastSon, holder)
-    if stmt == nil:
-      caseBranch.addSon(newNode(nkStmtList, ni.info, @[]))
-    else:
-      caseBranch.addSon(stmt)
-      nonTrivialFields += stmt.len
-
-    result.addSon(caseBranch)
-
-  # maybe no fields were destroyed?
-  if nonTrivialFields == 0:
-    result = nil
-
-proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode =
-  template maybeAddLine(e) =
-    let stmt = e
-    if stmt != nil:
-      if result == nil: result = newNode(nkStmtList)
-      result.addSon(stmt)
-
-  case field.kind
-  of nkRecCase:
-    maybeAddLine destroyCase(c, field, holder)
-  of nkSym:
-    maybeAddLine destroySym(c, field.sym, holder)
-  of nkRecList:
-    for son in field:
-      maybeAddLine destroyFieldOrFields(c, son, holder)
-  else:
-    internalAssert false
-
-proc generateDestructor(c: PContext, t: PType): PNode =
-  ## generate a destructor for a user-defined object or tuple type
-  ## returns nil if the destructor turns out to be trivial
-
-  # XXX: This may be true for some C-imported types such as
-  # Tposix_spawnattr
-  if t.n == nil or t.n.sons == nil: return
-  internalAssert t.n.kind == nkRecList
-  let destructedObj = newIdentNode(destructorParam, unknownLineInfo())
-  # call the destructods of all fields
-  result = destroyFieldOrFields(c, t.n, destructedObj)
-  # base classes' destructors will be automatically called by
-  # semProcAux for both auto-generated and user-defined destructors
-
-proc instantiateDestructor(c: PContext, typ: PType): PType =
-  # returns nil if a variable of type `typ` doesn't require a
-  # destructor. Otherwise, returns the type, which holds the
-  # destructor that must be used for the varialbe.
-  # The destructor is either user-defined or automatically
-  # generated by the compiler in a member-wise fashion.
-  var t = typ.skipGenericAlias
-  let typeHoldingUserDefinition = if t.kind == tyGenericInst: t.base else: t
-
-  if typeHoldingUserDefinition.destructor != nil:
-    # XXX: This is not entirely correct for recursive types, but we need
-    # it temporarily to hide the "destroy is already defined" problem
-    if typeHoldingUserDefinition.destructor notin
-        [analyzingDestructor, destructorIsTrivial]:
-      return typeHoldingUserDefinition
-    else:
-      return nil
-
-  t = t.skipTypes({tyGenericInst, tyAlias})
-  case t.kind
-  of tySequence, tyArray, tyOpenArray, tyVarargs:
-    t.destructor = analyzingDestructor
-    if instantiateDestructor(c, t.sons[0]) != nil:
-      t.destructor = getCompilerProc"nimDestroyRange"
-      return t
-    else:
-      return nil
-  of tyTuple, tyObject:
-    t.destructor = analyzingDestructor
-    let generated = generateDestructor(c, t)
-    if generated != nil:
-      internalAssert t.sym != nil
-      var i = t.sym.info
-      let fullDef = newNode(nkProcDef, i, @[
-        newIdentNode(destructorName, i),
-        emptyNode,
-        emptyNode,
-        newNode(nkFormalParams, i, @[
-          emptyNode,
-          newNode(nkIdentDefs, i, @[
-            newIdentNode(destructorParam, i),
-            symNodeFromType(c, makeVarType(c, t), t.sym.info),
-            emptyNode]),
-          ]),
-        newNode(nkPragma, i, @[destructorPragma]),
-        emptyNode,
-        generated
-        ])
-      let semantizedDef = semProc(c, fullDef)
-      t.destructor = semantizedDef[namePos].sym
-      return t
-    else:
-      t.destructor = destructorIsTrivial
-      return nil
-  else:
-    return nil
-
-proc createDestructorCall(c: PContext, s: PSym): PNode =
-  let varTyp = s.typ
-  if varTyp == nil or sfGlobal in s.flags: return
-  let destructableT = instantiateDestructor(c, varTyp)
-  if destructableT != nil:
-    let call = semStmt(c, newNode(nkCall, s.info, @[
-      useSym(destructableT.destructor, c.graph.usageSym),
-      useSym(s, c.graph.usageSym)]))
-    result = newNode(nkDefer, s.info, @[call])
-
-proc insertDestructors(c: PContext,
-                       varSection: PNode): tuple[outer, inner: PNode] =
-  # Accepts a var or let section.
-  #
-  # When a var section has variables with destructors
-  # the var section is split up and finally blocks are inserted
-  # immediately after all "destructable" vars
-  #
-  # In case there were no destrucable variables, the proc returns
-  # (nil, nil) and the enclosing stmt-list requires no modifications.
-  #
-  # Otherwise, after the try blocks are created, the rest of the enclosing
-  # stmt-list should be inserted in the most `inner` such block (corresponding
-  # to the last variable).
-  #
-  # `outer` is a statement list that should replace the original var section.
-  # It will include the new truncated var section followed by the outermost
-  # try block.
-  let totalVars = varSection.sonsLen
-  for j in countup(0, totalVars - 1):
-    let
-      varId = varSection[j][0]
-      varTyp = varId.sym.typ
-      info = varId.info
-
-    if varTyp == nil or sfGlobal in varId.sym.flags: continue
-    let destructableT = instantiateDestructor(c, varTyp)
-
-    if destructableT != nil:
-      var tryStmt = newNodeI(nkTryStmt, info)
-
-      if j < totalVars - 1:
-        var remainingVars = newNodeI(varSection.kind, info)
-        remainingVars.sons = varSection.sons[(j+1)..varSection.len-1]
-        let (outer, inner) = insertDestructors(c, remainingVars)
-        if outer != nil:
-          tryStmt.addSon(outer)
-          result.inner = inner
-        else:
-          result.inner = newNodeI(nkStmtList, info)
-          result.inner.addSon(remainingVars)
-          tryStmt.addSon(result.inner)
-      else:
-        result.inner = newNodeI(nkStmtList, info)
-        tryStmt.addSon(result.inner)
-
-      tryStmt.addSon(
-        newNode(nkFinally, info, @[
-          semStmt(c, newNode(nkCall, info, @[
-            useSym(destructableT.destructor, c.graph.usageSym),
-            useSym(varId.sym, c.graph.usageSym)]))]))
-
-      result.outer = newNodeI(nkStmtList, info)
-      varSection.sons.setLen(j+1)
-      result.outer.addSon(varSection)
-      result.outer.addSon(tryStmt)
-
-      return

compiler/semexprs.nim

@@ -32,7 +32,7 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     # XXX tyGenericInst here?
     if result.typ.kind == tyProc and tfUnresolved in result.typ.flags:
       localError(n.info, errProcHasNoConcreteType, n.renderTree)
-    if result.typ.kind == tyVar: result = newDeref(result)
+    if result.typ.kind in {tyVar, tyLent}: result = newDeref(result)
   elif {efWantStmt, efAllowStmt} * flags != {}:
     result.typ = newTypeS(tyVoid, c)
   else:
@@ -52,8 +52,7 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     result.typ = errorType(c)
   else:
     if efNoProcvarCheck notin flags: semProcvarCheck(c, result)
-    if result.typ.kind == tyVar: result = newDeref(result)
-    semDestructorCheck(c, result, flags)
+    if result.typ.kind in {tyVar, tyLent}: result = newDeref(result)
 
 proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   result = semExpr(c, n, flags)
@@ -66,7 +65,6 @@ proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     result.typ = errorType(c)
   else:
     semProcvarCheck(c, result)
-    semDestructorCheck(c, result, flags)
 
 proc semSymGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
   result = symChoice(c, n, s, scClosed)
@@ -183,9 +181,18 @@ proc semConv(c: PContext, n: PNode): PNode =
   result = newNodeI(nkConv, n.info)
   var targetType = semTypeNode(c, n.sons[0], nil).skipTypes({tyTypeDesc})
   maybeLiftType(targetType, c, n[0].info)
-  result.addSon copyTree(n.sons[0])
-  var op = semExprWithType(c, n.sons[1])
 
+  if targetType.kind in {tySink, tyLent}:
+    let baseType = semTypeNode(c, n.sons[1], nil).skipTypes({tyTypeDesc})
+    let t = newTypeS(targetType.kind, c)
+    t.rawAddSonNoPropagationOfTypeFlags baseType
+    result = newNodeI(nkType, n.info)
+    result.typ = makeTypeDesc(c, t)
+    return
+
+  result.addSon copyTree(n.sons[0])
+
+  var op = semExprWithType(c, n.sons[1])
   if targetType.isMetaType:
     let final = inferWithMetatype(c, targetType, op, true)
     result.addSon final
@@ -193,6 +200,8 @@ proc semConv(c: PContext, n: PNode): PNode =
     return
 
   result.typ = targetType
+  # XXX op is overwritten later on, this is likely added too early
+  # here or needs to be overwritten too then.
   addSon(result, op)
 
   if not isSymChoice(op):
@@ -246,8 +255,7 @@ proc semLowHigh(c: PContext, n: PNode, m: TMagic): PNode =
     localError(n.info, errXExpectsTypeOrValue, opToStr[m])
   else:
     n.sons[1] = semExprWithType(c, n.sons[1], {efDetermineType})
-    var typ = skipTypes(n.sons[1].typ, abstractVarRange +
-                                       {tyTypeDesc, tyFieldAccessor})
+    var typ = skipTypes(n.sons[1].typ, abstractVarRange + {tyTypeDesc})
     case typ.kind
     of tySequence, tyString, tyCString, tyOpenArray, tyVarargs:
       n.typ = getSysType(tyInt)
@@ -255,7 +263,7 @@ proc semLowHigh(c: PContext, n: PNode, m: TMagic): PNode =
       n.typ = typ.sons[0] # indextype
     of tyInt..tyInt64, tyChar, tyBool, tyEnum, tyUInt8, tyUInt16, tyUInt32:
       # do not skip the range!
-      n.typ = n.sons[1].typ.skipTypes(abstractVar + {tyFieldAccessor})
+      n.typ = n.sons[1].typ.skipTypes(abstractVar)
     of tyGenericParam:
       # prepare this for resolving in semtypinst:
       # we must use copyTree here in order to avoid creating a cycle
@@ -279,7 +287,7 @@ proc isOpImpl(c: PContext, n: PNode, flags: TExprFlags): PNode =
     n[1].typ != nil and n[1].typ.kind == tyTypeDesc and
     n[2].kind in {nkStrLit..nkTripleStrLit, nkType}
 
-  let t1 = n[1].typ.skipTypes({tyTypeDesc, tyFieldAccessor})
+  let t1 = n[1].typ.skipTypes({tyTypeDesc})
 
   if n[2].kind in {nkStrLit..nkTripleStrLit}:
     case n[2].strVal.normalize
@@ -353,7 +361,7 @@ proc changeType(n: PNode, newType: PType, check: bool) =
     for i in countup(0, sonsLen(n) - 1):
       changeType(n.sons[i], elemType(newType), check)
   of nkPar:
-    let tup = newType.skipTypes({tyGenericInst, tyAlias})
+    let tup = newType.skipTypes({tyGenericInst, tyAlias, tySink})
     if tup.kind != tyTuple:
       if tup.kind == tyObject: return
       globalError(n.info, "no tuple type for constructor")
@@ -396,7 +404,7 @@ proc arrayConstrType(c: PContext, n: PNode): PType =
   if sonsLen(n) == 0:
     rawAddSon(typ, newTypeS(tyEmpty, c)) # needs an empty basetype!
   else:
-    var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyOrdinal, tyAlias})
+    var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink})
     addSonSkipIntLit(typ, t)
   typ.sons[0] = makeRangeType(c, 0, sonsLen(n) - 1, n.info)
   result = typ
@@ -420,7 +428,7 @@ proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
     let yy = semExprWithType(c, x)
     var typ = yy.typ
     addSon(result, yy)
-    #var typ = skipTypes(result.sons[0].typ, {tyGenericInst, tyVar, tyOrdinal})
+    #var typ = skipTypes(result.sons[0].typ, {tyGenericInst, tyVar, tyLent, tyOrdinal})
     for i in countup(1, sonsLen(n) - 1):
       x = n.sons[i]
       if x.kind == nkExprColonExpr and sonsLen(x) == 2:
@@ -437,12 +445,12 @@ proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
       #addSon(result, fitNode(c, typ, n.sons[i]))
       inc(lastIndex)
     addSonSkipIntLit(result.typ, typ)
-    for i in 0 .. <result.len:
+    for i in 0 ..< result.len:
       result.sons[i] = fitNode(c, typ, result.sons[i], result.sons[i].info)
   result.typ.sons[0] = makeRangeType(c, 0, sonsLen(result) - 1, n.info)
 
 proc fixAbstractType(c: PContext, n: PNode) =
-  for i in 1 .. < n.len:
+  for i in 1 ..< n.len:
     let it = n.sons[i]
     # do not get rid of nkHiddenSubConv for OpenArrays, the codegen needs it:
     if it.kind == nkHiddenSubConv and
@@ -466,7 +474,7 @@ proc newHiddenAddrTaken(c: PContext, n: PNode): PNode =
     result = newNodeIT(nkHiddenAddr, n.info, makeVarType(c, n.typ))
     addSon(result, n)
     if isAssignable(c, n) notin {arLValue, arLocalLValue}:
-      localError(n.info, errVarForOutParamNeeded)
+      localError(n.info, errVarForOutParamNeededX, renderNotLValue(n))
 
 proc analyseIfAddressTaken(c: PContext, n: PNode): PNode =
   result = n
@@ -474,7 +482,7 @@ proc analyseIfAddressTaken(c: PContext, n: PNode): PNode =
   of nkSym:
     # n.sym.typ can be nil in 'check' mode ...
     if n.sym.typ != nil and
-        skipTypes(n.sym.typ, abstractInst-{tyTypeDesc}).kind != tyVar:
+        skipTypes(n.sym.typ, abstractInst-{tyTypeDesc}).kind notin {tyVar, tyLent}:
       incl(n.sym.flags, sfAddrTaken)
       result = newHiddenAddrTaken(c, n)
   of nkDotExpr:
@@ -482,12 +490,12 @@ proc analyseIfAddressTaken(c: PContext, n: PNode): PNode =
     if n.sons[1].kind != nkSym:
       internalError(n.info, "analyseIfAddressTaken")
       return
-    if skipTypes(n.sons[1].sym.typ, abstractInst-{tyTypeDesc}).kind != tyVar:
+    if skipTypes(n.sons[1].sym.typ, abstractInst-{tyTypeDesc}).kind notin {tyVar, tyLent}:
       incl(n.sons[1].sym.flags, sfAddrTaken)
       result = newHiddenAddrTaken(c, n)
   of nkBracketExpr:
     checkMinSonsLen(n, 1)
-    if skipTypes(n.sons[0].typ, abstractInst-{tyTypeDesc}).kind != tyVar:
+    if skipTypes(n.sons[0].typ, abstractInst-{tyTypeDesc}).kind notin {tyVar, tyLent}:
       if n.sons[0].kind == nkSym: incl(n.sons[0].sym.flags, sfAddrTaken)
       result = newHiddenAddrTaken(c, n)
   else:
@@ -502,7 +510,7 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) =
 
   # get the real type of the callee
   # it may be a proc var with a generic alias type, so we skip over them
-  var t = n.sons[0].typ.skipTypes({tyGenericInst, tyAlias})
+  var t = n.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink})
 
   if n.sons[0].kind == nkSym and n.sons[0].sym.magic in FakeVarParams:
     # BUGFIX: check for L-Value still needs to be done for the arguments!
@@ -510,9 +518,10 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) =
     for i in countup(1, sonsLen(n) - 1):
       if i < sonsLen(t) and t.sons[i] != nil and
           skipTypes(t.sons[i], abstractInst-{tyTypeDesc}).kind == tyVar:
-        if isAssignable(c, n.sons[i]) notin {arLValue, arLocalLValue}:
-          if n.sons[i].kind != nkHiddenAddr:
-            localError(n.sons[i].info, errVarForOutParamNeeded)
+        let it = n[i]
+        if isAssignable(c, it) notin {arLValue, arLocalLValue}:
+          if it.kind != nkHiddenAddr:
+            localError(it.info, errVarForOutParamNeededX, $it)
     return
   for i in countup(1, sonsLen(n) - 1):
     if n.sons[i].kind == nkHiddenCallConv:
@@ -540,7 +549,7 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
     var call = newNodeIT(nkCall, n.info, n.typ)
     call.add(n.sons[0])
     var allConst = true
-    for i in 1 .. < n.len:
+    for i in 1 ..< n.len:
       var a = getConstExpr(c.module, n.sons[i])
       if a == nil:
         allConst = false
@@ -551,7 +560,6 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
       result = semfold.getConstExpr(c.module, call)
       if result.isNil: result = n
       else: return result
-    result.typ = semfold.getIntervalType(callee.magic, call)
 
   block maybeLabelAsStatic:
     # XXX: temporary work-around needed for tlateboundstatic.
@@ -559,7 +567,7 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
     # done until we have a more robust infrastructure for
     # implicit statics.
     if n.len > 1:
-      for i in 1 .. <n.len:
+      for i in 1 ..< n.len:
         # see bug #2113, it's possible that n[i].typ for errornous code:
         if n[i].typ.isNil or n[i].typ.kind != tyStatic or
             tfUnresolved notin n[i].typ.flags:
@@ -574,14 +582,14 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
         optImplicitStatic notin gOptions: return
 
     if callee.magic notin ctfeWhitelist: return
-    if callee.kind notin {skProc, skConverter} or callee.isGenericRoutine:
+    if callee.kind notin {skProc, skFunc, skConverter} or callee.isGenericRoutine:
       return
 
     if n.typ != nil and typeAllowed(n.typ, skConst) != nil: return
 
     var call = newNodeIT(nkCall, n.info, n.typ)
     call.add(n.sons[0])
-    for i in 1 .. < n.len:
+    for i in 1 ..< n.len:
       let a = getConstExpr(c.module, n.sons[i])
       if a == nil: return n
       call.add(a)
@@ -616,10 +624,10 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
     # for typeof support.
     # for ``type(countup(1,3))``, see ``tests/ttoseq``.
     result = semOverloadedCall(c, n, nOrig,
-      {skProc, skMethod, skConverter, skMacro, skTemplate, skIterator}, flags)
+      {skProc, skFunc, skMethod, skConverter, skMacro, skTemplate, skIterator}, flags)
   else:
     result = semOverloadedCall(c, n, nOrig,
-      {skProc, skMethod, skConverter, skMacro, skTemplate}, flags)
+      {skProc, skFunc, skMethod, skConverter, skMacro, skTemplate}, flags)
 
   if result != nil:
     if result.sons[0].kind != nkSym:
@@ -655,7 +663,7 @@ proc bracketedMacro(n: PNode): PSym =
       result = nil
 
 proc setGenericParams(c: PContext, n: PNode) =
-  for i in 1 .. <n.len:
+  for i in 1 ..< n.len:
     n[i].typ = semTypeNode(c, n[i], nil)
 
 proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode =
@@ -671,6 +679,8 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode =
     analyseIfAddressTakenInCall(c, result)
     if callee.magic != mNone:
       result = magicsAfterOverloadResolution(c, result, flags)
+    if result.typ != nil: liftTypeBoundOps(c, result.typ, n.info)
+    #result = patchResolvedTypeBoundOp(c, result)
   if c.matchedConcept == nil:
     result = evalAtCompileTime(c, result)
 
@@ -693,7 +703,7 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
   else:
     n.sons[0] = semExpr(c, n.sons[0], {efInCall})
     let t = n.sons[0].typ
-    if t != nil and t.kind == tyVar:
+    if t != nil and t.kind in {tyVar, tyLent}:
       n.sons[0] = newDeref(n.sons[0])
     elif n.sons[0].kind == nkBracketExpr:
       let s = bracketedMacro(n.sons[0])
@@ -725,7 +735,7 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
             hasErrorType = true
             break
         if not hasErrorType:
-          add(msg, ")\n" & msgKindToString(errButExpected) & "\n" &
+          add(msg, ">\n" & msgKindToString(errButExpected) & "\n" &
               typeToString(n.sons[0].typ))
           localError(n.info, errGenerated, msg)
         return errorNode(c, n)
@@ -781,6 +791,19 @@ proc buildEchoStmt(c: PContext, n: PNode): PNode =
 
 proc semExprNoType(c: PContext, n: PNode): PNode =
   result = semExpr(c, n, {efWantStmt})
+  # make an 'if' expression an 'if' statement again for backwards
+  # compatibility (.discardable was a bad idea!); bug #6980
+  var isStmt = false
+  if result.kind == nkIfExpr:
+    isStmt = true
+    for condActionPair in result:
+      let action = condActionPair.lastSon
+      if not implicitlyDiscardable(action) and not
+          endsInNoReturn(action):
+        isStmt = false
+    if isStmt:
+      result.kind = nkIfStmt
+      result.typ = nil
   discardCheck(c, result)
 
 proc isTypeExpr(n: PNode): bool =
@@ -853,7 +876,7 @@ proc lookupInRecordAndBuildCheck(c: PContext, n, r: PNode, field: PIdent,
 
 const
   tyTypeParamsHolders = {tyGenericInst, tyCompositeTypeClass}
-  tyDotOpTransparent = {tyVar, tyPtr, tyRef, tyAlias}
+  tyDotOpTransparent = {tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink}
 
 proc readTypeParameter(c: PContext, typ: PType,
                        paramName: PIdent, info: TLineInfo): PNode =
@@ -928,7 +951,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     else:
       result = semMacroExpr(c, n, n, s, flags)
   of skTemplate:
-    if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0:
+    if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0 or
+       sfCustomPragma in sym.flags:
       markUsed(n.info, s, c.graph.usageSym)
       styleCheckUse(n.info, s)
       result = newSymNode(s, n.info)
@@ -986,8 +1010,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     while p != nil and p.selfSym == nil:
       p = p.next
     if p != nil and p.selfSym != nil:
-      var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyPtr, tyRef,
-                                         tyAlias})
+      var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef,
+                                         tyAlias, tySink})
       while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct})
       var check: PNode = nil
       if ty.kind == tyObject:
@@ -1087,9 +1111,6 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
         if field != nil:
           n.typ = makeTypeDesc(c, field.typ)
           return n
-          #n.typ = newTypeWithSons(c, tyFieldAccessor, @[ty, field.typ])
-          #n.typ.n = copyTree(n)
-          #return n
     else:
       tryReadingGenericParam(ty)
       return
@@ -1099,7 +1120,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
     return nil
   if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass:
     ty = ty.lastSon
-  ty = skipTypes(ty, {tyGenericInst, tyVar, tyPtr, tyRef, tyAlias})
+  ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink})
   while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct})
   var check: PNode = nil
   if ty.kind == tyObject:
@@ -1166,7 +1187,7 @@ proc semDeref(c: PContext, n: PNode): PNode =
   checkSonsLen(n, 1)
   n.sons[0] = semExprWithType(c, n.sons[0])
   result = n
-  var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyAlias})
+  var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyLent, tyAlias, tySink})
   case t.kind
   of tyRef, tyPtr: n.typ = t.lastSon
   else: result = nil
@@ -1186,13 +1207,12 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
   n.sons[0] = semExprWithType(c, n.sons[0],
                               {efNoProcvarCheck, efNoEvaluateGeneric})
   let arr = skipTypes(n.sons[0].typ, {tyGenericInst,
-                                      tyVar, tyPtr, tyRef, tyAlias})
+                                      tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink})
   case arr.kind
   of tyArray, tyOpenArray, tyVarargs, tySequence, tyString,
      tyCString:
     if n.len != 2: return nil
     n.sons[0] = makeDeref(n.sons[0])
-    c.p.bracketExpr = n.sons[0]
     for i in countup(1, sonsLen(n) - 1):
       n.sons[i] = semExprWithType(c, n.sons[i],
                                   flags*{efInTypeof, efDetermineType})
@@ -1213,10 +1233,9 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
   of tyTuple:
     if n.len != 2: return nil
     n.sons[0] = makeDeref(n.sons[0])
-    c.p.bracketExpr = n.sons[0]
     # [] operator for tuples requires constant expression:
     n.sons[1] = semConstExpr(c, n.sons[1])
-    if skipTypes(n.sons[1].typ, {tyGenericInst, tyRange, tyOrdinal, tyAlias}).kind in
+    if skipTypes(n.sons[1].typ, {tyGenericInst, tyRange, tyOrdinal, tyAlias, tySink}).kind in
         {tyInt..tyInt64}:
       var idx = getOrdValue(n.sons[1])
       if idx >= 0 and idx < sonsLen(arr): n.typ = arr.sons[int(idx)]
@@ -1230,7 +1249,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
             else: nil
     if s != nil:
       case s.kind
-      of skProc, skMethod, skConverter, skIterator:
+      of skProc, skFunc, skMethod, skConverter, skIterator:
         # type parameters: partial generic specialization
         n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s)
         result = explicitGenericInstantiation(c, n, s)
@@ -1251,17 +1270,13 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
       of skType:
         result = symNodeFromType(c, semTypeNode(c, n, nil), n.info)
       else:
-        c.p.bracketExpr = n.sons[0]
-    else:
-      c.p.bracketExpr = n.sons[0]
+        discard
 
 proc semArrayAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
-  let oldBracketExpr = c.p.bracketExpr
   result = semSubscript(c, n, flags)
   if result == nil:
     # overloaded [] operator:
     result = semExpr(c, buildOverloadedSubscripts(n, getIdent"[]"))
-  c.p.bracketExpr = oldBracketExpr
 
 proc propertyWriteAccess(c: PContext, n, nOrig, a: PNode): PNode =
   var id = considerQuotedIdent(a[1], a)
@@ -1281,7 +1296,7 @@ proc propertyWriteAccess(c: PContext, n, nOrig, a: PNode): PNode =
     #fixAbstractType(c, result)
     #analyseIfAddressTakenInCall(c, result)
 
-proc takeImplicitAddr(c: PContext, n: PNode): PNode =
+proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode =
   case n.kind
   of nkHiddenAddr, nkAddr: return n
   of nkHiddenDeref, nkDerefExpr: return n.sons[0]
@@ -1292,7 +1307,7 @@ proc takeImplicitAddr(c: PContext, n: PNode): PNode =
   if valid != arLValue:
     if valid == arLocalLValue:
       localError(n.info, errXStackEscape, renderTree(n, {renderNoComments}))
-    else:
+    elif not isLent:
       localError(n.info, errExprHasNoAddress)
   result = newNodeIT(nkHiddenAddr, n.info, makePtrType(c, n.typ))
   result.add(n)
@@ -1300,9 +1315,9 @@ proc takeImplicitAddr(c: PContext, n: PNode): PNode =
 proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
   if le.kind == nkHiddenDeref:
     var x = le.sons[0]
-    if x.typ.kind == tyVar and x.kind == nkSym and x.sym.kind == skResult:
+    if x.typ.kind in {tyVar, tyLent} and x.kind == nkSym and x.sym.kind == skResult:
       n.sons[0] = x # 'result[]' --> 'result'
-      n.sons[1] = takeImplicitAddr(c, ri)
+      n.sons[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent)
       x.typ.flags.incl tfVarIsPtr
       #echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info
 
@@ -1330,7 +1345,6 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
   of nkBracketExpr:
     # a[i] = x
     # --> `[]=`(a, i, x)
-    let oldBracketExpr = c.p.bracketExpr
     a = semSubscript(c, a, {efLValue})
     if a == nil:
       result = buildOverloadedSubscripts(n.sons[0], getIdent"[]=")
@@ -1340,9 +1354,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
         return n
       else:
         result = semExprNoType(c, result)
-        c.p.bracketExpr = oldBracketExpr
         return result
-    c.p.bracketExpr = oldBracketExpr
   of nkCurlyExpr:
     # a{i} = x -->  `{}=`(a, i, x)
     result = buildOverloadedSubscripts(n.sons[0], getIdent"{}=")
@@ -1362,7 +1374,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
   # a = b # both are vars, means: a[] = b[]
   # a = b # b no 'var T' means: a = addr(b)
   var le = a.typ
-  if (skipTypes(le, {tyGenericInst, tyAlias}).kind != tyVar and
+  if (skipTypes(le, {tyGenericInst, tyAlias, tySink}).kind != tyVar and
         isAssignable(c, a) == arNone) or
       skipTypes(le, abstractVar).kind in {tyOpenArray, tyVarargs}:
     # Direct assignment to a discriminant is allowed!
@@ -1378,18 +1390,24 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
     if lhsIsResult:
       n.typ = enforceVoidContext
       if c.p.owner.kind != skMacro and resultTypeIsInferrable(lhs.sym.typ):
-        if cmpTypes(c, lhs.typ, rhs.typ) == isGeneric:
+        var rhsTyp = rhs.typ
+        if rhsTyp.kind in tyUserTypeClasses and rhsTyp.isResolvedUserTypeClass:
+          rhsTyp = rhsTyp.lastSon
+        if cmpTypes(c, lhs.typ, rhsTyp) in {isGeneric, isEqual}:
           internalAssert c.p.resultSym != nil
-          lhs.typ = rhs.typ
-          c.p.resultSym.typ = rhs.typ
-          c.p.owner.typ.sons[0] = rhs.typ
+          lhs.typ = rhsTyp
+          c.p.resultSym.typ = rhsTyp
+          c.p.owner.typ.sons[0] = rhsTyp
         else:
-          typeMismatch(n.info, lhs.typ, rhs.typ)
+          typeMismatch(n.info, lhs.typ, rhsTyp)
 
     n.sons[1] = fitNode(c, le, rhs, n.info)
-    if tfHasAsgn in lhs.typ.flags and not lhsIsResult and
-        mode != noOverloadedAsgn:
-      return overloadedAsgn(c, lhs, n.sons[1])
+    if not newDestructors:
+      if tfHasAsgn in lhs.typ.flags and not lhsIsResult and
+          mode != noOverloadedAsgn:
+        return overloadedAsgn(c, lhs, n.sons[1])
+    else:
+      liftTypeBoundOps(c, lhs.typ, lhs.info)
 
     fixAbstractType(c, n)
     asgnToResultVar(c, n, n.sons[0], n.sons[1])
@@ -1398,7 +1416,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
 proc semReturn(c: PContext, n: PNode): PNode =
   result = n
   checkSonsLen(n, 1)
-  if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro} or (
+  if c.p.owner.kind in {skConverter, skMethod, skProc, skFunc, skMacro} or (
      c.p.owner.kind == skIterator and c.p.owner.typ.callConv == ccClosure):
     if n.sons[0].kind != nkEmpty:
       # transform ``return expr`` to ``result = expr; return``
@@ -1419,11 +1437,7 @@ proc semProcBody(c: PContext, n: PNode): PNode =
   openScope(c)
   result = semExpr(c, n)
   if c.p.resultSym != nil and not isEmptyType(result.typ):
-    # transform ``expr`` to ``result = expr``, but not if the expr is already
-    # ``result``:
-    if result.kind == nkSym and result.sym == c.p.resultSym:
-      discard
-    elif result.kind == nkNilLit:
+    if result.kind == nkNilLit:
       # or ImplicitlyDiscardable(result):
       # new semantic: 'result = x' triggers the void context
       result.typ = nil
@@ -1453,23 +1467,24 @@ proc semProcBody(c: PContext, n: PNode): PNode =
   closeScope(c)
 
 proc semYieldVarResult(c: PContext, n: PNode, restype: PType) =
-  var t = skipTypes(restype, {tyGenericInst, tyAlias})
+  var t = skipTypes(restype, {tyGenericInst, tyAlias, tySink})
   case t.kind
-  of tyVar:
+  of tyVar, tyLent:
+    if t.kind == tyVar: t.flags.incl tfVarIsPtr # bugfix for #4048, #4910, #6892
     if n.sons[0].kind in {nkHiddenStdConv, nkHiddenSubConv}:
       n.sons[0] = n.sons[0].sons[1]
-
-    n.sons[0] = takeImplicitAddr(c, n.sons[0])
+    n.sons[0] = takeImplicitAddr(c, n.sons[0], t.kind == tyLent)
   of tyTuple:
-    for i in 0.. <t.sonsLen:
-      var e = skipTypes(t.sons[i], {tyGenericInst, tyAlias})
-      if e.kind == tyVar:
+    for i in 0..<t.sonsLen:
+      var e = skipTypes(t.sons[i], {tyGenericInst, tyAlias, tySink})
+      if e.kind in {tyVar, tyLent}:
+        if e.kind == tyVar: e.flags.incl tfVarIsPtr # bugfix for #4048, #4910, #6892
         if n.sons[0].kind == nkPar:
-          n.sons[0].sons[i] = takeImplicitAddr(c, n.sons[0].sons[i])
+          n.sons[0].sons[i] = takeImplicitAddr(c, n.sons[0].sons[i], e.kind == tyLent)
         elif n.sons[0].kind in {nkHiddenStdConv, nkHiddenSubConv} and
              n.sons[0].sons[1].kind == nkPar:
           var a = n.sons[0].sons[1]
-          a.sons[i] = takeImplicitAddr(c, a.sons[i])
+          a.sons[i] = takeImplicitAddr(c, a.sons[i], false)
         else:
           localError(n.sons[0].info, errXExpected, "tuple constructor")
   else: discard
@@ -1621,9 +1636,10 @@ proc semExpandToAst(c: PContext, n: PNode): PNode =
   # Preserve the magic symbol in order to be handled in evals.nim
   internalAssert n.sons[0].sym.magic == mExpandToAst
   #n.typ = getSysSym("NimNode").typ # expandedSym.getReturnType
-  n.typ = if getCompilerProc("NimNode") != nil: sysTypeFromName"NimNode"
-          else: sysTypeFromName"PNimrodNode"
-  result = n
+  if n.kind == nkStmtList and n.len == 1: result = n[0]
+  else: result = n
+  result.typ = if getCompilerProc("NimNode") != nil: sysTypeFromName"NimNode"
+               else: sysTypeFromName"PNimrodNode"
 
 proc semExpandToAst(c: PContext, n: PNode, magicSym: PSym,
                     flags: TExprFlags = {}): PNode =
@@ -1653,7 +1669,7 @@ proc processQuotations(n: var PNode, op: string,
   elif n.kind == nkAccQuoted and op == "``":
     returnQuote n[0]
 
-  for i in 0 .. <n.safeLen:
+  for i in 0 ..< n.safeLen:
     processQuotations(n.sons[i], op, quotes, ids)
 
 proc semQuoteAst(c: PContext, n: PNode): PNode =
@@ -1661,7 +1677,7 @@ proc semQuoteAst(c: PContext, n: PNode): PNode =
   # We transform the do block into a template with a param for
   # each interpolation. We'll pass this template to getAst.
   var
-    quotedBlock = n{-1}
+    quotedBlock = n[^1]
     op = if n.len == 3: expectString(c, n[1]) else: "``"
     quotes = newSeq[PNode](1)
       # the quotes will be added to a nkCall statement
@@ -1781,6 +1797,13 @@ proc setMs(n: PNode, s: PSym): PNode =
   n.sons[0] = newSymNode(s)
   n.sons[0].info = n.info
 
+proc extractImports(n: PNode; result: PNode) =
+  if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
+    result.add copyTree(n)
+    n.kind = nkEmpty
+    return
+  for i in 0..<n.safeLen: extractImports(n[i], result)
+
 proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
   # this is a hotspot in the compiler!
   # DON'T forget to update ast.SpecialSemMagics if you add a magic here!
@@ -1821,7 +1844,7 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     dec c.inParallelStmt
   of mSpawn:
     result = setMs(n, s)
-    for i in 1 .. <n.len:
+    for i in 1 ..< n.len:
       result.sons[i] = semExpr(c, n.sons[i])
     let typ = result[^1].typ
     if not typ.isEmptyType:
@@ -1852,6 +1875,20 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
       analyseIfAddressTakenInCall(c, result)
       if callee.magic != mNone:
         result = magicsAfterOverloadResolution(c, result, flags)
+  of mRunnableExamples:
+    if gCmd == cmdDoc and n.len >= 2 and n.lastSon.kind == nkStmtList:
+      if sfMainModule in c.module.flags:
+        let inp = toFullPath(c.module.info)
+        if c.runnableExamples == nil:
+          c.runnableExamples = newTree(nkStmtList,
+            newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp))))
+        let imports = newTree(nkStmtList)
+        extractImports(n.lastSon, imports)
+        for imp in imports: c.runnableExamples.add imp
+        c.runnableExamples.add newTree(nkBlockStmt, emptyNode, copyTree n.lastSon)
+      result = setMs(n, s)
+    else:
+      result = emptyNode
   else:
     result = semDirectOp(c, n, flags)
 
@@ -1930,17 +1967,17 @@ proc semSetConstr(c: PContext, n: PNode): PNode =
         n.sons[i].sons[2] = semExprWithType(c, n.sons[i].sons[2])
         if typ == nil:
           typ = skipTypes(n.sons[i].sons[1].typ,
-                          {tyGenericInst, tyVar, tyOrdinal, tyAlias})
+                          {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink})
         n.sons[i].typ = n.sons[i].sons[2].typ # range node needs type too
       elif n.sons[i].kind == nkRange:
         # already semchecked
         if typ == nil:
           typ = skipTypes(n.sons[i].sons[0].typ,
-                          {tyGenericInst, tyVar, tyOrdinal, tyAlias})
+                          {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink})
       else:
         n.sons[i] = semExprWithType(c, n.sons[i])
         if typ == nil:
-          typ = skipTypes(n.sons[i].typ, {tyGenericInst, tyVar, tyOrdinal, tyAlias})
+          typ = skipTypes(n.sons[i].typ, {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink})
     if not isOrdinalType(typ):
       localError(n.info, errOrdinalTypeExpected)
       typ = makeRangeType(c, 0, MaxSetElements-1, n.info)
@@ -2072,7 +2109,7 @@ proc semBlock(c: PContext, n: PNode): PNode =
 proc semExport(c: PContext, n: PNode): PNode =
   var x = newNodeI(n.kind, n.info)
   #let L = if n.kind == nkExportExceptStmt: L = 1 else: n.len
-  for i in 0.. <n.len:
+  for i in 0..<n.len:
     let a = n.sons[i]
     var o: TOverloadIter
     var s = initOverloadIter(o, c, a)
@@ -2103,7 +2140,7 @@ proc shouldBeBracketExpr(n: PNode): bool =
     let b = a[0]
     if b.kind in nkSymChoices:
       for i in 0..<b.len:
-        if b[i].sym.magic == mArrGet:
+        if b[i].kind == nkSym and b[i].sym.magic == mArrGet:
           let be = newNodeI(nkBracketExpr, n.info)
           for i in 1..<a.len: be.add(a[i])
           n.sons[0] = be
@@ -2115,12 +2152,16 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   if nfSem in n.flags: return
   case n.kind
   of nkIdent, nkAccQuoted:
-    let checks = if efNoEvaluateGeneric in flags: {checkUndeclared}
-                 else: {checkUndeclared, checkModule, checkAmbiguity}
+    let checks = if efNoEvaluateGeneric in flags:
+        {checkUndeclared, checkPureEnumFields}
+      elif efInCall in flags:
+        {checkUndeclared, checkModule, checkPureEnumFields}
+      else:
+        {checkUndeclared, checkModule, checkAmbiguity, checkPureEnumFields}
     var s = qualifiedLookUp(c, n, checks)
     if c.matchedConcept == nil: semCaptureSym(s, c.p.owner)
     result = semSym(c, n, s, flags)
-    if s.kind in {skProc, skMethod, skConverter, skIterator}:
+    if s.kind in {skProc, skFunc, skMethod, skConverter, skIterator}:
       #performProcvarCheck(c, n, s)
       result = symChoice(c, n, s, scClosed)
       if result.kind == nkSym:
@@ -2209,13 +2250,13 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
         # XXX think about this more (``set`` procs)
         if n.len == 2:
           result = semConv(c, n)
+        elif contains(c.ambiguousSymbols, s.id) and n.len == 1:
+          errorUseQualifier(c, n.info, s)
         elif n.len == 1:
           result = semObjConstr(c, n, flags)
-        elif contains(c.ambiguousSymbols, s.id):
-          errorUseQualifier(c, n.info, s)
         elif s.magic == mNone: result = semDirectOp(c, n, flags)
         else: result = semMagic(c, n, s, flags)
-      of skProc, skMethod, skConverter, skIterator:
+      of skProc, skFunc, skMethod, skConverter, skIterator:
         if s.magic == mNone: result = semDirectOp(c, n, flags)
         else: result = semMagic(c, n, s, flags)
       else:
@@ -2329,6 +2370,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   of nkPragma: pragma(c, c.p.owner, n, stmtPragmas)
   of nkIteratorDef: result = semIterator(c, n)
   of nkProcDef: result = semProc(c, n)
+  of nkFuncDef: result = semFunc(c, n)
   of nkMethodDef: result = semMethod(c, n)
   of nkConverterDef: result = semConverterDef(c, n)
   of nkMacroDef: result = semMacroDef(c, n)
@@ -2357,6 +2399,11 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     if not n.sons[0].typ.isEmptyType and not implicitlyDiscardable(n.sons[0]):
       localError(n.info, errGenerated, "'defer' takes a 'void' expression")
     #localError(n.info, errGenerated, "'defer' not allowed in this context")
+  of nkGotoState, nkState:
+    if n.len != 1 and n.len != 2: illFormedAst(n)
+    for i in 0 ..< n.len:
+      n.sons[i] = semExpr(c, n.sons[i])
+  of nkComesFrom: discard "ignore the comes from information for now"
   else:
     localError(n.info, errInvalidExpressionX,
                renderTree(n, {renderNoComments}))

compiler/semfields.nim

@@ -89,7 +89,7 @@ proc semForObjectFields(c: TFieldsCtx, typ, forLoop, father: PNode) =
     access.sons[1] = newSymNode(typ.sons[0].sym, forLoop.info)
     caseStmt.add(semExprWithType(c.c, access))
     # copy the branches over, but replace the fields with the for loop body:
-    for i in 1 .. <typ.len:
+    for i in 1 ..< typ.len:
       var branch = copyTree(typ[i])
       let L = branch.len
       branch.sons[L-1] = newNodeI(nkStmtList, forLoop.info)

compiler/semfold.nim

@@ -30,7 +30,12 @@ proc newIntNodeT(intVal: BiggestInt, n: PNode): PNode =
   case skipTypes(n.typ, abstractVarRange).kind
   of tyInt:
     result = newIntNode(nkIntLit, intVal)
-    result.typ = getIntLitType(result)
+    # See bug #6989. 'pred' et al only produce an int literal type if the
+    # original type was 'int', not a distinct int etc.
+    if n.typ.kind == tyInt:
+      result.typ = getIntLitType(result)
+    else:
+      result.typ = n.typ
     # hrm, this is not correct: 1 + high(int) shouldn't produce tyInt64 ...
     #setIntLitType(result)
   of tyChar:
@@ -92,26 +97,6 @@ proc pickIntRange(a, b: PType): PType =
 proc isIntRangeOrLit(t: PType): bool =
   result = isIntRange(t) or isIntLit(t)
 
-proc pickMinInt(n: PNode): BiggestInt =
-  if n.kind in {nkIntLit..nkUInt64Lit}:
-    result = n.intVal
-  elif isIntLit(n.typ):
-    result = n.typ.n.intVal
-  elif isIntRange(n.typ):
-    result = firstOrd(n.typ)
-  else:
-    internalError(n.info, "pickMinInt")
-
-proc pickMaxInt(n: PNode): BiggestInt =
-  if n.kind in {nkIntLit..nkUInt64Lit}:
-    result = n.intVal
-  elif isIntLit(n.typ):
-    result = n.typ.n.intVal
-  elif isIntRange(n.typ):
-    result = lastOrd(n.typ)
-  else:
-    internalError(n.info, "pickMaxInt")
-
 proc makeRange(typ: PType, first, last: BiggestInt): PType =
   let minA = min(first, last)
   let maxA = max(first, last)
@@ -137,116 +122,6 @@ proc makeRangeF(typ: PType, first, last: BiggestFloat): PType =
   result.n = n
   addSonSkipIntLit(result, skipTypes(typ, {tyRange}))
 
-proc getIntervalType*(m: TMagic, n: PNode): PType =
-  # Nim requires interval arithmetic for ``range`` types. Lots of tedious
-  # work but the feature is very nice for reducing explicit conversions.
-  const ordIntLit = {nkIntLit..nkUInt64Lit}
-  result = n.typ
-
-  template commutativeOp(opr: untyped) =
-    let a = n.sons[1]
-    let b = n.sons[2]
-    if isIntRangeOrLit(a.typ) and isIntRangeOrLit(b.typ):
-      result = makeRange(pickIntRange(a.typ, b.typ),
-                         opr(pickMinInt(a), pickMinInt(b)),
-                         opr(pickMaxInt(a), pickMaxInt(b)))
-
-  template binaryOp(opr: untyped) =
-    let a = n.sons[1]
-    let b = n.sons[2]
-    if isIntRange(a.typ) and b.kind in {nkIntLit..nkUInt64Lit}:
-      result = makeRange(a.typ,
-                         opr(pickMinInt(a), pickMinInt(b)),
-                         opr(pickMaxInt(a), pickMaxInt(b)))
-
-  case m
-  of mUnaryMinusI, mUnaryMinusI64:
-    let a = n.sons[1].typ
-    if isIntRange(a):
-      # (1..3) * (-1) == (-3.. -1)
-      result = makeRange(a, 0|-|lastOrd(a), 0|-|firstOrd(a))
-  of mUnaryMinusF64:
-    let a = n.sons[1].typ
-    if isFloatRange(a):
-      result = makeRangeF(a, -getFloat(a.n.sons[1]),
-                             -getFloat(a.n.sons[0]))
-  of mAbsF64:
-    let a = n.sons[1].typ
-    if isFloatRange(a):
-      # abs(-5.. 1) == (1..5)
-      if a.n[0].floatVal <= 0.0:
-        result = makeRangeF(a, 0.0, abs(getFloat(a.n.sons[0])))
-      else:
-        result = makeRangeF(a, abs(getFloat(a.n.sons[1])),
-                               abs(getFloat(a.n.sons[0])))
-  of mAbsI:
-    let a = n.sons[1].typ
-    if isIntRange(a):
-      if a.n[0].intVal <= 0:
-        result = makeRange(a, 0, `|abs|`(getInt(a.n.sons[0])))
-      else:
-        result = makeRange(a, `|abs|`(getInt(a.n.sons[1])),
-                              `|abs|`(getInt(a.n.sons[0])))
-  of mSucc:
-    let a = n.sons[1].typ
-    let b = n.sons[2].typ
-    if isIntRange(a) and isIntLit(b):
-      # (-5.. 1) + 6 == (-5 + 6)..(-1 + 6)
-      result = makeRange(a, pickMinInt(n.sons[1]) |+| pickMinInt(n.sons[2]),
-                            pickMaxInt(n.sons[1]) |+| pickMaxInt(n.sons[2]))
-  of mPred:
-    let a = n.sons[1].typ
-    let b = n.sons[2].typ
-    if isIntRange(a) and isIntLit(b):
-      result = makeRange(a, pickMinInt(n.sons[1]) |-| pickMinInt(n.sons[2]),
-                            pickMaxInt(n.sons[1]) |-| pickMaxInt(n.sons[2]))
-  of mAddI, mAddU:
-    commutativeOp(`|+|`)
-  of mMulI, mMulU:
-    commutativeOp(`|*|`)
-  of mSubI, mSubU:
-    binaryOp(`|-|`)
-  of mBitandI:
-    # since uint64 is still not even valid for 'range' (since it's no ordinal
-    # yet), we exclude it from the list (see bug #1638) for now:
-    var a = n.sons[1]
-    var b = n.sons[2]
-    # symmetrical:
-    if b.kind notin ordIntLit: swap(a, b)
-    if b.kind in ordIntLit:
-      let x = b.intVal|+|1
-      if (x and -x) == x and x >= 0:
-        result = makeRange(n.typ, 0, b.intVal)
-  of mModU:
-    let a = n.sons[1]
-    let b = n.sons[2]
-    if b.kind in ordIntLit:
-      if b.intVal >= 0:
-        result = makeRange(n.typ, 0, b.intVal-1)
-      else:
-        result = makeRange(n.typ, b.intVal+1, 0)
-  of mModI:
-    # so ... if you ever wondered about modulo's signedness; this defines it:
-    let a = n.sons[1]
-    let b = n.sons[2]
-    if b.kind in {nkIntLit..nkUInt64Lit}:
-      if b.intVal >= 0:
-        result = makeRange(n.typ, -(b.intVal-1), b.intVal-1)
-      else:
-        result = makeRange(n.typ, b.intVal+1, -(b.intVal+1))
-  of mDivI, mDivU:
-    binaryOp(`|div|`)
-  of mMinI:
-    commutativeOp(min)
-  of mMaxI:
-    commutativeOp(max)
-  else: discard
-
-discard """
-  mShlI,
-  mShrI, mAddF64, mSubF64, mMulF64, mDivF64, mMaxF64, mMinF64
-"""
-
 proc evalIs(n, a: PNode): PNode =
   # XXX: This should use the standard isOpImpl
   internalAssert a.kind == nkSym and a.sym.kind == skType
@@ -350,6 +225,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mDivF64:
     if getFloat(b) == 0.0:
       if getFloat(a) == 0.0: result = newFloatNodeT(NaN, n)
+      elif getFloat(b).classify == fcNegZero: result = newFloatNodeT(-Inf, n)
       else: result = newFloatNodeT(Inf, n)
     else:
       result = newFloatNodeT(getFloat(a) / getFloat(b), n)
@@ -499,7 +375,12 @@ proc getAppType(n: PNode): PNode =
     result = newStrNodeT("console", n)
 
 proc rangeCheck(n: PNode, value: BiggestInt) =
-  if value < firstOrd(n.typ) or value > lastOrd(n.typ):
+  var err = false
+  if n.typ.skipTypes({tyRange}).kind in {tyUInt..tyUInt64}:
+    err = value <% firstOrd(n.typ) or value >% lastOrd(n.typ, fixedUnsigned=true)
+  else:
+    err = value < firstOrd(n.typ) or value > lastOrd(n.typ)
+  if err:
     localError(n.info, errGenerated, "cannot convert " & $value &
                                      " to " & typeToString(n.typ))
 
@@ -538,7 +419,7 @@ proc getArrayConstr(m: PSym, n: PNode): PNode =
 
 proc foldArrayAccess(m: PSym, n: PNode): PNode =
   var x = getConstExpr(m, n.sons[0])
-  if x == nil or x.typ.skipTypes({tyGenericInst, tyAlias}).kind == tyTypeDesc:
+  if x == nil or x.typ.skipTypes({tyGenericInst, tyAlias, tySink}).kind == tyTypeDesc:
     return
 
   var y = getConstExpr(m, n.sons[1])
@@ -616,6 +497,8 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
       of mCpuEndian: result = newIntNodeT(ord(CPU[targetCPU].endian), n)
       of mHostOS: result = newStrNodeT(toLowerAscii(platform.OS[targetOS].name), n)
       of mHostCPU: result = newStrNodeT(platform.CPU[targetCPU].name.toLowerAscii, n)
+      of mBuildOS: result = newStrNodeT(toLowerAscii(platform.OS[platform.hostOS].name), n)
+      of mBuildCPU: result = newStrNodeT(platform.CPU[platform.hostCPU].name.toLowerAscii, n)
       of mAppType: result = getAppType(n)
       of mNaN: result = newFloatNodeT(NaN, n)
       of mInf: result = newFloatNodeT(Inf, n)
@@ -628,7 +511,7 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
           result = newStrNodeT(lookupSymbol(s.name), n)
       else:
         result = copyTree(s.ast)
-    of {skProc, skMethod}:
+    of {skProc, skFunc, skMethod}:
       result = n
     of skType:
       # XXX gensym'ed symbols can come here and cannot be resolved. This is
@@ -652,7 +535,7 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
   of nkCallKinds:
     if n.sons[0].kind != nkSym: return
     var s = n.sons[0].sym
-    if s.kind != skProc: return
+    if s.kind != skProc and s.kind != skFunc: return
     try:
       case s.magic
       of mNone:
@@ -734,13 +617,13 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
       if a == nil: return nil
       result.sons[i] = a
     incl(result.flags, nfAllConst)
-  of nkObjConstr:
-    result = copyTree(n)
-    for i in countup(1, sonsLen(n) - 1):
-      var a = getConstExpr(m, n.sons[i].sons[1])
-      if a == nil: return nil
-      result.sons[i].sons[1] = a
-    incl(result.flags, nfAllConst)
+  #of nkObjConstr:
+  #  result = copyTree(n)
+  #  for i in countup(1, sonsLen(n) - 1):
+  #    var a = getConstExpr(m, n.sons[i].sons[1])
+  #    if a == nil: return nil
+  #    result.sons[i].sons[1] = a
+  #  incl(result.flags, nfAllConst)
   of nkPar:
     # tuple constructor
     result = copyTree(n)
@@ -783,5 +666,8 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
       result.typ = n.typ
   of nkBracketExpr: result = foldArrayAccess(m, n)
   of nkDotExpr: result = foldFieldAccess(m, n)
+  of nkStmtListExpr:
+    if n.len == 2 and n[0].kind == nkComesFrom:
+      result = getConstExpr(m, n[1])
   else:
     discard

compiler/semgnrc.nim

@@ -61,7 +61,7 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
   of skUnknown:
     # Introduced in this pass! Leave it as an identifier.
     result = n
-  of skProc, skMethod, skIterator, skConverter, skModule:
+  of skProc, skFunc, skMethod, skIterator, skConverter, skModule:
     result = symChoice(c, n, s, scOpen)
   of skTemplate:
     if macroToExpandSym(s):
@@ -105,6 +105,10 @@ proc lookup(c: PContext, n: PNode, flags: TSemGenericFlags,
   result = n
   let ident = considerQuotedIdent(n)
   var s = searchInScopes(c, ident).skipAlias(n)
+  if s == nil:
+    s = strTableGet(c.pureEnumFields, ident)
+    if s != nil and contains(c.ambiguousSymbols, s.id):
+      s = nil
   if s == nil:
     if ident.id notin ctx.toMixin and withinMixin notin flags:
       errorUndeclaredIdentifier(c, n.info, ident.s)
@@ -182,7 +186,7 @@ proc semGenericStmt(c: PContext, n: PNode,
     let a = n.sym
     let b = getGenSym(c, a)
     if b != a: n.sym = b
-  of nkEmpty, succ(nkSym)..nkNilLit:
+  of nkEmpty, succ(nkSym)..nkNilLit, nkComesFrom:
     # see tests/compile/tgensymgeneric.nim:
     # We need to open the gensym'ed symbol again so that the instantiation
     # creates a fresh copy; but this is wrong the very first reason for gensym
@@ -206,7 +210,7 @@ proc semGenericStmt(c: PContext, n: PNode,
         considerQuotedIdent(fn).id notin ctx.toMixin:
       errorUndeclaredIdentifier(c, n.info, fn.renderTree)
 
-    var first = ord(withinConcept in flags)
+    var first = int ord(withinConcept in flags)
     var mixinContext = false
     if s != nil:
       incl(s.flags, sfUsed)
@@ -239,7 +243,7 @@ proc semGenericStmt(c: PContext, n: PNode,
       of skUnknown, skParam:
         # Leave it as an identifier.
         discard
-      of skProc, skMethod, skIterator, skConverter, skModule:
+      of skProc, skFunc, skMethod, skIterator, skConverter, skModule:
         result.sons[0] = sc
         # do not check of 's.magic==mRoof' here because it might be some
         # other '^' but after overload resolution the proper one:
@@ -331,8 +335,10 @@ proc semGenericStmt(c: PContext, n: PNode,
     n.sons[L - 2] = semGenericStmt(c, n.sons[L-2], flags, ctx)
     for i in countup(0, L - 3):
       addTempDecl(c, n.sons[i], skForVar)
+    openScope(c)
     n.sons[L - 1] = semGenericStmt(c, n.sons[L-1], flags, ctx)
     closeScope(c)
+    closeScope(c)
   of nkBlockStmt, nkBlockExpr, nkBlockType:
     checkSonsLen(n, 2)
     openScope(c)
@@ -433,7 +439,7 @@ proc semGenericStmt(c: PContext, n: PNode,
       for j in countup(0, L-3):
         addTempDecl(c, getIdentNode(a.sons[j]), skParam)
   of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef,
-     nkIteratorDef, nkLambdaKinds:
+     nkFuncDef, nkIteratorDef, nkLambdaKinds:
     checkSonsLen(n, bodyPos + 1)
     if n.sons[namePos].kind != nkEmpty:
       addTempDecl(c, getIdentNode(n.sons[0]), skProc)

compiler/seminst.nim

@@ -36,7 +36,7 @@ proc rawPushProcCon(c: PContext, owner: PSym) =
   c.p = x
 
 proc rawHandleSelf(c: PContext; owner: PSym) =
-  const callableSymbols = {skProc, skMethod, skConverter, skIterator, skMacro}
+  const callableSymbols = {skProc, skFunc, skMethod, skConverter, skIterator, skMacro}
   if c.selfName != nil and owner.kind in callableSymbols and owner.typ != nil:
     let params = owner.typ.n
     if params.len > 1:
@@ -121,14 +121,14 @@ proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) =
       idTablePut(symMap, s, x)
       n.sym = x
   else:
-    for i in 0 .. <safeLen(n): freshGenSyms(n.sons[i], owner, orig, symMap)
+    for i in 0 ..< safeLen(n): freshGenSyms(n.sons[i], owner, orig, symMap)
 
 proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind)
 
 proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
   if n.sons[bodyPos].kind != nkEmpty:
     let procParams = result.typ.n
-    for i in 1 .. <procParams.len:
+    for i in 1 ..< procParams.len:
       addDecl(c, procParams[i].sym)
     maybeAddResult(c, result, result.ast)
 
@@ -138,7 +138,7 @@ proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
     var symMap: TIdTable
     initIdTable symMap
     if params != nil:
-      for i in 1 .. <params.len:
+      for i in 1 ..< params.len:
         let param = params[i].sym
         if sfGenSym in param.flags:
           idTablePut(symMap, params[i].sym, result.typ.n[param.position+1].sym)
@@ -211,7 +211,7 @@ proc instantiateProcType(c: PContext, pt: TIdTable,
   let originalParams = result.n
   result.n = originalParams.shallowCopy
 
-  for i in 1 .. <result.len:
+  for i in 1 ..< result.len:
     # twrong_field_caching requires these 'resetIdTable' calls:
     if i > 1:
       resetIdTable(cl.symMap)
@@ -240,6 +240,8 @@ proc instantiateProcType(c: PContext, pt: TIdTable,
   resetIdTable(cl.localCache)
   result.sons[0] = replaceTypeVarsT(cl, result.sons[0])
   result.n.sons[0] = originalParams[0].copyTree
+  if result.sons[0] != nil:
+    propagateToOwner(result, result.sons[0])
 
   eraseVoidParams(result)
   skipIntLiteralParams(result)

compiler/semmacrosanity.nim

@@ -42,7 +42,7 @@ proc annotateType*(n: PNode, t: PType) =
   of nkObjConstr:
     let x = t.skipTypes(abstractPtrs)
     n.typ = t
-    for i in 1 .. <n.len:
+    for i in 1 ..< n.len:
       var j = i-1
       let field = x.n.ithField(j)
       if field.isNil:
@@ -53,7 +53,7 @@ proc annotateType*(n: PNode, t: PType) =
   of nkPar:
     if x.kind == tyTuple:
       n.typ = t
-      for i in 0 .. <n.len:
+      for i in 0 ..< n.len:
         if i >= x.len: globalError n.info, "invalid field at index " & $i
         else: annotateType(n.sons[i], x.sons[i])
     elif x.kind == tyProc and x.callConv == ccClosure:

compiler/semmagic.nim

@@ -38,9 +38,7 @@ proc skipAddr(n: PNode): PNode {.inline.} =
 proc semArrGet(c: PContext; n: PNode; flags: TExprFlags): PNode =
   result = newNodeI(nkBracketExpr, n.info)
   for i in 1..<n.len: result.add(n[i])
-  let oldBracketExpr = c.p.bracketExpr
   result = semSubscript(c, result, flags)
-  c.p.bracketExpr = oldBracketExpr
   if result.isNil:
     let x = copyTree(n)
     x.sons[0] = newIdentNode(getIdent"[]", n.info)
@@ -110,7 +108,7 @@ proc uninstantiate(t: PType): PType =
     else: t
 
 proc evalTypeTrait(traitCall: PNode, operand: PType, context: PSym): PNode =
-  const skippedTypes = {tyTypeDesc, tyAlias}
+  const skippedTypes = {tyTypeDesc, tyAlias, tySink}
   let trait = traitCall[0]
   internalAssert trait.kind == nkSym
   var operand = operand.skipTypes(skippedTypes)
@@ -129,7 +127,7 @@ proc evalTypeTrait(traitCall: PNode, operand: PType, context: PSym): PNode =
   of "not":
     return typeWithSonsResult(tyNot, @[operand])
   of "name":
-    result = newStrNode(nkStrLit, operand.typeToString(preferName))
+    result = newStrNode(nkStrLit, operand.typeToString(preferTypeName))
     result.typ = newType(tyString, context)
     result.info = traitCall.info
   of "arity":
@@ -146,8 +144,14 @@ proc evalTypeTrait(traitCall: PNode, operand: PType, context: PSym): PNode =
     result = res.base.toNode(traitCall.info)
   of "stripGenericParams":
     result = uninstantiate(operand).toNode(traitCall.info)
+  of "supportsCopyMem":
+    let t = operand.skipTypes({tyVar, tyLent, tyGenericInst, tyAlias, tySink, tyInferred})
+    let complexObj = containsGarbageCollectedRef(t) or
+                     hasDestructor(t)
+    result = newIntNodeT(ord(not complexObj), traitCall)
   else:
-    internalAssert false
+    localError(traitCall.info, "unknown trait")
+    result = emptyNode
 
 proc semTypeTraits(c: PContext, n: PNode): PNode =
   checkMinSonsLen(n, 2)
@@ -164,7 +168,9 @@ proc semTypeTraits(c: PContext, n: PNode): PNode =
 proc semOrd(c: PContext, n: PNode): PNode =
   result = n
   let parType = n.sons[1].typ
-  if isOrdinalType(parType) or parType.kind == tySet:
+  if isOrdinalType(parType):
+    discard
+  elif parType.kind == tySet:
     result.typ = makeRangeType(c, firstOrd(parType), lastOrd(parType), n.info)
   else:
     localError(n.info, errOrdinalTypeExpected)
@@ -246,7 +252,11 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     result = semTypeOf(c, n.sons[1])
   of mArrGet: result = semArrGet(c, n, flags)
   of mArrPut: result = semArrPut(c, n, flags)
-  of mAsgn: result = semAsgnOpr(c, n)
+  of mAsgn:
+    if n[0].sym.name.s == "=":
+      result = semAsgnOpr(c, n)
+    else:
+      result = n
   of mIsPartOf: result = semIsPartOf(c, n, flags)
   of mTypeTrait: result = semTypeTraits(c, n)
   of mAstToStr:
@@ -264,35 +274,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
   of mDotDot:
     result = n
   of mRoof:
-    let bracketExpr = if n.len == 3: n.sons[2] else: c.p.bracketExpr
-    if bracketExpr.isNil:
-      localError(n.info, "no surrounding array access context for '^'")
-      result = n.sons[1]
-    elif bracketExpr.checkForSideEffects != seNoSideEffect:
-      localError(n.info, "invalid context for '^' as '$#' has side effects" %
-        renderTree(bracketExpr))
-      result = n.sons[1]
-    elif bracketExpr.typ.isStrangeArray:
-      localError(n.info, "invalid context for '^' as len!=high+1 for '$#'" %
-        renderTree(bracketExpr))
-      result = n.sons[1]
-    else:
-      # ^x  is rewritten to: len(a)-x
-      let lenExpr = newNodeI(nkCall, n.info)
-      lenExpr.add newIdentNode(getIdent"len", n.info)
-      lenExpr.add bracketExpr
-      let lenExprB = semExprWithType(c, lenExpr)
-      if lenExprB.typ.isNil or not isOrdinalType(lenExprB.typ):
-        localError(n.info, "'$#' has to be of an ordinal type for '^'" %
-          renderTree(lenExpr))
-        result = n.sons[1]
-      else:
-        result = newNodeIT(nkCall, n.info, getSysType(tyInt))
-        let subi = getSysMagic("-", mSubI)
-        #echo "got ", typeToString(subi.typ)
-        result.add newSymNode(subi, n.info)
-        result.add lenExprB
-        result.add n.sons[1]
+    localError(n.info, "builtin roof operator is not supported anymore")
   of mPlugin:
     let plugin = getPlugin(n[0].sym)
     if plugin.isNil:

compiler/semobjconstr.nim

@@ -39,13 +39,19 @@ proc mergeInitStatus(existing: var InitStatus, newStatus: InitStatus) =
   of initUnknown:
     discard
 
+proc invalidObjConstr(n: PNode) =
+  if n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.s[0] == ':':
+    localError(n.info, "incorrect object construction syntax; use a space after the colon")
+  else:
+    localError(n.info, "incorrect object construction syntax")
+
 proc locateFieldInInitExpr(field: PSym, initExpr: PNode): PNode =
   # Returns the assignment nkExprColonExpr node or nil
   let fieldId = field.name.id
-  for i in 1 .. <initExpr.len:
+  for i in 1 ..< initExpr.len:
     let assignment = initExpr[i]
     if assignment.kind != nkExprColonExpr:
-      localError(initExpr.info, "incorrect object construction syntax")
+      invalidObjConstr(assignment)
       continue
 
     if fieldId == considerQuotedIdent(assignment[0]).id:
@@ -78,13 +84,13 @@ proc caseBranchMatchesExpr(branch, matched: PNode): bool =
 
 proc pickCaseBranch(caseExpr, matched: PNode): PNode =
   # XXX: Perhaps this proc already exists somewhere
-  let endsWithElse = caseExpr{-1}.kind == nkElse
+  let endsWithElse = caseExpr[^1].kind == nkElse
   for i in 1 .. caseExpr.len - 1 - int(endsWithElse):
     if caseExpr[i].caseBranchMatchesExpr(matched):
       return caseExpr[i]
 
   if endsWithElse:
-    return caseExpr{-1}
+    return caseExpr[^1]
 
 iterator directFieldsInRecList(recList: PNode): PNode =
   # XXX: We can remove this case by making all nkOfBranch nodes
@@ -136,17 +142,20 @@ proc semConstructFields(c: PContext, recNode: PNode,
 
   of nkRecCase:
     template fieldsPresentInBranch(branchIdx: int): string =
-      fieldsPresentInInitExpr(recNode[branchIdx]{-1}, initExpr)
+      let branch = recNode[branchIdx]
+      let fields = branch[branch.len - 1]
+      fieldsPresentInInitExpr(fields, initExpr)
 
     template checkMissingFields(branchNode: PNode) =
-      checkForMissingFields(branchNode{-1}, initExpr)
+      let fields = branchNode[branchNode.len - 1]
+      checkForMissingFields(fields, initExpr)
 
     let discriminator = recNode.sons[0];
     internalAssert discriminator.kind == nkSym
     var selectedBranch = -1
 
-    for i in 1 .. <recNode.len:
-      let innerRecords = recNode[i]{-1}
+    for i in 1 ..< recNode.len:
+      let innerRecords = recNode[i][^1]
       let status = semConstructFields(c, innerRecords, initExpr, flags)
       if status notin {initNone, initUnknown}:
         mergeInitStatus(result, status)
@@ -220,7 +229,7 @@ proc semConstructFields(c: PContext, recNode: PNode,
         else:
           # All bets are off. If any of the branches has a mandatory
           # fields we must produce an error:
-          for i in 1 .. <recNode.len: checkMissingFields recNode[i]
+          for i in 1 ..< recNode.len: checkMissingFields recNode[i]
 
   of nkSym:
     let field = recNode.sym
@@ -250,9 +259,13 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
   var t = semTypeNode(c, n.sons[0], nil)
   result = newNodeIT(nkObjConstr, n.info, t)
   for child in n: result.add child
-  
-  t = skipTypes(t, {tyGenericInst, tyAlias})
-  if t.kind == tyRef: t = skipTypes(t.sons[0], {tyGenericInst, tyAlias})
+
+  if t == nil:
+    localError(n.info, errGenerated, "object constructor needs an object type")
+    return
+
+  t = skipTypes(t, {tyGenericInst, tyAlias, tySink})
+  if t.kind == tyRef: t = skipTypes(t.sons[0], {tyGenericInst, tyAlias, tySink})
   if t.kind != tyObject:
     localError(n.info, errGenerated, "object constructor needs an object type")
     return
@@ -277,16 +290,16 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
   # Since we were traversing the object fields, it's possible that
   # not all of the fields specified in the constructor was visited.
   # We'll check for such fields here:
-  for i in 1.. <result.len:
+  for i in 1..<result.len:
     let field = result[i]
     if nfSem notin field.flags:
       if field.kind != nkExprColonExpr:
-        localError(n.info, "incorrect object construction syntax")
+        invalidObjConstr(field)
         continue
       let id = considerQuotedIdent(field[0])
       # This node was not processed. There are two possible reasons:
       # 1) It was shadowed by a field with the same name on the left
-      for j in 1 .. <i:
+      for j in 1 ..< i:
         let prevId = considerQuotedIdent(result[j][0])
         if prevId.id == id.id:
           localError(field.info, errFieldInitTwice, id.s)

compiler/semparallel.nim

@@ -81,7 +81,7 @@ proc initAnalysisCtx(): AnalysisCtx =
   result.guards = @[]
 
 proc lookupSlot(c: AnalysisCtx; s: PSym): int =
-  for i in 0.. <c.locals.len:
+  for i in 0..<c.locals.len:
     if c.locals[i].v == s or c.locals[i].alias == s: return i
   return -1
 
@@ -94,7 +94,7 @@ proc getSlot(c: var AnalysisCtx; v: PSym): ptr MonotonicVar =
   return addr(c.locals[L])
 
 proc gatherArgs(c: var AnalysisCtx; n: PNode) =
-  for i in 0.. <n.safeLen:
+  for i in 0..<n.safeLen:
     let root = getRoot n[i]
     if root != nil:
       block addRoot:
@@ -119,7 +119,7 @@ proc checkLocal(c: AnalysisCtx; n: PNode) =
     if s >= 0 and c.locals[s].stride != nil:
       localError(n.info, "invalid usage of counter after increment")
   else:
-    for i in 0 .. <n.safeLen: checkLocal(c, n.sons[i])
+    for i in 0 ..< n.safeLen: checkLocal(c, n.sons[i])
 
 template `?`(x): untyped = x.renderTree
 
@@ -180,7 +180,7 @@ proc stride(c: AnalysisCtx; n: PNode): BiggestInt =
     if s >= 0 and c.locals[s].stride != nil:
       result = c.locals[s].stride.intVal
   else:
-    for i in 0 .. <n.safeLen: result += stride(c, n.sons[i])
+    for i in 0 ..< n.safeLen: result += stride(c, n.sons[i])
 
 proc subStride(c: AnalysisCtx; n: PNode): PNode =
   # substitute with stride:
@@ -192,7 +192,7 @@ proc subStride(c: AnalysisCtx; n: PNode): PNode =
       result = n
   elif n.safeLen > 0:
     result = shallowCopy(n)
-    for i in 0 .. <n.len: result.sons[i] = subStride(c, n.sons[i])
+    for i in 0 ..< n.len: result.sons[i] = subStride(c, n.sons[i])
   else:
     result = n
 
@@ -251,7 +251,7 @@ proc checkSlicesAreDisjoint(c: var AnalysisCtx) =
 proc analyse(c: var AnalysisCtx; n: PNode)
 
 proc analyseSons(c: var AnalysisCtx; n: PNode) =
-  for i in 0 .. <safeLen(n): analyse(c, n[i])
+  for i in 0 ..< safeLen(n): analyse(c, n[i])
 
 proc min(a, b: PNode): PNode =
   if a.isNil: result = b
@@ -293,11 +293,11 @@ proc analyseCall(c: var AnalysisCtx; n: PNode; op: PSym) =
 proc analyseCase(c: var AnalysisCtx; n: PNode) =
   analyse(c, n.sons[0])
   let oldFacts = c.guards.len
-  for i in 1.. <n.len:
+  for i in 1..<n.len:
     let branch = n.sons[i]
     setLen(c.guards, oldFacts)
     addCaseBranchFacts(c.guards, n, i)
-    for i in 0 .. <branch.len:
+    for i in 0 ..< branch.len:
       analyse(c, branch.sons[i])
   setLen(c.guards, oldFacts)
 
@@ -307,14 +307,14 @@ proc analyseIf(c: var AnalysisCtx; n: PNode) =
   addFact(c.guards, canon(n.sons[0].sons[0]))
 
   analyse(c, n.sons[0].sons[1])
-  for i in 1.. <n.len:
+  for i in 1..<n.len:
     let branch = n.sons[i]
     setLen(c.guards, oldFacts)
     for j in 0..i-1:
       addFactNeg(c.guards, canon(n.sons[j].sons[0]))
     if branch.len > 1:
       addFact(c.guards, canon(branch.sons[0]))
-    for i in 0 .. <branch.len:
+    for i in 0 ..< branch.len:
       analyse(c, branch.sons[i])
   setLen(c.guards, oldFacts)
 
@@ -387,7 +387,7 @@ proc analyse(c: var AnalysisCtx; n: PNode) =
         addFactNeg(c.guards, canon(n.sons[0]))
     dec c.inLoop
   of nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef,
-      nkMacroDef, nkTemplateDef, nkConstSection, nkPragma:
+      nkMacroDef, nkTemplateDef, nkConstSection, nkPragma, nkFuncDef:
     discard
   else:
     analyseSons(c, n)
@@ -407,7 +407,7 @@ proc transformSlices(n: PNode): PNode =
       return result
   if n.safeLen > 0:
     result = shallowCopy(n)
-    for i in 0 .. < n.len:
+    for i in 0 ..< n.len:
       result.sons[i] = transformSlices(n.sons[i])
   else:
     result = n
@@ -415,7 +415,7 @@ proc transformSlices(n: PNode): PNode =
 proc transformSpawn(owner: PSym; n, barrier: PNode): PNode
 proc transformSpawnSons(owner: PSym; n, barrier: PNode): PNode =
   result = shallowCopy(n)
-  for i in 0 .. < n.len:
+  for i in 0 ..< n.len:
     result.sons[i] = transformSpawn(owner, n.sons[i], barrier)
 
 proc transformSpawn(owner: PSym; n, barrier: PNode): PNode =

compiler/sempass2.nim

@@ -11,23 +11,14 @@ import
   intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
   wordrecg, strutils, options, guards, writetracking
 
+when defined(useDfa):
+  import dfa
+
 # Second semantic checking pass over the AST. Necessary because the old
 # way had some inherent problems. Performs:
 #
 # * effect+exception tracking
 # * "usage before definition" checking
-# * checks for invalid usages of compiletime magics (not implemented)
-# * checks for invalid usages of NimNode (not implemented)
-# * later: will do an escape analysis for closures at least
-
-# Predefined effects:
-#   io, time (time dependent), gc (performs GC'ed allocation), exceptions,
-#   side effect (accesses global), store (stores into *type*),
-#   store_unknown (performs some store) --> store(any)|store(x)
-#   load (loads from *type*), recursive (recursive call), unsafe,
-#   endless (has endless loops), --> user effects are defined over *patterns*
-#   --> a TR macro can annotate the proc with user defined annotations
-#   --> the effect system can access these
 
 # ------------------------ exception and tag tracking -------------------------
 
@@ -248,6 +239,7 @@ proc useVar(a: PEffects, n: PNode) =
         (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem):
       #if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n)
       markGcUnsafe(a, s)
+      markSideEffect(a, s)
     else:
       markSideEffect(a, s)
 
@@ -256,7 +248,7 @@ type
   TIntersection = seq[tuple[id, count: int]] # a simple count table
 
 proc addToIntersection(inter: var TIntersection, s: int) =
-  for j in 0.. <inter.len:
+  for j in 0..<inter.len:
     if s == inter[j].id:
       inc inter[j].count
       return
@@ -290,7 +282,7 @@ proc createTag(n: PNode): PNode =
 proc addEffect(a: PEffects, e: PNode, useLineInfo=true) =
   assert e.kind != nkRaiseStmt
   var aa = a.exc
-  for i in a.bottom .. <aa.len:
+  for i in a.bottom ..< aa.len:
     if sameType(aa[i].excType, e.excType):
       if not useLineInfo or gCmd == cmdDoc: return
       elif aa[i].info == e.info: return
@@ -298,7 +290,7 @@ proc addEffect(a: PEffects, e: PNode, useLineInfo=true) =
 
 proc addTag(a: PEffects, e: PNode, useLineInfo=true) =
   var aa = a.tags
-  for i in 0 .. <aa.len:
+  for i in 0 ..< aa.len:
     if sameType(aa[i].typ.skipTypes(skipPtrs), e.typ.skipTypes(skipPtrs)):
       if not useLineInfo or gCmd == cmdDoc: return
       elif aa[i].info == e.info: return
@@ -353,12 +345,12 @@ proc trackTryStmt(tracked: PEffects, n: PNode) =
   inc tracked.inTryStmt
   track(tracked, n.sons[0])
   dec tracked.inTryStmt
-  for i in oldState.. <tracked.init.len:
+  for i in oldState..<tracked.init.len:
     addToIntersection(inter, tracked.init[i])
 
   var branches = 1
   var hasFinally = false
-  for i in 1 .. < n.len:
+  for i in 1 ..< n.len:
     let b = n.sons[i]
     let blen = sonsLen(b)
     if b.kind == nkExceptBranch:
@@ -372,7 +364,7 @@ proc trackTryStmt(tracked: PEffects, n: PNode) =
 
       setLen(tracked.init, oldState)
       track(tracked, b.sons[blen-1])
-      for i in oldState.. <tracked.init.len:
+      for i in oldState..<tracked.init.len:
         addToIntersection(inter, tracked.init[i])
     else:
       assert b.kind == nkFinally
@@ -428,7 +420,7 @@ proc documentEffect(n, x: PNode, effectType: TSpecialWord, idx: int): PNode =
 
     # warning: hack ahead:
     var effects = newNodeI(nkBracket, n.info, real.len)
-    for i in 0 .. <real.len:
+    for i in 0 ..< real.len:
       var t = typeToString(real[i].typ)
       if t.startsWith("ref "): t = substr(t, 4)
       effects.sons[i] = newIdentNode(getIdent(t), n.info)
@@ -526,13 +518,15 @@ proc notNilCheck(tracked: PEffects, n: PNode, paramType: PType) =
     procVarcheck skipConvAndClosure(n)
   #elif n.kind in nkSymChoices:
   #  echo "came here"
+  let paramType = paramType.skipTypesOrNil(abstractInst)
   if paramType != nil and tfNotNil in paramType.flags and
       n.typ != nil and tfNotNil notin n.typ.flags:
     if n.kind == nkAddr:
       # addr(x[]) can't be proven, but addr(x) can:
       if not containsNode(n, {nkDerefExpr, nkHiddenDeref}): return
     elif (n.kind == nkSym and n.sym.kind in routineKinds) or
-         n.kind in procDefs+{nkObjConstr, nkBracket}:
+         (n.kind in procDefs+{nkObjConstr, nkBracket, nkClosure, nkStrLit..nkTripleStrLit}) or
+         (n.kind in nkCallKinds and n[0].kind == nkSym and n[0].sym.magic == mArrToSeq):
       # 'p' is not nil obviously:
       return
     case impliesNotNil(tracked.guards, n)
@@ -590,20 +584,23 @@ proc trackOperand(tracked: PEffects, n: PNode, paramType: PType) =
   if paramType != nil and paramType.kind == tyVar:
     if n.kind == nkSym and isLocalVar(tracked, n.sym):
       makeVolatile(tracked, n.sym)
+  if paramType != nil and paramType.kind == tyProc and tfGcSafe in paramType.flags:
+    let argtype = skipTypes(a.typ, abstractInst)
+    # XXX figure out why this can be a non tyProc here. See httpclient.nim for an
+    # example that triggers it.
+    if argtype.kind == tyProc and notGcSafe(argtype) and not tracked.inEnforcedGcSafe:
+      localError(n.info, $n & " is not GC safe")
   notNilCheck(tracked, n, paramType)
 
 proc breaksBlock(n: PNode): bool =
-  case n.kind
-  of nkStmtList, nkStmtListExpr:
-    for c in n:
-      if breaksBlock(c): return true
-  of nkBreakStmt, nkReturnStmt, nkRaiseStmt:
-    return true
-  of nkCallKinds:
-    if n.sons[0].kind == nkSym and sfNoReturn in n.sons[0].sym.flags:
-      return true
-  else:
-    discard
+  # sematic check doesn't allow statements after raise, break, return or
+  # call to noreturn proc, so it is safe to check just the last statements
+  var it = n
+  while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0:
+    it = it.lastSon
+
+  result = it.kind in {nkBreakStmt, nkReturnStmt, nkRaiseStmt} or
+    it.kind in nkCallKinds and it[0].kind == nkSym and sfNoReturn in it[0].sym.flags
 
 proc trackCase(tracked: PEffects, n: PNode) =
   track(tracked, n.sons[0])
@@ -615,16 +612,16 @@ proc trackCase(tracked: PEffects, n: PNode) =
         warnProveField in gNotes
   var inter: TIntersection = @[]
   var toCover = 0
-  for i in 1.. <n.len:
+  for i in 1..<n.len:
     let branch = n.sons[i]
     setLen(tracked.init, oldState)
     if interesting:
       setLen(tracked.guards, oldFacts)
       addCaseBranchFacts(tracked.guards, n, i)
-    for i in 0 .. <branch.len:
+    for i in 0 ..< branch.len:
       track(tracked, branch.sons[i])
     if not breaksBlock(branch.lastSon): inc toCover
-    for i in oldState.. <tracked.init.len:
+    for i in oldState..<tracked.init.len:
       addToIntersection(inter, tracked.init[i])
 
   setLen(tracked.init, oldState)
@@ -644,10 +641,10 @@ proc trackIf(tracked: PEffects, n: PNode) =
   var toCover = 0
   track(tracked, n.sons[0].sons[1])
   if not breaksBlock(n.sons[0].sons[1]): inc toCover
-  for i in oldState.. <tracked.init.len:
+  for i in oldState..<tracked.init.len:
     addToIntersection(inter, tracked.init[i])
 
-  for i in 1.. <n.len:
+  for i in 1..<n.len:
     let branch = n.sons[i]
     setLen(tracked.guards, oldFacts)
     for j in 0..i-1:
@@ -655,10 +652,10 @@ proc trackIf(tracked: PEffects, n: PNode) =
     if branch.len > 1:
       addFact(tracked.guards, branch.sons[0])
     setLen(tracked.init, oldState)
-    for i in 0 .. <branch.len:
+    for i in 0 ..< branch.len:
       track(tracked, branch.sons[i])
     if not breaksBlock(branch.lastSon): inc toCover
-    for i in oldState.. <tracked.init.len:
+    for i in oldState..<tracked.init.len:
       addToIntersection(inter, tracked.init[i])
   setLen(tracked.init, oldState)
   if lastSon(n).len == 1:
@@ -670,7 +667,7 @@ proc trackIf(tracked: PEffects, n: PNode) =
 proc trackBlock(tracked: PEffects, n: PNode) =
   if n.kind in {nkStmtList, nkStmtListExpr}:
     var oldState = -1
-    for i in 0.. <n.len:
+    for i in 0..<n.len:
       if hasSubnodeWith(n.sons[i], nkBreakStmt):
         # block:
         #   x = def
@@ -703,7 +700,7 @@ proc track(tracked: PEffects, n: PNode) =
     n.sons[0].info = n.info
     #throws(tracked.exc, n.sons[0])
     addEffect(tracked, n.sons[0], useLineInfo=false)
-    for i in 0 .. <safeLen(n):
+    for i in 0 ..< safeLen(n):
       track(tracked, n.sons[i])
   of nkCallKinds:
     # p's effects are ours too:
@@ -742,23 +739,23 @@ proc track(tracked: PEffects, n: PNode) =
           if not (a.kind == nkSym and a.sym == tracked.owner):
             markSideEffect(tracked, a)
     if a.kind != nkSym or a.sym.magic != mNBindSym:
-      for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i))
+      for i in 1 ..< len(n): trackOperand(tracked, n.sons[i], paramType(op, i))
     if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, mNewSeq}:
       # may not look like an assignment, but it is:
       let arg = n.sons[1]
       initVarViaNew(tracked, arg)
-      if {tfNeedsInit} * arg.typ.lastSon.flags != {}:
+      if arg.typ.len != 0 and {tfNeedsInit} * arg.typ.lastSon.flags != {}:
         if a.sym.magic == mNewSeq and n[2].kind in {nkCharLit..nkUInt64Lit} and
             n[2].intVal == 0:
           # var s: seq[notnil];  newSeq(s, 0)  is a special case!
           discard
         else:
           message(arg.info, warnProveInit, $arg)
-    for i in 0 .. <safeLen(n):
+    for i in 0 ..< safeLen(n):
       track(tracked, n.sons[i])
   of nkDotExpr:
     guardDotAccess(tracked, n)
-    for i in 0 .. <len(n): track(tracked, n.sons[i])
+    for i in 0 ..< len(n): track(tracked, n.sons[i])
   of nkCheckedFieldExpr:
     track(tracked, n.sons[0])
     if warnProveField in gNotes: checkFieldAccess(tracked.guards, n)
@@ -806,13 +803,13 @@ proc track(tracked: PEffects, n: PNode) =
   of nkForStmt, nkParForStmt:
     # we are very conservative here and assume the loop is never executed:
     let oldState = tracked.init.len
-    for i in 0 .. <len(n):
+    for i in 0 ..< len(n):
       track(tracked, n.sons[i])
     setLen(tracked.init, oldState)
   of nkObjConstr:
     when false: track(tracked, n.sons[0])
     let oldFacts = tracked.guards.len
-    for i in 1 .. <len(n):
+    for i in 1 ..< len(n):
       let x = n.sons[i]
       track(tracked, x)
       if x.sons[0].kind == nkSym and sfDiscriminant in x.sons[0].sym.flags:
@@ -823,7 +820,7 @@ proc track(tracked: PEffects, n: PNode) =
     let oldLocked = tracked.locked.len
     let oldLockLevel = tracked.currLockLevel
     var enforcedGcSafety = false
-    for i in 0 .. <pragmaList.len:
+    for i in 0 ..< pragmaList.len:
       let pragma = whichPragma(pragmaList.sons[i])
       if pragma == wLocks:
         lockLocations(tracked, pragmaList.sons[i])
@@ -835,14 +832,14 @@ proc track(tracked: PEffects, n: PNode) =
     setLen(tracked.locked, oldLocked)
     tracked.currLockLevel = oldLockLevel
   of nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef,
-      nkMacroDef, nkTemplateDef, nkLambda, nkDo:
+      nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef:
     discard
   of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv:
     if n.len == 2: track(tracked, n.sons[1])
   of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64:
     if n.len == 1: track(tracked, n.sons[0])
   else:
-    for i in 0 .. <safeLen(n): track(tracked, n.sons[i])
+    for i in 0 ..< safeLen(n): track(tracked, n.sons[i])
 
 proc subtypeRelation(spec, real: PNode): bool =
   result = safeInheritanceDiff(real.excType, spec.typ) <= 0
@@ -854,7 +851,7 @@ proc checkRaisesSpec(spec, real: PNode, msg: string, hints: bool;
   var used = initIntSet()
   for r in items(real):
     block search:
-      for s in 0 .. <spec.len:
+      for s in 0 ..< spec.len:
         if effectPredicate(spec[s], r):
           used.incl(s)
           break search
@@ -864,7 +861,7 @@ proc checkRaisesSpec(spec, real: PNode, msg: string, hints: bool;
       popInfoContext()
   # hint about unnecessarily listed exception types:
   if hints:
-    for s in 0 .. <spec.len:
+    for s in 0 ..< spec.len:
       if not used.contains(s):
         message(spec[s].info, hintXDeclaredButNotUsed, renderTree(spec[s]))
 
@@ -937,7 +934,7 @@ proc trackProc*(s: PSym, body: PNode) =
   track(t, body)
   if not isEmptyType(s.typ.sons[0]) and
       {tfNeedsInit, tfNotNil} * s.typ.sons[0].flags != {} and
-      s.kind in {skProc, skConverter, skMethod}:
+      s.kind in {skProc, skFunc, skConverter, skMethod}:
     var res = s.ast.sons[resultPos].sym # get result symbol
     if res.id notin t.init:
       message(body.info, warnProveInit, "result")
@@ -979,10 +976,13 @@ proc trackProc*(s: PSym, body: PNode) =
     message(s.info, warnLockLevel,
       "declared lock level is $1, but real lock level is $2" %
         [$s.typ.lockLevel, $t.maxLockLevel])
-  when useWriteTracking: trackWrites(s, body)
+  when defined(useDfa):
+    if s.kind == skFunc:
+      dataflowAnalysis(s, body)
+      when false: trackWrites(s, body)
 
 proc trackTopLevelStmt*(module: PSym; n: PNode) =
-  if n.kind in {nkPragma, nkMacroDef, nkTemplateDef, nkProcDef,
+  if n.kind in {nkPragma, nkMacroDef, nkTemplateDef, nkProcDef, nkFuncDef,
                 nkTypeSection, nkConverterDef, nkMethodDef, nkIteratorDef}:
     return
   var effects = newNode(nkEffectList, n.info)

compiler/semstmts.nim

@@ -17,7 +17,7 @@ proc semDiscard(c: PContext, n: PNode): PNode =
   checkSonsLen(n, 1)
   if n.sons[0].kind != nkEmpty:
     n.sons[0] = semExprWithType(c, n.sons[0])
-    if isEmptyType(n.sons[0].typ) or n.sons[0].typ.kind == tyNone:
+    if isEmptyType(n.sons[0].typ) or n.sons[0].typ.kind == tyNone or n.sons[0].kind == nkTypeOfExpr:
       localError(n.info, errInvalidDiscard)
 
 proc semBreakOrContinue(c: PContext, n: PNode): PNode =
@@ -97,26 +97,12 @@ template semProcvarCheck(c: PContext, n: PNode) =
 
 proc semProc(c: PContext, n: PNode): PNode
 
-include semdestruct
-
-proc semDestructorCheck(c: PContext, n: PNode, flags: TExprFlags) {.inline.} =
-  if efAllowDestructor notin flags and
-      n.kind in nkCallKinds+{nkObjConstr,nkBracket}:
-    if instantiateDestructor(c, n.typ) != nil:
-      localError(n.info, warnDestructor)
-  # This still breaks too many things:
-  when false:
-    if efDetermineType notin flags and n.typ.kind == tyTypeDesc and
-        c.p.owner.kind notin {skTemplate, skMacro}:
-      localError(n.info, errGenerated, "value expected, but got a type")
-
 proc semExprBranch(c: PContext, n: PNode): PNode =
   result = semExpr(c, n)
   if result.typ != nil:
     # XXX tyGenericInst here?
     semProcvarCheck(c, result)
-    if result.typ.kind == tyVar: result = newDeref(result)
-    semDestructorCheck(c, result, {})
+    if result.typ.kind in {tyVar, tyLent}: result = newDeref(result)
 
 proc semExprBranchScope(c: PContext, n: PNode): PNode =
   openScope(c)
@@ -164,9 +150,12 @@ proc discardCheck(c: PContext, result: PNode) =
         while n.kind in skipForDiscardable: n = n.lastSon
         var s = "expression '" & $n & "' is of type '" &
             result.typ.typeToString & "' and has to be discarded"
+        if result.info.line != n.info.line or
+           result.info.fileIndex != n.info.fileIndex:
+          s.add "; start of expression here: " & $result.info
         if result.typ.kind == tyProc:
           s.add "; for a function call use ()"
-        localError(n.info,  s)
+        localError(n.info, s)
 
 proc semIf(c: PContext, n: PNode): PNode =
   result = n
@@ -179,14 +168,14 @@ proc semIf(c: PContext, n: PNode): PNode =
       it.sons[0] = forceBool(c, semExprWithType(c, it.sons[0]))
       when not newScopeForIf: openScope(c)
       it.sons[1] = semExprBranch(c, it.sons[1])
-      typ = commonType(typ, it.sons[1].typ)
+      typ = commonType(typ, it.sons[1])
       closeScope(c)
     elif it.len == 1:
       hasElse = true
       it.sons[0] = semExprBranchScope(c, it.sons[0])
-      typ = commonType(typ, it.sons[0].typ)
+      typ = commonType(typ, it.sons[0])
     else: illFormedAst(it)
-  if isEmptyType(typ) or typ.kind == tyNil or not hasElse:
+  if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or not hasElse:
     for it in n: discardCheck(c, it.lastSon)
     result.kind = nkIfStmt
     # propagate any enforced VoidContext:
@@ -194,7 +183,8 @@ proc semIf(c: PContext, n: PNode): PNode =
   else:
     for it in n:
       let j = it.len-1
-      it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info)
+      if not endsInNoReturn(it.sons[j]):
+        it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info)
     result.kind = nkIfExpr
     result.typ = typ
 
@@ -227,7 +217,7 @@ proc semCase(c: PContext, n: PNode): PNode =
       semCaseBranch(c, n, x, i, covered)
       var last = sonsLen(x)-1
       x.sons[last] = semExprBranchScope(c, x.sons[last])
-      typ = commonType(typ, x.sons[last].typ)
+      typ = commonType(typ, x.sons[last])
     of nkElifBranch:
       chckCovered = false
       checkSonsLen(x, 2)
@@ -235,13 +225,13 @@ proc semCase(c: PContext, n: PNode): PNode =
       x.sons[0] = forceBool(c, semExprWithType(c, x.sons[0]))
       when not newScopeForIf: openScope(c)
       x.sons[1] = semExprBranch(c, x.sons[1])
-      typ = commonType(typ, x.sons[1].typ)
+      typ = commonType(typ, x.sons[1])
       closeScope(c)
     of nkElse:
       chckCovered = false
       checkSonsLen(x, 1)
       x.sons[0] = semExprBranchScope(c, x.sons[0])
-      typ = commonType(typ, x.sons[0].typ)
+      typ = commonType(typ, x.sons[0])
       hasElse = true
     else:
       illFormedAst(x)
@@ -251,7 +241,7 @@ proc semCase(c: PContext, n: PNode): PNode =
     else:
       localError(n.info, errNotAllCasesCovered)
   closeScope(c)
-  if isEmptyType(typ) or typ.kind == tyNil or not hasElse:
+  if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or not hasElse:
     for i in 1..n.len-1: discardCheck(c, n.sons[i].lastSon)
     # propagate any enforced VoidContext:
     if typ == enforceVoidContext:
@@ -260,7 +250,8 @@ proc semCase(c: PContext, n: PNode): PNode =
     for i in 1..n.len-1:
       var it = n.sons[i]
       let j = it.len-1
-      it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info)
+      if not endsInNoReturn(it.sons[j]):
+        it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info)
     result.typ = typ
 
 proc semTry(c: PContext, n: PNode): PNode =
@@ -384,7 +375,7 @@ proc checkNilable(v: PSym) =
       {tfNotNil, tfNeedsInit} * v.typ.flags != {}:
     if v.ast.isNil:
       message(v.info, warnProveInit, v.name.s)
-    elif tfNeedsInit in v.typ.flags and tfNotNil notin v.ast.typ.flags:
+    elif tfNotNil in v.typ.flags and tfNotNil notin v.ast.typ.flags:
       message(v.info, warnProveInit, v.name.s)
 
 include semasgn
@@ -399,7 +390,7 @@ proc addToVarSection(c: PContext; result: var PNode; orig, identDefs: PNode) =
   # in order for this transformation to be correct.
   let L = identDefs.len
   let value = identDefs[L-1]
-  if value.typ != nil and tfHasAsgn in value.typ.flags:
+  if value.typ != nil and tfHasAsgn in value.typ.flags and not newDestructors:
     # the spec says we need to rewrite 'var x = T()' to 'var x: T; x = T()':
     identDefs.sons[L-1] = emptyNode
     if result.kind != nkStmtList:
@@ -420,15 +411,6 @@ proc addToVarSection(c: PContext; result: var PNode; orig, identDefs: PNode) =
   else:
     result.add identDefs
 
-proc addDefer(c: PContext; result: var PNode; s: PSym) =
-  let deferDestructorCall = createDestructorCall(c, s)
-  if deferDestructorCall != nil:
-    if result.kind != nkStmtList:
-      let oldResult = result
-      result = newNodeI(nkStmtList, result.info)
-      result.add oldResult
-    result.add deferDestructorCall
-
 proc isDiscardUnderscore(v: PSym): bool =
   if v.name.s == "_":
     v.flags.incl(sfGenSym)
@@ -437,8 +419,6 @@ proc isDiscardUnderscore(v: PSym): bool =
 proc semUsing(c: PContext; n: PNode): PNode =
   result = ast.emptyNode
   if not isTopLevel(c): localError(n.info, errXOnlyAtModuleScope, "using")
-  if not experimentalMode(c):
-    localError(n.info, "use the {.experimental.} pragma to enable 'using'")
   for i in countup(0, sonsLen(n)-1):
     var a = n.sons[i]
     if gCmd == cmdIdeTools: suggestStmt(c, a)
@@ -466,20 +446,21 @@ proc hasEmpty(typ: PType): bool =
       result = result or hasEmpty(s)
 
 proc makeDeref(n: PNode): PNode =
-  var t = skipTypes(n.typ, {tyGenericInst, tyAlias})
+  var t = n.typ
   if t.kind in tyUserTypeClasses and t.isResolvedUserTypeClass:
     t = t.lastSon
+  t = skipTypes(t, {tyGenericInst, tyAlias, tySink})
   result = n
-  if t.kind == tyVar:
+  if t.kind in {tyVar, tyLent}:
     result = newNodeIT(nkHiddenDeref, n.info, t.sons[0])
     addSon(result, n)
-    t = skipTypes(t.sons[0], {tyGenericInst, tyAlias})
+    t = skipTypes(t.sons[0], {tyGenericInst, tyAlias, tySink})
   while t.kind in {tyPtr, tyRef}:
     var a = result
     let baseTyp = t.lastSon
     result = newNodeIT(nkHiddenDeref, n.info, baseTyp)
     addSon(result, a)
-    t = skipTypes(baseTyp, {tyGenericInst, tyAlias})
+    t = skipTypes(baseTyp, {tyGenericInst, tyAlias, tySink})
 
 proc fillPartialObject(c: PContext; n: PNode; typ: PType) =
   if n.len == 2:
@@ -493,6 +474,7 @@ proc fillPartialObject(c: PContext; n: PNode; typ: PType) =
       addSon(obj.n, newSymNode(field))
       n.sons[0] = makeDeref x
       n.sons[1] = newSymNode(field)
+      n.typ = field.typ
     else:
       localError(n.info, "implicit object field construction " &
         "requires a .partial object, but got " & typeToString(obj))
@@ -552,8 +534,9 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
 
     # this can only happen for errornous var statements:
     if typ == nil: continue
-    typeAllowedCheck(a.info, typ, symkind)
-    var tup = skipTypes(typ, {tyGenericInst, tyAlias})
+    typeAllowedCheck(a.info, typ, symkind, if c.matchedConcept != nil: {taConcept} else: {})
+    liftTypeBoundOps(c, typ, a.info)
+    var tup = skipTypes(typ, {tyGenericInst, tyAlias, tySink})
     if a.kind == nkVarTuple:
       if tup.kind != tyTuple:
         localError(a.info, errXExpected, "tuple")
@@ -608,7 +591,6 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         if def.kind == nkPar: v.ast = def[j]
         setVarType(v, tup.sons[j])
         b.sons[j] = newSymNode(v)
-      addDefer(c, result, v)
       checkNilable(v)
       if sfCompileTime in v.flags: hasCompileTime = true
   if hasCompileTime: vm.setupCompileTimeVar(c.module, c.cache, result)
@@ -670,7 +652,7 @@ proc semForVars(c: PContext, n: PNode): PNode =
   result = n
   var length = sonsLen(n)
   let iterBase = n.sons[length-2].typ
-  var iter = skipTypes(iterBase, {tyGenericInst, tyAlias})
+  var iter = skipTypes(iterBase, {tyGenericInst, tyAlias, tySink})
   # length == 3 means that there is one for loop variable
   # and thus no tuple unpacking:
   if iter.kind != tyTuple or length == 3:
@@ -696,18 +678,26 @@ proc semForVars(c: PContext, n: PNode): PNode =
       if sfGenSym notin v.flags and not isDiscardUnderscore(v):
         addForVarDecl(c, v)
   inc(c.p.nestedLoopCounter)
+  openScope(c)
   n.sons[length-1] = semStmt(c, n.sons[length-1])
+  closeScope(c)
   dec(c.p.nestedLoopCounter)
 
 proc implicitIterator(c: PContext, it: string, arg: PNode): PNode =
   result = newNodeI(nkCall, arg.info)
   result.add(newIdentNode(it.getIdent, arg.info))
-  if arg.typ != nil and arg.typ.kind == tyVar:
+  if arg.typ != nil and arg.typ.kind in {tyVar, tyLent}:
     result.add newDeref(arg)
   else:
     result.add arg
   result = semExprNoDeref(c, result, {efWantIterator})
 
+proc isTrivalStmtExpr(n: PNode): bool =
+  for i in 0 .. n.len-2:
+    if n[i].kind notin {nkEmpty, nkCommentStmt}:
+      return false
+  result = true
+
 proc semFor(c: PContext, n: PNode): PNode =
   result = n
   checkMinSonsLen(n, 3)
@@ -715,6 +705,9 @@ proc semFor(c: PContext, n: PNode): PNode =
   openScope(c)
   n.sons[length-2] = semExprNoDeref(c, n.sons[length-2], {efWantIterator})
   var call = n.sons[length-2]
+  if call.kind == nkStmtListExpr and isTrivalStmtExpr(call):
+    call = call.lastSon
+    n.sons[length-2] = call
   let isCallExpr = call.kind in nkCallKinds
   if isCallExpr and call[0].kind == nkSym and
       call[0].sym.magic in {mFields, mFieldPairs, mOmpParFor}:
@@ -752,6 +745,12 @@ proc semRaise(c: PContext, n: PNode): PNode =
     if typ.kind != tyRef or typ.lastSon.kind != tyObject:
       localError(n.info, errExprCannotBeRaised)
 
+    # check if the given object inherits from Exception
+    if not typ.lastSon.isException():
+        localError(n.info, "raised object of type $1 does not inherit from Exception",
+                           [typeToString(typ)])
+
+
 proc addGenericParamListToScope(c: PContext, n: PNode) =
   if n.kind != nkGenericParams: illFormedAst(n)
   for i in countup(0, sonsLen(n)-1):
@@ -774,24 +773,55 @@ proc typeSectionLeftSidePass(c: PContext, n: PNode) =
     checkSonsLen(a, 3)
     let name = a.sons[0]
     var s: PSym
-    if name.kind == nkDotExpr:
-      s = qualifiedLookUp(c, name, {checkUndeclared, checkModule})
-      if s.kind != skType or
-         s.typ.skipTypes(abstractPtrs).kind != tyObject or
-         tfPartial notin s.typ.skipTypes(abstractPtrs).flags:
-        localError(name.info, "only .partial objects can be extended")
+    if name.kind == nkDotExpr and a[2].kind == nkObjectTy:
+      let pkgName = considerQuotedIdent(name[0])
+      let typName = considerQuotedIdent(name[1])
+      let pkg = c.graph.packageSyms.strTableGet(pkgName)
+      if pkg.isNil or pkg.kind != skPackage:
+        localError(name.info, "unknown package name: " & pkgName.s)
+      else:
+        let typsym = pkg.tab.strTableGet(typName)
+        if typsym.isNil:
+          s = semIdentDef(c, name[1], skType)
+          s.typ = newTypeS(tyObject, c)
+          s.typ.sym = s
+          s.flags.incl sfForward
+          pkg.tab.strTableAdd s
+          addInterfaceDecl(c, s)
+        elif typsym.kind == skType and sfForward in typsym.flags:
+          s = typsym
+          addInterfaceDecl(c, s)
+        else:
+          localError(name.info, typsym.name.s & " is not a type that can be forwarded")
+          s = typsym
     else:
       s = semIdentDef(c, name, skType)
       s.typ = newTypeS(tyForward, c)
       s.typ.sym = s             # process pragmas:
       if name.kind == nkPragmaExpr:
         pragma(c, s, name.sons[1], typePragmas)
+      if sfForward in s.flags:
+        # check if the symbol already exists:
+        let pkg = c.module.owner
+        if not isTopLevel(c) or pkg.isNil:
+          localError(name.info, "only top level types in a package can be 'package'")
+        else:
+          let typsym = pkg.tab.strTableGet(s.name)
+          if typsym != nil:
+            if sfForward notin typsym.flags or sfNoForward notin typsym.flags:
+              typeCompleted(typsym)
+              typsym.info = s.info
+            else:
+              localError(name.info, "cannot complete type '" & s.name.s & "' twice; " &
+                      "previous type completion was here: " & $typsym.info)
+            s = typsym
       # add it here, so that recursive types are possible:
       if sfGenSym notin s.flags: addInterfaceDecl(c, s)
+
     a.sons[0] = newSymNode(s)
 
 proc checkCovariantParamsUsages(genericType: PType) =
-  var body = genericType{-1}
+  var body = genericType[^1]
 
   proc traverseSubTypes(t: PType): bool =
     template error(msg) = localError(genericType.sym.info, msg)
@@ -826,7 +856,7 @@ proc checkCovariantParamsUsages(genericType: PType) =
 
     of tyGenericInvocation:
       let targetBody = t[0]
-      for i in 1 .. <t.len:
+      for i in 1 ..< t.len:
         let param = t[i]
         if param.kind == tyGenericParam:
           if tfCovariant in param.flags:
@@ -856,11 +886,11 @@ proc checkCovariantParamsUsages(genericType: PType) =
       for fieldType in t.sons:
         subresult traverseSubTypes(fieldType)
 
-    of tyPtr, tyRef, tyVar:
+    of tyPtr, tyRef, tyVar, tyLent:
       if t.base.kind == tyGenericParam: return true
       return traverseSubTypes(t.base)
 
-    of tyDistinct, tyAlias:
+    of tyDistinct, tyAlias, tySink:
       return traverseSubTypes(t.lastSon)
 
     of tyGenericInst:
@@ -970,10 +1000,10 @@ proc checkForMetaFields(n: PNode) =
   of nkSym:
     let t = n.sym.typ
     case t.kind
-    of tySequence, tySet, tyArray, tyOpenArray, tyVar, tyPtr, tyRef,
-       tyProc, tyGenericInvocation, tyGenericInst, tyAlias:
-      let start = ord(t.kind in {tyGenericInvocation, tyGenericInst})
-      for i in start .. <t.sons.len:
+    of tySequence, tySet, tyArray, tyOpenArray, tyVar, tyLent, tyPtr, tyRef,
+       tyProc, tyGenericInvocation, tyGenericInst, tyAlias, tySink:
+      let start = int ord(t.kind in {tyGenericInvocation, tyGenericInst})
+      for i in start ..< t.sons.len:
         checkMeta(t.sons[i])
     else:
       checkMeta(t)
@@ -996,7 +1026,7 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
         # type aliases are hard:
         var t = semTypeNode(c, x, nil)
         assert t != nil
-        if s.typ != nil and s.typ.kind != tyAlias:
+        if s.typ != nil and s.typ.kind notin {tyAlias, tySink}:
           if t.kind in {tyProc, tyGenericInst} and not t.isMetaType:
             assignType(s.typ, t)
             s.typ.id = t.id
@@ -1007,6 +1037,8 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
       checkConstructedType(s.info, s.typ)
       if s.typ.kind in {tyObject, tyTuple} and not s.typ.n.isNil:
         checkForMetaFields(s.typ.n)
+  instAllTypeBoundOp(c, n.info)
+
 
 proc semAllTypeSections(c: PContext; n: PNode): PNode =
   proc gatherStmts(c: PContext; n: PNode; result: PNode) {.nimcall.} =
@@ -1061,9 +1093,11 @@ proc semTypeSection(c: PContext, n: PNode): PNode =
   ## to allow the type definitions in the section to reference each other
   ## without regard for the order of their definitions.
   if sfNoForward notin c.module.flags or nfSem notin n.flags:
+    inc c.inTypeContext
     typeSectionLeftSidePass(c, n)
     typeSectionRightSidePass(c, n)
     typeSectionFinalPass(c, n)
+    dec c.inTypeContext
   result = n
 
 proc semParamList(c: PContext, n, genericParams: PNode, s: PSym) =
@@ -1099,7 +1133,7 @@ proc addResultNode(c: PContext, n: PNode) =
 
 proc copyExcept(n: PNode, i: int): PNode =
   result = copyNode(n)
-  for j in 0.. <n.len:
+  for j in 0..<n.len:
     if j != i: result.add(n.sons[j])
 
 proc lookupMacro(c: PContext, n: PNode): PSym =
@@ -1113,9 +1147,9 @@ proc semProcAnnotation(c: PContext, prc: PNode;
                        validPragmas: TSpecialWords): PNode =
   var n = prc.sons[pragmasPos]
   if n == nil or n.kind == nkEmpty: return
-  for i in countup(0, <n.len):
+  for i in countup(0, n.len-1):
     var it = n.sons[i]
-    var key = if it.kind == nkExprColonExpr: it.sons[0] else: it
+    var key = if it.kind in nkPragmaCallKinds and it.len >= 1: it.sons[0] else: it
     let m = lookupMacro(c, key)
     if m == nil:
       if key.kind == nkIdent and key.ident.id == ord(wDelegator):
@@ -1125,6 +1159,9 @@ proc semProcAnnotation(c: PContext, prc: PNode;
         else:
           localError(prc.info, errOnlyACallOpCanBeDelegator)
       continue
+    elif sfCustomPragma in m.flags:
+      continue # semantic check for custom pragma happens later in semProcAux
+
     # we transform ``proc p {.m, rest.}`` into ``m(do: proc p {.rest.})`` and
     # let the semantic checker deal with it:
     var x = newNodeI(nkCall, n.info)
@@ -1133,10 +1170,12 @@ proc semProcAnnotation(c: PContext, prc: PNode;
     if prc[pragmasPos].kind != nkEmpty and prc[pragmasPos].len == 0:
       prc.sons[pragmasPos] = emptyNode
 
-    if it.kind == nkExprColonExpr:
-      # pass pragma argument to the macro too:
-      x.add(it.sons[1])
+    if it.kind in nkPragmaCallKinds and it.len > 1:
+      # pass pragma arguments to the macro too:
+      for i in 1..<it.len:
+        x.add(it.sons[i])
     x.add(prc)
+
     # recursion assures that this works for multiple macro annotations too:
     result = semExpr(c, x)
     # since a proc annotation can set pragmas, we process these here again.
@@ -1233,7 +1272,7 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
   s.typ = n.typ
   for i in 1..<params.len:
     if params[i].typ.kind in {tyTypeDesc, tyGenericParam,
-                              tyFromExpr, tyFieldAccessor}+tyTypeClasses:
+                              tyFromExpr}+tyTypeClasses:
       localError(params[i].info, "cannot infer type of parameter: " &
                  params[i].sym.name.s)
     #params[i].sym.owner = s
@@ -1264,7 +1303,7 @@ proc activate(c: PContext, n: PNode) =
     of nkLambdaKinds:
       discard semLambda(c, n, {})
     of nkCallKinds:
-      for i in 1 .. <n.len: activate(c, n[i])
+      for i in 1 ..< n.len: activate(c, n[i])
     else:
       discard
 
@@ -1277,9 +1316,26 @@ proc maybeAddResult(c: PContext, s: PSym, n: PNode) =
 proc semOverride(c: PContext, s: PSym, n: PNode) =
   case s.name.s.normalize
   of "destroy", "=destroy":
-    doDestructorStuff(c, s, n)
-    if not experimentalMode(c):
-      localError n.info, "use the {.experimental.} pragma to enable destructors"
+    if newDestructors:
+      let t = s.typ
+      var noError = false
+      if t.len == 2 and t.sons[0] == nil and t.sons[1].kind == tyVar:
+        var obj = t.sons[1].sons[0]
+        while true:
+          incl(obj.flags, tfHasAsgn)
+          if obj.kind in {tyGenericBody, tyGenericInst}: obj = obj.lastSon
+          elif obj.kind == tyGenericInvocation: obj = obj.sons[0]
+          else: break
+        if obj.kind in {tyObject, tyDistinct}:
+          if obj.destructor.isNil:
+            obj.destructor = s
+          else:
+            localError(n.info, errGenerated,
+              "cannot bind another '" & s.name.s & "' to: " & typeToString(obj))
+          noError = true
+      if not noError and sfSystemModule notin s.owner.flags:
+        localError(n.info, errGenerated,
+          "signature for '" & s.name.s & "' must be proc[T: object](x: var T)")
     incl(s.flags, sfUsed)
   of "deepcopy", "=deepcopy":
     if s.typ.len == 2 and
@@ -1304,7 +1360,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
       localError(n.info, errGenerated,
                  "signature for 'deepCopy' must be proc[T: ptr|ref](x: T): T")
     incl(s.flags, sfUsed)
-  of "=":
+  of "=", "=sink":
     if s.magic == mAsgn: return
     incl(s.flags, sfUsed)
     let t = s.typ
@@ -1322,14 +1378,16 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
           objB = objB.sons[0]
         else: break
       if obj.kind in {tyObject, tyDistinct} and sameType(obj, objB):
-        if obj.assignment.isNil:
-          obj.assignment = s
+        let opr = if s.name.s == "=": addr(obj.assignment) else: addr(obj.sink)
+        if opr[].isNil:
+          opr[] = s
         else:
           localError(n.info, errGenerated,
-                     "cannot bind another '=' to: " & typeToString(obj))
+                     "cannot bind another '" & s.name.s & "' to: " & typeToString(obj))
         return
-    localError(n.info, errGenerated,
-               "signature for '=' must be proc[T: object](x: var T; y: T)")
+    if sfSystemModule notin s.owner.flags:
+      localError(n.info, errGenerated,
+                "signature for '" & s.name.s & "' must be proc[T: object](x: var T; y: T)")
   else:
     if sfOverriden in s.flags:
       localError(n.info, errGenerated,
@@ -1368,9 +1426,9 @@ proc semMethodPrototype(c: PContext; s: PSym; n: PNode) =
     for col in countup(1, sonsLen(tt)-1):
       let t = tt.sons[col]
       if t != nil and t.kind == tyGenericInvocation:
-        var x = skipTypes(t.sons[0], {tyVar, tyPtr, tyRef, tyGenericInst,
+        var x = skipTypes(t.sons[0], {tyVar, tyLent, tyPtr, tyRef, tyGenericInst,
                                       tyGenericInvocation, tyGenericBody,
-                                      tyAlias})
+                                      tyAlias, tySink})
         if x.kind == tyObject and t.len-1 == n.sons[genericParamsPos].len:
           foundObj = true
           x.methods.safeAdd((col,s))
@@ -1504,8 +1562,11 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   s.options = gOptions
   if sfOverriden in s.flags or s.name.s[0] == '=': semOverride(c, s, n)
   if s.name.s[0] in {'.', '('}:
-    if s.name.s in [".", ".()", ".=", "()"] and not experimentalMode(c):
+    if s.name.s in [".", ".()", ".="] and not experimentalMode(c) and not newDestructors:
       message(n.info, warnDeprecated, "overloaded '.' and '()' operators are now .experimental; " & s.name.s)
+    elif s.name.s == "()" and not experimentalMode(c):
+      message(n.info, warnDeprecated, "overloaded '()' operators are now .experimental; " & s.name.s)
+
   if n.sons[bodyPos].kind != nkEmpty:
     # for DLL generation it is annoying to check for sfImportc!
     if sfBorrow in s.flags:
@@ -1579,6 +1640,10 @@ proc semIterator(c: PContext, n: PNode): PNode =
     n[namePos].sym.owner = getCurrOwner(c)
     n[namePos].sym.kind = skIterator
   result = semProcAux(c, n, skIterator, iteratorPragmas)
+  # bug #7093: if after a macro transformation we don't have an
+  # nkIteratorDef aynmore, return. The iterator then might have been
+  # sem'checked already. (Or not, if the macro skips it.)
+  if result.kind != n.kind: return
   var s = result.sons[namePos].sym
   var t = s.typ
   if t.sons[0] == nil and s.typ.callConv != ccClosure:
@@ -1603,11 +1668,18 @@ proc semIterator(c: PContext, n: PNode): PNode =
 proc semProc(c: PContext, n: PNode): PNode =
   result = semProcAux(c, n, skProc, procPragmas)
 
+proc semFunc(c: PContext, n: PNode): PNode =
+  result = semProcAux(c, n, skFunc, procPragmas)
+
 proc semMethod(c: PContext, n: PNode): PNode =
   if not isTopLevel(c): localError(n.info, errXOnlyAtModuleScope, "method")
   result = semProcAux(c, n, skMethod, methodPragmas)
   # macros can transform converters to nothing:
   if namePos >= result.safeLen: return result
+  # bug #7093: if after a macro transformation we don't have an
+  # nkIteratorDef aynmore, return. The iterator then might have been
+  # sem'checked already. (Or not, if the macro skips it.)
+  if result.kind != nkMethodDef: return
   var s = result.sons[namePos].sym
   # we need to fix the 'auto' return type for the dispatcher here (see tautonotgeneric
   # test case):
@@ -1626,6 +1698,10 @@ proc semConverterDef(c: PContext, n: PNode): PNode =
   result = semProcAux(c, n, skConverter, converterPragmas)
   # macros can transform converters to nothing:
   if namePos >= result.safeLen: return result
+  # bug #7093: if after a macro transformation we don't have an
+  # nkIteratorDef aynmore, return. The iterator then might have been
+  # sem'checked already. (Or not, if the macro skips it.)
+  if result.kind != nkConverterDef: return
   var s = result.sons[namePos].sym
   var t = s.typ
   if t.sons[0] == nil: localError(n.info, errXNeedsReturnType, "converter")
@@ -1637,6 +1713,10 @@ proc semMacroDef(c: PContext, n: PNode): PNode =
   result = semProcAux(c, n, skMacro, macroPragmas)
   # macros can transform macros to nothing:
   if namePos >= result.safeLen: return result
+  # bug #7093: if after a macro transformation we don't have an
+  # nkIteratorDef aynmore, return. The iterator then might have been
+  # sem'checked already. (Or not, if the macro skips it.)
+  if result.kind != nkMacroDef: return
   var s = result.sons[namePos].sym
   var t = s.typ
   var allUntyped = true
@@ -1661,7 +1741,7 @@ proc evalInclude(c: PContext, n: PNode): PNode =
         excl(c.includedFiles, f)
 
 proc setLine(n: PNode, info: TLineInfo) =
-  for i in 0 .. <safeLen(n): setLine(n.sons[i], info)
+  for i in 0 ..< safeLen(n): setLine(n.sons[i], info)
   n.info = info
 
 proc semPragmaBlock(c: PContext, n: PNode): PNode =
@@ -1669,7 +1749,7 @@ proc semPragmaBlock(c: PContext, n: PNode): PNode =
   pragma(c, nil, pragmaList, exprPragmas)
   result = semExpr(c, n.sons[1])
   n.sons[1] = result
-  for i in 0 .. <pragmaList.len:
+  for i in 0 ..< pragmaList.len:
     case whichPragma(pragmaList.sons[i])
     of wLine: setLine(result, pragmaList.sons[i].info)
     of wLocks, wGcSafe:
@@ -1802,11 +1882,12 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
       else:
         n.typ = n.sons[i].typ
         if not isEmptyType(n.typ): n.kind = nkStmtListExpr
-      case n.sons[i].kind
-      of LastBlockStmts:
+      if n.sons[i].kind in LastBlockStmts or
+         n.sons[i].kind in nkCallKinds and n.sons[i][0].kind == nkSym and sfNoReturn in n.sons[i][0].sym.flags:
         for j in countup(i + 1, length - 1):
           case n.sons[j].kind
-          of nkPragma, nkCommentStmt, nkNilLit, nkEmpty: discard
+          of nkPragma, nkCommentStmt, nkNilLit, nkEmpty, nkBlockExpr,
+             nkBlockStmt, nkState: discard
           else: localError(n.sons[j].info, errStmtInvalidAfterReturn)
       else: discard
 

compiler/semtempl.nim

@@ -75,7 +75,7 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule): PNode =
       a = nextOverloadIter(o, c, n)
 
 proc semBindStmt(c: PContext, n: PNode, toBind: var IntSet): PNode =
-  for i in 0 .. < n.len:
+  for i in 0 ..< n.len:
     var a = n.sons[i]
     # If 'a' is an overloaded symbol, we used to use the first symbol
     # as a 'witness' and use the fact that subsequent lookups will yield
@@ -95,7 +95,7 @@ proc semBindStmt(c: PContext, n: PNode, toBind: var IntSet): PNode =
   result = newNodeI(nkEmpty, n.info)
 
 proc semMixinStmt(c: PContext, n: PNode, toMixin: var IntSet): PNode =
-  for i in 0 .. < n.len:
+  for i in 0 ..< n.len:
     toMixin.incl(considerQuotedIdent(n.sons[i]).id)
   result = newNodeI(nkEmpty, n.info)
 
@@ -113,13 +113,9 @@ type
     owner: PSym
     cursorInBody: bool # only for nimsuggest
     scopeN: int
-    bracketExpr: PNode
 
 template withBracketExpr(ctx, x, body: untyped) =
-  let old = ctx.bracketExpr
-  ctx.bracketExpr = x
   body
-  ctx.bracketExpr = old
 
 proc getIdentNode(c: var TemplCtx, n: PNode): PNode =
   case n.kind
@@ -163,7 +159,7 @@ proc onlyReplaceParams(c: var TemplCtx, n: PNode): PNode =
         result = newSymNode(s, n.info)
         styleCheckUse(n.info, s)
   else:
-    for i in 0 .. <n.safeLen:
+    for i in 0 ..< n.safeLen:
       result.sons[i] = onlyReplaceParams(c, n.sons[i])
 
 proc newGenSym(kind: TSymKind, n: PNode, c: var TemplCtx): PSym =
@@ -301,21 +297,9 @@ proc semPattern(c: PContext, n: PNode): PNode
 
 proc semTemplBodySons(c: var TemplCtx, n: PNode): PNode =
   result = n
-  for i in 0.. < n.len:
+  for i in 0 ..< n.len:
     result.sons[i] = semTemplBody(c, n.sons[i])
 
-proc oprIsRoof(n: PNode): bool =
-  const roof = "^"
-  case n.kind
-  of nkIdent: result = n.ident.s == roof
-  of nkSym: result = n.sym.name.s == roof
-  of nkAccQuoted:
-    if n.len == 1:
-      result = oprIsRoof(n.sons[0])
-  of nkOpenSymChoice, nkClosedSymChoice:
-    result = oprIsRoof(n.sons[0])
-  else: discard
-
 proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
   result = n
   semIdeForTemplateOrGenericCheck(n, c.cursorInBody)
@@ -347,7 +331,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
   of nkMixinStmt:
     if c.scopeN > 0: result = semTemplBodySons(c, n)
     else: result = semMixinStmt(c.c, n, c.toMixin)
-  of nkEmpty, nkSym..nkNilLit:
+  of nkEmpty, nkSym..nkNilLit, nkComesFrom:
     discard
   of nkIfStmt:
     for i in countup(0, sonsLen(n)-1):
@@ -382,8 +366,10 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
     n.sons[L-2] = semTemplBody(c, n.sons[L-2])
     for i in countup(0, L - 3):
       addLocalDecl(c, n.sons[i], skForVar)
+    openScope(c)
     n.sons[L-1] = semTemplBody(c, n.sons[L-1])
     closeScope(c)
+    closeScope(c)
   of nkBlockStmt, nkBlockExpr, nkBlockType:
     checkSonsLen(n, 2)
     openScope(c)
@@ -449,6 +435,8 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
         a.sons[2] = semTemplBody(c, a.sons[2])
   of nkProcDef, nkLambdaKinds:
     result = semRoutineInTemplBody(c, n, skProc)
+  of nkFuncDef:
+    result = semRoutineInTemplBody(c, n, skFunc)
   of nkMethodDef:
     result = semRoutineInTemplBody(c, n, skMethod)
   of nkIteratorDef:
@@ -504,8 +492,6 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
       result = semTemplBodySons(c, n)
   of nkCallKinds-{nkPostfix}:
     result = semTemplBodySons(c, n)
-    if c.bracketExpr != nil and n.len == 2 and oprIsRoof(n.sons[0]):
-      result.add c.bracketExpr
   of nkDotExpr, nkAccQuoted:
     # dotExpr is ambiguous: note that we explicitly allow 'x.TemplateParam',
     # so we use the generic code for nkDotExpr too
@@ -542,7 +528,7 @@ proc semTemplBodyDirty(c: var TemplCtx, n: PNode): PNode =
     result = semTemplBodyDirty(c, n.sons[0])
   of nkBindStmt:
     result = semBindStmt(c.c, n, c.toBind)
-  of nkEmpty, nkSym..nkNilLit:
+  of nkEmpty, nkSym..nkNilLit, nkComesFrom:
     discard
   else:
     # dotExpr is ambiguous: note that we explicitly allow 'x.TemplateParam',
@@ -622,7 +608,10 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
   popOwner(c)
   s.ast = n
   result = n
-  if n.sons[bodyPos].kind == nkEmpty:
+  if sfCustomPragma in s.flags:
+    if n.sons[bodyPos].kind != nkEmpty:
+      localError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s)
+  elif n.sons[bodyPos].kind == nkEmpty:
     localError(n.info, errImplOfXexpected, s.name.s)
   var proto = searchForProc(c, c.currentScope, s)
   if proto == nil:

compiler/semtypes.nim

@@ -88,7 +88,9 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
       if not isPure: strTableAdd(c.module.tab, e)
     addSon(result.n, newSymNode(e))
     styleCheckDef(e)
-    if sfGenSym notin e.flags and not isPure: addDecl(c, e)
+    if sfGenSym notin e.flags:
+      if not isPure: addDecl(c, e)
+      else: importPureEnumField(c, e)
     if isPure and strTableIncl(symbols, e):
       wrongRedefinition(e.info, e.name.s)
     inc(counter)
@@ -99,7 +101,7 @@ proc semSet(c: PContext, n: PNode, prev: PType): PType =
   if sonsLen(n) == 2:
     var base = semTypeNode(c, n.sons[1], nil)
     addSonSkipIntLit(result, base)
-    if base.kind in {tyGenericInst, tyAlias}: base = lastSon(base)
+    if base.kind in {tyGenericInst, tyAlias, tySink}: base = lastSon(base)
     if base.kind != tyGenericParam:
       if not isOrdinalType(base):
         localError(n.info, errOrdinalTypeExpected)
@@ -136,7 +138,7 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType =
   if n.len < 1:
     result = newConstraint(c, kind)
   else:
-    let isCall = ord(n.kind in nkCallKinds+{nkBracketExpr})
+    let isCall = int ord(n.kind in nkCallKinds+{nkBracketExpr})
     let n = if n[0].kind == nkBracket: n[0] else: n
     checkMinSonsLen(n, 1)
     var t = semTypeNode(c, n.lastSon, nil)
@@ -151,7 +153,7 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType =
         isNilable = true
       else:
         let region = semTypeNode(c, ni, nil)
-        if region.skipTypes({tyGenericInst, tyAlias}).kind notin {
+        if region.skipTypes({tyGenericInst, tyAlias, tySink}).kind notin {
               tyError, tyObject}:
           message n[i].info, errGenerated, "region needs to be an object type"
         addSonSkipIntLit(result, region)
@@ -233,7 +235,10 @@ proc semRange(c: PContext, n: PNode, prev: PType): PType =
           n.sons[1].floatVal < 0.0:
         incl(result.flags, tfNeedsInit)
     else:
-      localError(n.sons[0].info, errRangeExpected)
+      if n[1].kind == nkInfix and considerQuotedIdent(n[1][0]).s == "..<":
+        localError(n[0].info, "range types need to be constructed with '..', '..<' is not supported")
+      else:
+        localError(n.sons[0].info, errRangeExpected)
       result = newOrPrevType(tyError, prev, c)
   else:
     localError(n.info, errXExpectsOneTypeParam, "range")
@@ -281,9 +286,11 @@ proc semArray(c: PContext, n: PNode, prev: PType): PType =
     # 3 = length(array indx base)
     let indx = semArrayIndex(c, n[1])
     var indxB = indx
-    if indxB.kind in {tyGenericInst, tyAlias}: indxB = lastSon(indxB)
+    if indxB.kind in {tyGenericInst, tyAlias, tySink}: indxB = lastSon(indxB)
     if indxB.kind notin {tyGenericParam, tyStatic, tyFromExpr}:
-      if not isOrdinalType(indxB):
+      if indxB.skipTypes({tyRange}).kind in {tyUInt, tyUInt64}:
+        discard
+      elif not isOrdinalType(indxB):
         localError(n.sons[1].info, errOrdinalTypeExpected)
       elif enumHasHoles(indxB):
         localError(n.sons[1].info, errEnumXHasHoles,
@@ -291,7 +298,9 @@ proc semArray(c: PContext, n: PNode, prev: PType): PType =
     base = semTypeNode(c, n.sons[2], nil)
     # ensure we only construct a tyArray when there was no error (bug #3048):
     result = newOrPrevType(tyArray, prev, c)
-    addSonSkipIntLit(result, indx)
+    # bug #6682: Do not propagate initialization requirements etc for the
+    # index type:
+    rawAddSonNoPropagationOfTypeFlags(result, indx)
     addSonSkipIntLit(result, base)
   else:
     localError(n.info, errArrayExpectsTwoTypeParams)
@@ -313,11 +322,8 @@ proc semTypeIdent(c: PContext, n: PNode): PSym =
   if n.kind == nkSym:
     result = getGenSym(c, n.sym)
   else:
-    when defined(nimfix):
-      result = pickSym(c, n, skType)
-      if result.isNil:
-        result = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared})
-    else:
+    result = pickSym(c, n, {skType, skGenericParam})
+    if result.isNil:
       result = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared})
     if result != nil:
       markUsed(n.info, result, c.graph.usageSym)
@@ -491,8 +497,8 @@ proc semCaseBranchSetElem(c: PContext, t, b: PNode,
 
 proc semCaseBranch(c: PContext, t, branch: PNode, branchIndex: int,
                    covered: var BiggestInt) =
-
-  for i in countup(0, sonsLen(branch) - 2):
+  let lastIndex = sonsLen(branch) - 2
+  for i in 0..lastIndex:
     var b = branch.sons[i]
     if b.kind == nkRange:
       branch.sons[i] = b
@@ -510,14 +516,21 @@ proc semCaseBranch(c: PContext, t, branch: PNode, branchIndex: int,
         branch.sons[i] = skipConv(fitNode(c, t.sons[0].typ, r, r.info))
         inc(covered)
       else:
+        if r.kind == nkCurly:
+          r = r.deduplicate
+
         # first element is special and will overwrite: branch.sons[i]:
         branch.sons[i] = semCaseBranchSetElem(c, t, r[0], covered)
+
         # other elements have to be added to ``branch``
-        for j in 1 .. <r.len:
+        for j in 1 ..< r.len:
           branch.add(semCaseBranchSetElem(c, t, r[j], covered))
           # caution! last son of branch must be the actions to execute:
-          var L = branch.len
-          swap(branch.sons[L-2], branch.sons[L-1])
+          swap(branch.sons[^2], branch.sons[^1])
+    checkForOverlap(c, t, i, branchIndex)
+
+  # Elements added above needs to be checked for overlaps.
+  for i in lastIndex.succ..(sonsLen(branch) - 2):
     checkForOverlap(c, t, i, branchIndex)
 
 proc semRecordNodeAux(c: PContext, n: PNode, check: var IntSet, pos: var int,
@@ -669,7 +682,7 @@ proc skipGenericInvocation(t: PType): PType {.inline.} =
   result = t
   if result.kind == tyGenericInvocation:
     result = result.sons[0]
-  while result.kind in {tyGenericInst, tyGenericBody, tyRef, tyPtr, tyAlias}:
+  while result.kind in {tyGenericInst, tyGenericBody, tyRef, tyPtr, tyAlias, tySink}:
     result = lastSon(result)
 
 proc addInheritedFields(c: PContext, check: var IntSet, pos: var int,
@@ -832,7 +845,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
       result = liftingWalk(paramType.sons[0], true)
 
   of tySequence, tySet, tyArray, tyOpenArray,
-     tyVar, tyPtr, tyRef, tyProc:
+     tyVar, tyLent, tyPtr, tyRef, tyProc:
     # XXX: this is a bit strange, but proc(s: seq)
     # produces tySequence(tyGenericParam, tyNone).
     # This also seems to be true when creating aliases
@@ -844,7 +857,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
                                   @[newTypeS(paramType.kind, c)])
       result = addImplicitGeneric(typ)
     else:
-      for i in 0 .. <paramType.len:
+      for i in 0 ..< paramType.len:
         if paramType.sons[i] == paramType:
           globalError(info, errIllegalRecursionInTypeX, typeToString(paramType))
         var lifted = liftingWalk(paramType.sons[i])
@@ -895,7 +908,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
       result.shouldHaveMeta
 
   of tyGenericInvocation:
-    for i in 1 .. <paramType.len:
+    for i in 1 ..< paramType.len:
       let lifted = liftingWalk(paramType.sons[i])
       if lifted != nil: paramType.sons[i] = lifted
 
@@ -988,7 +1001,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
       if isType: localError(a.info, "':' expected")
       if kind in {skTemplate, skMacro}:
         typ = newTypeS(tyExpr, c)
-    elif skipTypes(typ, {tyGenericInst, tyAlias}).kind == tyVoid:
+    elif skipTypes(typ, {tyGenericInst, tyAlias, tySink}).kind == tyVoid:
       continue
     for j in countup(0, length-3):
       var arg = newSymG(skParam, a.sons[j], c)
@@ -1020,7 +1033,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
   if r != nil:
     # turn explicit 'void' return type into 'nil' because the rest of the
     # compiler only checks for 'nil':
-    if skipTypes(r, {tyGenericInst, tyAlias}).kind != tyVoid:
+    if skipTypes(r, {tyGenericInst, tyAlias, tySink}).kind != tyVoid:
       # 'auto' as a return type does not imply a generic:
       if r.kind == tyAnything:
         # 'p(): auto' and 'p(): expr' are equivalent, but the rest of the
@@ -1043,6 +1056,12 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
           result.flags.incl tfIterator
           # XXX Would be nice if we could get rid of this
       result.sons[0] = r
+      let oldFlags = result.flags
+      propagateToOwner(result, r)
+      if oldFlags != result.flags:
+        # XXX This rather hacky way keeps 'tflatmap' compiling:
+        if tfHasMeta notin oldFlags:
+          result.flags.excl tfHasMeta
       result.n.typ = r
 
   if genericParams != nil and genericParams.len > 0:
@@ -1137,14 +1156,14 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType =
 
     if m.state != csMatch:
       let err = "cannot instantiate " & typeToString(t) & "\n" &
-                "got: (" & describeArgs(c, n) & ")\n" &
-                "but expected: (" & describeArgs(c, t.n, 0) & ")"
+                "got: <" & describeArgs(c, n) & ">\n" &
+                "but expected: <" & describeArgs(c, t.n, 0) & ">"
       localError(n.info, errGenerated, err)
       return newOrPrevType(tyError, prev, c)
 
     var isConcrete = true
 
-    for i in 1 .. <m.call.len:
+    for i in 1 ..< m.call.len:
       var typ = m.call[i].typ
       if typ.kind == tyTypeDesc and typ.sons[0].kind == tyNone:
         isConcrete = false
@@ -1165,7 +1184,10 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType =
 
   # special check for generic object with
   # generic/partial specialized parent
-  let tx = result.skipTypes(abstractPtrs)
+  let tx = result.skipTypes(abstractPtrs, 50)
+  if tx.isNil:
+    localError(n.info, "invalid recursion in type '$1'" % typeToString(result[0]))
+    return errorType(c)
   if tx != result and tx.kind == tyObject and tx.sons[0] != nil:
     semObjectTypeForInheritedGenericInst(c, n, tx)
 
@@ -1213,8 +1235,6 @@ template modifierTypeKindOfNode(n: PNode): TTypeKind =
 
 proc semTypeClass(c: PContext, n: PNode, prev: PType): PType =
   # if n.sonsLen == 0: return newConstraint(c, tyTypeClass)
-  if nfBase2 in n.flags:
-    message(n.info, warnDeprecated, "use 'concept' instead; 'generic'")
   let
     pragmas = n[1]
     inherited = n[2]
@@ -1293,8 +1313,7 @@ proc symFromExpectedTypeNode(c: PContext, n: PNode): PSym =
 
 proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
   result = nil
-  when defined(nimsuggest):
-    inc c.inTypeContext
+  inc c.inTypeContext
 
   if gCmd == cmdIdeTools: suggestExpr(c, n)
   case n.kind
@@ -1325,7 +1344,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
       result = semRangeAux(c, n, prev)
     elif n[0].kind == nkNilLit and n.len == 2:
       result = semTypeNode(c, n.sons[1], prev)
-      if result.skipTypes({tyGenericInst, tyAlias}).kind in NilableTypes+GenericTypes:
+      if result.skipTypes({tyGenericInst, tyAlias, tySink}).kind in NilableTypes+GenericTypes:
         if tfNotNil in result.flags:
           result = freshType(result, prev)
           result.flags.excl(tfNotNil)
@@ -1353,7 +1372,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
         case n.len
         of 3:
           result = semTypeNode(c, n.sons[1], prev)
-          if result.skipTypes({tyGenericInst, tyAlias}).kind in NilableTypes+GenericTypes and
+          if result.skipTypes({tyGenericInst, tyAlias, tySink}).kind in NilableTypes+GenericTypes+{tyForward} and
               n.sons[2].kind == nkNilLit:
             result = freshType(result, prev)
             result.flags.incl(tfNotNil)
@@ -1391,6 +1410,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     of mSet: result = semSet(c, n, prev)
     of mOrdinal: result = semOrdinal(c, n, prev)
     of mSeq: result = semContainer(c, n, tySequence, "seq", prev)
+    of mOpt: result = semContainer(c, n, tyOpt, "opt", prev)
     of mVarargs: result = semVarargs(c, n, prev)
     of mTypeDesc: result = makeTypeDesc(c, semTypeNode(c, n[1], nil))
     of mExpr:
@@ -1405,7 +1425,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     of mVar:
       result = newOrPrevType(tyVar, prev, c)
       var base = semTypeNode(c, n.sons[1], nil)
-      if base.kind == tyVar:
+      if base.kind in {tyVar, tyLent}:
         localError(n.info, errVarVarTypeNotAllowed)
         base = base.sons[0]
       addSonSkipIntLit(result, base)
@@ -1415,9 +1435,13 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     else: result = semGeneric(c, n, s, prev)
   of nkDotExpr:
     let typeExpr = semExpr(c, n)
-    if typeExpr.typ.kind == tyFromExpr:
-      return typeExpr.typ
-    if typeExpr.typ.kind != tyTypeDesc:
+    if typeExpr.typ.isNil:
+      localError(n.info, "object constructor needs an object type;" &
+          " for named arguments use '=' instead of ':'")
+      result = errorType(c)
+    elif typeExpr.typ.kind == tyFromExpr:
+      result = typeExpr.typ
+    elif typeExpr.typ.kind != tyTypeDesc:
       localError(n.info, errTypeExpected)
       result = errorType(c)
     else:
@@ -1508,8 +1532,13 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     localError(n.info, errTypeExpected)
     result = newOrPrevType(tyError, prev, c)
   n.typ = result
-  when defined(nimsuggest):
-    dec c.inTypeContext
+  dec c.inTypeContext
+  if c.inTypeContext == 0: instAllTypeBoundOp(c, n.info)
+
+when false:
+  proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
+    result = semTypeNodeInner(c, n, prev)
+    instAllTypeBoundOp(c, n.info)
 
 proc setMagicType(m: PSym, kind: TTypeKind, size: int) =
   # source : https://en.wikipedia.org/wiki/Data_structure_alignment#x86
@@ -1590,11 +1619,19 @@ proc processMagicType(c: PContext, m: PSym) =
     setMagicType(m, tySet, 0)
   of mSeq:
     setMagicType(m, tySequence, 0)
+  of mOpt:
+    setMagicType(m, tyOpt, 0)
   of mOrdinal:
     setMagicType(m, tyOrdinal, 0)
     rawAddSon(m.typ, newTypeS(tyNone, c))
   of mPNimrodNode:
     incl m.typ.flags, tfTriggersCompileTime
+  of mException: discard
+  of mBuiltinType:
+    case m.name.s
+    of "lent": setMagicType(m, tyLent, ptrSize)
+    of "sink": setMagicType(m, tySink, 0)
+    else: localError(m.info, errTypeExpected)
   else: localError(m.info, errTypeExpected)
 
 proc semGenericConstraints(c: PContext, x: PType): PType =
@@ -1609,8 +1646,8 @@ proc semGenericParamList(c: PContext, n: PNode, father: PType = nil): PNode =
     var a = n.sons[i]
     if a.kind != nkIdentDefs: illFormedAst(n)
     let L = a.len
-    var def = a{-1}
-    let constraint = a{-2}
+    var def = a[^1]
+    let constraint = a[^2]
     var typ: PType
 
     if constraint.kind != nkEmpty:

compiler/semtypinst.nim

@@ -17,7 +17,7 @@ const
 proc checkPartialConstructedType(info: TLineInfo, t: PType) =
   if tfAcyclic in t.flags and skipTypes(t, abstractInst).kind != tyObject:
     localError(info, errInvalidPragmaX, "acyclic")
-  elif t.kind == tyVar and t.sons[0].kind == tyVar:
+  elif t.kind in {tyVar, tyLent} and t.sons[0].kind in {tyVar, tyLent}:
     localError(info, errVarVarTypeNotAllowed)
 
 proc checkConstructedType*(info: TLineInfo, typ: PType) =
@@ -25,7 +25,7 @@ proc checkConstructedType*(info: TLineInfo, typ: PType) =
   if t.kind in tyTypeClasses: discard
   elif tfAcyclic in t.flags and skipTypes(t, abstractInst).kind != tyObject:
     localError(info, errInvalidPragmaX, "acyclic")
-  elif t.kind == tyVar and t.sons[0].kind == tyVar:
+  elif t.kind in {tyVar, tyLent} and t.sons[0].kind in {tyVar, tyLent}:
     localError(info, errVarVarTypeNotAllowed)
   elif computeSize(t) == szIllegalRecursion:
     localError(info, errIllegalRecursionInTypeX, typeToString(t))
@@ -77,7 +77,7 @@ type
     topLayer*: TIdTable
     nextLayer*: ptr LayeredIdTable
 
-  TReplTypeVars* {.final.} = object
+  TReplTypeVars* = object
     c*: PContext
     typeMap*: ptr LayeredIdTable # map PType to PType
     symMap*: TIdTable         # map PSym to PSym
@@ -133,7 +133,7 @@ proc prepareNode(cl: var TReplTypeVars, n: PNode): PNode =
   result.typ = t
   if result.kind == nkSym: result.sym = replaceTypeVarsS(cl, n.sym)
   let isCall = result.kind in nkCallKinds
-  for i in 0 .. <n.safeLen:
+  for i in 0 ..< n.safeLen:
     # XXX HACK: ``f(a, b)``, avoid to instantiate `f`
     if isCall and i == 0: result.add(n[i])
     else: result.add(prepareNode(cl, n[i]))
@@ -151,7 +151,7 @@ proc hasGenericArguments*(n: PNode): bool =
            (n.sym.kind == skType and
             n.sym.typ.flags * {tfGenericTypeParam, tfImplicitTypeParam} != {})
   else:
-    for i in 0.. <n.safeLen:
+    for i in 0..<n.safeLen:
       if hasGenericArguments(n.sons[i]): return true
     return false
 
@@ -166,13 +166,13 @@ proc reResolveCallsWithTypedescParams(cl: var TReplTypeVars, n: PNode): PNode =
   # overload resolution is executed again (which may trigger generateInstance).
   if n.kind in nkCallKinds and sfFromGeneric in n[0].sym.flags:
     var needsFixing = false
-    for i in 1 .. <n.safeLen:
+    for i in 1 ..< n.safeLen:
       if isTypeParam(n[i]): needsFixing = true
     if needsFixing:
       n.sons[0] = newSymNode(n.sons[0].sym.owner)
-      return cl.c.semOverloadedCall(cl.c, n, n, {skProc}, {})
+      return cl.c.semOverloadedCall(cl.c, n, n, {skProc, skFunc}, {})
 
-  for i in 0 .. <n.safeLen:
+  for i in 0 ..< n.safeLen:
     n.sons[i] = reResolveCallsWithTypedescParams(cl, n[i])
 
   return n
@@ -261,6 +261,17 @@ proc instCopyType*(cl: var TReplTypeVars, t: PType): PType =
   if not (t.kind in tyMetaTypes or
          (t.kind == tyStatic and t.n == nil)):
     result.flags.excl tfInstClearedFlags
+  when false:
+    if newDestructors:
+      result.assignment = nil
+      #result.destructor = nil
+      result.sink = nil
+
+template typeBound(c, newty, oldty, field, info) =
+  let opr = newty.field
+  if opr != nil and sfFromGeneric notin opr.flags:
+    # '=' needs to be instantiated for generics when the type is constructed:
+    newty.field = c.instTypeBoundOp(c, opr, oldty, info, attachedAsgn, 1)
 
 proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   # tyGenericInvocation[A, tyGenericInvocation[A, B]]
@@ -357,11 +368,10 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
         assert newbody.kind in {tyRef, tyPtr}
         assert newbody.lastSon.typeInst == nil
         newbody.lastSon.typeInst = result
-    let asgn = newbody.assignment
-    if asgn != nil and sfFromGeneric notin asgn.flags:
-      # '=' needs to be instantiated for generics when the type is constructed:
-      newbody.assignment = cl.c.instTypeBoundOp(cl.c, asgn, result, cl.info,
-                                                attachedAsgn, 1)
+    if newDestructors:
+      cl.c.typesWithOps.add((newbody, result))
+    else:
+      typeBound(cl.c, newbody, result, assignment, cl.info)
     let methods = skipTypes(bbody, abstractPtrs).methods
     for col, meth in items(methods):
       # we instantiate the known methods belonging to that type, this causes
@@ -375,11 +385,11 @@ proc eraseVoidParams*(t: PType) =
   if t.sons[0] != nil and t.sons[0].kind == tyVoid:
     t.sons[0] = nil
 
-  for i in 1 .. <t.sonsLen:
+  for i in 1 ..< t.sonsLen:
     # don't touch any memory unless necessary
     if t.sons[i].kind == tyVoid:
       var pos = i
-      for j in i+1 .. <t.sonsLen:
+      for j in i+1 ..< t.sonsLen:
         if t.sons[j].kind != tyVoid:
           t.sons[pos] = t.sons[j]
           t.n.sons[pos] = t.n.sons[j]
@@ -389,7 +399,7 @@ proc eraseVoidParams*(t: PType) =
       return
 
 proc skipIntLiteralParams*(t: PType) =
-  for i in 0 .. <t.sonsLen:
+  for i in 0 ..< t.sonsLen:
     let p = t.sons[i]
     if p == nil: continue
     let skipped = p.skipIntLit
@@ -490,7 +500,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
     bailout()
     result = instCopyType(cl, t)
     idTablePut(cl.localCache, t, result)
-    for i in 1 .. <result.sonsLen:
+    for i in 1 ..< result.sonsLen:
       result.sons[i] = replaceTypeVarsT(cl, result.sons[i])
     propagateToOwner(result, result.lastSon)
 
@@ -508,11 +518,12 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
           var r = replaceTypeVarsT(cl, result.sons[i])
           if result.kind == tyObject:
             # carefully coded to not skip the precious tyGenericInst:
-            let r2 = r.skipTypes({tyAlias})
+            let r2 = r.skipTypes({tyAlias, tySink})
             if r2.kind in {tyPtr, tyRef}:
               r = skipTypes(r2, {tyPtr, tyRef})
           result.sons[i] = r
-          propagateToOwner(result, r)
+          if result.kind != tyArray or i != 0:
+            propagateToOwner(result, r)
       # bug #4677: Do not instantiate effect lists
       result.n = replaceTypeVarsN(cl, result.n, ord(result.kind==tyProc))
       case result.kind
@@ -529,6 +540,17 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
 
       else: discard
 
+proc instAllTypeBoundOp*(c: PContext, info: TLineInfo) =
+  if not newDestructors: return
+  var i = 0
+  while i < c.typesWithOps.len:
+    let (newty, oldty) = c.typesWithOps[i]
+    typeBound(c, newty, oldty, destructor, info)
+    typeBound(c, newty, oldty, sink, info)
+    typeBound(c, newty, oldty, assignment, info)
+    inc i
+  setLen(c.typesWithOps, 0)
+
 proc initTypeVars*(p: PContext, typeMap: ptr LayeredIdTable, info: TLineInfo;
                    owner: PSym): TReplTypeVars =
   initIdTable(result.symMap)

compiler/sighashes.nim

@@ -87,6 +87,7 @@ type
     CoProc
     CoType
     CoOwnerSig
+    CoIgnoreRange
 
 proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag])
 
@@ -136,7 +137,7 @@ proc hashTree(c: var MD5Context, n: PNode) =
   of nkStrLit..nkTripleStrLit:
     c &= n.strVal
   else:
-    for i in 0.. <n.len: hashTree(c, n.sons[i])
+    for i in 0..<n.len: hashTree(c, n.sons[i])
 
 proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) =
   if t == nil:
@@ -159,14 +160,15 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) =
     return
   else:
     discard
-  c &= char(t.kind)
   case t.kind
   of tyBool, tyChar, tyInt..tyUInt64:
     # no canonicalization for integral types, so that e.g. ``pid_t`` is
     # produced instead of ``NI``:
+    c &= char(t.kind)
     if t.sym != nil and {sfImportc, sfExportc} * t.sym.flags != {}:
       c.hashSym(t.sym)
   of tyObject, tyEnum:
+    c &= char(t.kind)
     if t.typeInst != nil:
       assert t.typeInst.kind == tyGenericInst
       for i in countup(1, sonsLen(t.typeInst) - 2):
@@ -199,26 +201,35 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) =
     if t.len > 0 and t.sons[0] != nil:
       hashType c, t.sons[0], flags
   of tyRef, tyPtr, tyGenericBody, tyVar:
+    c &= char(t.kind)
     c.hashType t.lastSon, flags
     if tfVarIsPtr in t.flags: c &= ".varisptr"
-  of tyFromExpr, tyFieldAccessor:
+  of tyFromExpr:
+    c &= char(t.kind)
     c.hashTree(t.n)
   of tyTuple:
+    c &= char(t.kind)
     if t.n != nil and CoType notin flags:
       assert(sonsLen(t.n) == sonsLen(t))
       for i in countup(0, sonsLen(t.n) - 1):
         assert(t.n.sons[i].kind == nkSym)
         c &= t.n.sons[i].sym.name.s
         c &= ':'
-        c.hashType(t.sons[i], flags)
+        c.hashType(t.sons[i], flags+{CoIgnoreRange})
         c &= ','
     else:
-      for i in countup(0, sonsLen(t) - 1): c.hashType t.sons[i], flags
-  of tyRange, tyStatic:
-    #if CoType notin flags:
+      for i in countup(0, sonsLen(t) - 1): c.hashType t.sons[i], flags+{CoIgnoreRange}
+  of tyRange:
+    if CoIgnoreRange notin flags:
+      c &= char(t.kind)
+      c.hashTree(t.n)
+    c.hashType(t.sons[0], flags)
+  of tyStatic:
+    c &= char(t.kind)
     c.hashTree(t.n)
     c.hashType(t.sons[0], flags)
   of tyProc:
+    c &= char(t.kind)
     c &= (if tfIterator in t.flags: "iterator " else: "proc ")
     if CoProc in flags and t.n != nil:
       let params = t.n
@@ -230,14 +241,18 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) =
         c &= ','
       c.hashType(t.sons[0], flags)
     else:
-      for i in 0.. <t.len: c.hashType(t.sons[i], flags)
+      for i in 0..<t.len: c.hashType(t.sons[i], flags)
     c &= char(t.callConv)
     if CoType notin flags:
       if tfNoSideEffect in t.flags: c &= ".noSideEffect"
       if tfThread in t.flags: c &= ".thread"
     if tfVarargs in t.flags: c &= ".varargs"
+  of tyArray:
+    c &= char(t.kind)
+    for i in 0..<t.len: c.hashType(t.sons[i], flags-{CoIgnoreRange})
   else:
-    for i in 0.. <t.len: c.hashType(t.sons[i], flags)
+    c &= char(t.kind)
+    for i in 0..<t.len: c.hashType(t.sons[i], flags)
   if tfNotNil in t.flags and CoType notin flags: c &= "not nil"
 
 when defined(debugSigHashes):

compiler/sigmatch.nim

@@ -24,7 +24,7 @@ type
 
   CandidateError* = object
     sym*: PSym
-    unmatchedVarParam*: int
+    unmatchedVarParam*, firstMismatch*: int
     diagnostics*: seq[string]
 
   CandidateErrors* = seq[CandidateError]
@@ -55,6 +55,7 @@ type
                              # a distrinct type
     typedescMatched*: bool
     isNoCall*: bool          # misused for generic type instantiations C[T]
+    mutabilityProblem*: uint8 # tyVar mismatch
     inferredTypes: seq[PType] # inferred types during the current signature
                               # matching. they will be reset if the matching
                               # is not successful. may replace the bindings
@@ -66,8 +67,9 @@ type
                               # or when the explain pragma is used. may be
                               # triggered with an idetools command in the
                               # future.
-    mutabilityProblem*: uint8 # tyVar mismatch
-    inheritancePenalty: int  # to prefer closest father object type
+    inheritancePenalty: int   # to prefer closest father object type
+    firstMismatch*: int       # position of the first type mismatch for
+                              # better error messages
 
   TTypeRelFlag* = enum
     trDontBind
@@ -180,7 +182,8 @@ proc sumGeneric(t: PType): int =
   while true:
     case t.kind
     of tyGenericInst, tyArray, tyRef, tyPtr, tyDistinct,
-        tyOpenArray, tyVarargs, tySet, tyRange, tySequence, tyGenericBody:
+        tyOpenArray, tyVarargs, tySet, tyRange, tySequence, tyGenericBody,
+        tyLent:
       t = t.lastSon
       inc result
     of tyOr:
@@ -200,14 +203,14 @@ proc sumGeneric(t: PType): int =
       inc result
     of tyGenericInvocation, tyTuple, tyProc, tyAnd:
       result += ord(t.kind in {tyGenericInvocation, tyAnd})
-      for i in 0 .. <t.len:
+      for i in 0 ..< t.len:
         if t.sons[i] != nil:
           result += t.sons[i].sumGeneric
       break
     of tyStatic:
       return t.sons[0].sumGeneric + 1
     of tyGenericParam, tyExpr, tyStmt: break
-    of tyAlias: t = t.lastSon
+    of tyAlias, tySink: t = t.lastSon
     of tyBool, tyChar, tyEnum, tyObject, tyPointer,
         tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128,
         tyUInt..tyUInt64, tyCompositeTypeClass:
@@ -220,11 +223,12 @@ proc sumGeneric(t: PType): int =
 proc complexDisambiguation(a, b: PType): int =
   # 'a' matches better if *every* argument matches better or equal than 'b'.
   var winner = 0
-  for i in 1 .. <min(a.len, b.len):
+  for i in 1 ..< min(a.len, b.len):
     let x = a.sons[i].sumGeneric
     let y = b.sons[i].sumGeneric
     #if ggDebug:
-    #  echo "came her ", typeToString(a.sons[i]), " ", typeToString(b.sons[i])
+    #echo "came herA ", typeToString(a.sons[i]), " ", x
+    #echo "came herB ", typeToString(b.sons[i]), " ", y
     if x != y:
       if winner == 0:
         if x > y: winner = 1
@@ -239,8 +243,8 @@ proc complexDisambiguation(a, b: PType): int =
   result = winner
   when false:
     var x, y: int
-    for i in 1 .. <a.len: x += a.sons[i].sumGeneric
-    for i in 1 .. <b.len: y += b.sons[i].sumGeneric
+    for i in 1 ..< a.len: x += a.sons[i].sumGeneric
+    for i in 1 ..< b.len: y += b.sons[i].sumGeneric
     result = x - y
 
 proc writeMatches*(c: TCandidate) =
@@ -275,7 +279,7 @@ proc cmpCandidates*(a, b: TCandidate): int =
 proc argTypeToString(arg: PNode; prefer: TPreferedDesc): string =
   if arg.kind in nkSymChoices:
     result = typeToString(arg[0].typ, prefer)
-    for i in 1 .. <arg.len:
+    for i in 1 ..< arg.len:
       result.add(" | ")
       result.add typeToString(arg[i].typ, prefer)
   elif arg.typ == nil:
@@ -389,7 +393,16 @@ proc isConvertibleToRange(f, a: PType): bool =
   # be less picky for tyRange, as that it is used for array indexing:
   if f.kind in {tyInt..tyInt64, tyUInt..tyUInt64} and
      a.kind in {tyInt..tyInt64, tyUInt..tyUInt64}:
-    result = true
+    case f.kind
+    of tyInt, tyInt64: result = true
+    of tyInt8: result = a.kind in {tyInt8, tyInt}
+    of tyInt16: result = a.kind in {tyInt8, tyInt16, tyInt}
+    of tyInt32: result = a.kind in {tyInt8, tyInt16, tyInt32, tyInt}
+    of tyUInt, tyUInt64: result = true
+    of tyUInt8: result = a.kind in {tyUInt8, tyUInt}
+    of tyUInt16: result = a.kind in {tyUInt8, tyUInt16, tyUInt}
+    of tyUInt32: result = a.kind in {tyUInt8, tyUInt16, tyUInt32, tyUInt}
+    else: result = false
   elif f.kind in {tyFloat..tyFloat128} and
        a.kind in {tyFloat..tyFloat128}:
     result = true
@@ -454,7 +467,7 @@ proc skipToObject(t: PType; skipped: var SkippedPtr): PType =
       inc ptrs
       skipped = skippedPtr
       r = r.lastSon
-    of tyGenericBody, tyGenericInst, tyAlias:
+    of tyGenericBody, tyGenericInst, tyAlias, tySink:
       r = r.lastSon
     else:
       break
@@ -514,7 +527,7 @@ proc allowsNil(f: PType): TTypeRelation {.inline.} =
   result = if tfNotNil notin f.flags: isSubtype else: isNone
 
 proc inconsistentVarTypes(f, a: PType): bool {.inline.} =
-  result = f.kind != a.kind and (f.kind == tyVar or a.kind == tyVar)
+  result = f.kind != a.kind and (f.kind in {tyVar, tyLent} or a.kind in {tyVar, tyLent})
 
 proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
   ## For example we have:
@@ -580,7 +593,7 @@ proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
 
     # Note: We have to do unification for the parameters before the
     # return type!
-    for i in 1 .. <f.sonsLen:
+    for i in 1 ..< f.sonsLen:
       checkParam(f.sons[i], a.sons[i])
 
     if f.sons[0] != nil:
@@ -658,7 +671,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
   var typeParams: seq[(PSym, PType)]
 
   if ff.kind == tyUserTypeClassInst:
-    for i in 1 .. <(ff.len - 1):
+    for i in 1 ..< (ff.len - 1):
       var
         typeParamName = ff.base.sons[i-1].sym.name
         typ = ff.sons[i]
@@ -879,7 +892,7 @@ proc inferStaticsInRange(c: var TCandidate,
     doInferStatic(lowerBound, upperBound.intVal + 1 - lengthOrd(concrete))
 
 template subtypeCheck() =
-  if result <= isSubrange and f.lastSon.skipTypes(abstractInst).kind in {tyRef, tyPtr, tyVar}:
+  if result <= isSubrange and f.lastSon.skipTypes(abstractInst).kind in {tyRef, tyPtr, tyVar, tyLent}:
     result = isNone
 
 proc isCovariantPtr(c: var TCandidate, f, a: PType): bool =
@@ -887,7 +900,7 @@ proc isCovariantPtr(c: var TCandidate, f, a: PType): bool =
   assert f.kind == a.kind
 
   template baseTypesCheck(lhs, rhs: PType): bool =
-    lhs.kind notin {tyPtr, tyRef, tyVar} and
+    lhs.kind notin {tyPtr, tyRef, tyVar, tyLent} and
       typeRel(c, lhs, rhs, {trNoCovariance}) == isSubtype
 
   case f.kind
@@ -902,6 +915,26 @@ proc isCovariantPtr(c: var TCandidate, f, a: PType): bool =
   else:
     return false
 
+when false:
+  proc maxNumericType(prev, candidate: PType): PType =
+    let c = candidate.skipTypes({tyRange})
+    template greater(s) =
+      if c.kind in s: result = c
+    case prev.kind
+    of tyInt: greater({tyInt64})
+    of tyInt8: greater({tyInt, tyInt16, tyInt32, tyInt64})
+    of tyInt16: greater({tyInt, tyInt32, tyInt64})
+    of tyInt32: greater({tyInt64})
+
+    of tyUInt: greater({tyUInt64})
+    of tyUInt8: greater({tyUInt, tyUInt16, tyUInt32, tyUInt64})
+    of tyUInt16: greater({tyUInt, tyUInt32, tyUInt64})
+    of tyUInt32: greater({tyUInt64})
+
+    of tyFloat32: greater({tyFloat64, tyFloat128})
+    of tyFloat64: greater({tyFloat128})
+    else: discard
+
 proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
                  flags: TTypeRelFlags = {}): TTypeRelation =
   # typeRel can be used to establish various relationships between types:
@@ -936,7 +969,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
                                      tfExplicit notin aOrig.flags
 
     aOrig = if useTypeLoweringRuleInTypeClass:
-          aOrig.skipTypes({tyTypeDesc, tyFieldAccessor})
+          aOrig.skipTypes({tyTypeDesc})
         else:
           aOrig
 
@@ -973,17 +1006,17 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
   template doBind: bool = trDontBind notin flags
 
   # var and static arguments match regular modifier-free types
-  var a = aOrig.skipTypes({tyStatic, tyVar}).maybeSkipDistinct(c.calleeSym)
+  var a = aOrig.skipTypes({tyStatic, tyVar, tyLent}).maybeSkipDistinct(c.calleeSym)
   # XXX: Theoretically, maybeSkipDistinct could be called before we even
   # start the param matching process. This could be done in `prepareOperand`
   # for example, but unfortunately `prepareOperand` is not called in certain
   # situation when nkDotExpr are rotated to nkDotCalls
 
-  if aOrig.kind == tyAlias:
+  if aOrig.kind in {tyAlias, tySink}:
     return typeRel(c, f, lastSon(aOrig))
 
   if a.kind == tyGenericInst and
-      skipTypes(f, {tyVar}).kind notin {
+      skipTypes(f, {tyVar, tyLent}).kind notin {
         tyGenericBody, tyGenericInvocation,
         tyGenericInst, tyGenericParam} + tyTypeClasses:
     return typeRel(c, f, lastSon(a))
@@ -1047,9 +1080,10 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
            else: isNone
 
   of tyUserTypeClass, tyUserTypeClassInst:
-    if c.c.matchedConcept != nil:
+    if c.c.matchedConcept != nil and c.c.matchedConcept.depth <= 4:
       # consider this: 'var g: Node' *within* a concept where 'Node'
       # is a concept too (tgraph)
+      inc c.c.matchedConcept.depth
       let x = typeRel(c, a, f, flags + {trDontBind})
       if x >= isGeneric:
         return isGeneric
@@ -1094,8 +1128,8 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
   of tyFloat32:  result = handleFloatRange(f, a)
   of tyFloat64:  result = handleFloatRange(f, a)
   of tyFloat128: result = handleFloatRange(f, a)
-  of tyVar:
-    if aOrig.kind == tyVar: result = typeRel(c, f.base, aOrig.base)
+  of tyVar, tyLent:
+    if aOrig.kind == f.kind: result = typeRel(c, f.base, aOrig.base)
     else: result = typeRel(c, f.base, aOrig, flags + {trNoCovariance})
     subtypeCheck()
   of tyArray:
@@ -1112,7 +1146,12 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
           fRange = prev
       let ff = f.sons[1].skipTypes({tyTypeDesc})
       let aa = a.sons[1].skipTypes({tyTypeDesc})
-      result = typeRel(c, ff, aa)
+
+      if f.sons[0].kind != tyGenericParam and aa.kind == tyEmpty:
+        result = isGeneric
+      else:
+        result = typeRel(c, ff, aa)
+
       if result < isGeneric:
         if nimEnableCovariance and
            trNoCovariance notin flags and
@@ -1288,18 +1327,19 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
     of tyString: result = isConvertible
     of tyPtr:
       # ptr[Tag, char] is not convertible to 'cstring' for now:
-      if a.len == 1 and a.sons[0].kind == tyChar: result = isConvertible
-    of tyArray:
-      if (firstOrd(a.sons[0]) == 0) and
-          (skipTypes(a.sons[0], {tyRange}).kind in {tyInt..tyInt64}) and
-          (a.sons[1].kind == tyChar):
-        result = isConvertible
+      if a.len == 1:
+        let pointsTo = a.sons[0].skipTypes(abstractInst)
+        if pointsTo.kind == tyChar: result = isConvertible
+        elif pointsTo.kind == tyArray and firstOrd(pointsTo.sons[0]) == 0 and
+            skipTypes(pointsTo.sons[0], {tyRange}).kind in {tyInt..tyInt64} and
+            pointsTo.sons[1].kind == tyChar:
+          result = isConvertible
     else: discard
 
   of tyEmpty, tyVoid:
     if a.kind == f.kind: result = isEqual
 
-  of tyAlias:
+  of tyAlias, tySink:
     result = typeRel(c, lastSon(f), a)
 
   of tyGenericInst:
@@ -1373,7 +1413,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
     # XXX: This is very hacky. It should be moved back into liftTypeParam
     if x.kind in {tyGenericInst, tyArray} and
        c.calleeSym != nil and
-       c.calleeSym.kind == skProc:
+       c.calleeSym.kind in {skProc, skFunc} and c.call != nil:
       let inst = prepareMetatypeForSigmatch(c.c, c.bindings, c.call.info, f)
       return typeRel(c, inst, a)
 
@@ -1450,8 +1490,13 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
   of tyOr:
     considerPreviousT:
       result = isNone
+      let oldInheritancePenalty = c.inheritancePenalty
+      var maxInheritance = 0
       for branch in f.sons:
+        c.inheritancePenalty = 0
         let x = typeRel(c, branch, aOrig)
+        maxInheritance = max(maxInheritance, c.inheritancePenalty)
+
         # 'or' implies maximum matching result:
         if x > result: result = x
       if result >= isSubtype:
@@ -1459,6 +1504,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
         bindingRet result
       else:
         result = isNone
+      c.inheritancePenalty = oldInheritancePenalty + maxInheritance
 
   of tyNot:
     considerPreviousT:
@@ -1479,7 +1525,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
     considerPreviousT:
       let targetKind = f.sons[0].kind
       let effectiveArgType = a.skipTypes({tyRange, tyGenericInst,
-                                          tyBuiltInTypeClass, tyAlias})
+                                          tyBuiltInTypeClass, tyAlias, tySink})
       let typeClassMatches = targetKind == effectiveArgType.kind and
                              not effectiveArgType.isEmptyContainer
       if typeClassMatches or
@@ -1549,11 +1595,19 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
           result = isNone
       else:
         if f.sonsLen > 0 and f.sons[0].kind != tyNone:
+          let oldInheritancePenalty = c.inheritancePenalty
           result = typeRel(c, f.lastSon, a, flags + {trDontBind})
           if doBind and result notin {isNone, isGeneric}:
             let concrete = concreteType(c, a)
             if concrete == nil: return isNone
             put(c, f, concrete)
+          # bug #6526
+          if result in {isEqual, isSubtype}:
+            # 'T: Class' is a *better* match than just 'T'
+            # but 'T: Subclass' is even better:
+            c.inheritancePenalty = oldInheritancePenalty - c.inheritancePenalty -
+                                  100 * ord(result == isEqual)
+            result = isGeneric
         else:
           result = isGeneric
 
@@ -1575,9 +1629,18 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
     elif x.kind == tyGenericParam:
       result = isGeneric
     else:
+      # Special type binding rule for numeric types.
+      # See section "Generic type inference for numeric types" of the
+      # manual for further details:
+      when false:
+        let rebinding = maxNumericType(x.skipTypes({tyRange}), a)
+        if rebinding != nil:
+          put(c, f, rebinding)
+          result = isGeneric
+        else:
+          discard
       result = typeRel(c, x, a) # check if it fits
       if result > isGeneric: result = isGeneric
-
   of tyStatic:
     let prev = PType(idTableGet(c.bindings, f))
     if prev == nil:
@@ -1587,7 +1650,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
           if not exprStructuralEquivalent(f.n, aOrig.n):
             result = isNone
         if result != isNone: put(c, f, aOrig)
-      elif aOrig.n != nil:
+      elif aOrig.n != nil and aOrig.n.typ != nil:
         result = typeRel(c, f.lastSon, aOrig.n.typ)
         if result != isNone:
           var boundType = newTypeWithSons(c.c, tyStatic, @[aOrig.n.typ])
@@ -1806,8 +1869,11 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType,
 
   var r = typeRel(m, f, a)
 
+  # This special typing rule for macros and templates is not documented
+  # anywhere and breaks symmetry. It's hard to get rid of though, my
+  # custom seqs example fails to compile without this:
   if r != isNone and m.calleeSym != nil and
-     m.calleeSym.kind in {skMacro, skTemplate}:
+    m.calleeSym.kind in {skMacro, skTemplate}:
     # XXX: duplicating this is ugly, but we cannot (!) move this
     # directly into typeRel using return-like templates
     incMatches(m, r)
@@ -1832,7 +1898,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType,
       bothMetaCounter < 100:
     lastBindingsLength = m.bindings.counter
     inc(bothMetaCounter)
-    if arg.kind in {nkProcDef, nkIteratorDef} + nkLambdaKinds:
+    if arg.kind in {nkProcDef, nkFuncDef, nkIteratorDef} + nkLambdaKinds:
       result = c.semInferredLambda(c, m.bindings, arg)
     elif arg.kind != nkSym:
       return nil
@@ -1865,7 +1931,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType,
     else:
       result = implicitConv(nkHiddenStdConv, f, arg, m, c)
   of isInferred, isInferredConvertible:
-    if arg.kind in {nkProcDef, nkIteratorDef} + nkLambdaKinds:
+    if arg.kind in {nkProcDef, nkFuncDef, nkIteratorDef} + nkLambdaKinds:
       result = c.semInferredLambda(c, m.bindings, arg)
     elif arg.kind != nkSym:
       return nil
@@ -1952,7 +2018,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
     z.calleeSym = m.calleeSym
     var best = -1
     for i in countup(0, sonsLen(arg) - 1):
-      if arg.sons[i].sym.kind in {skProc, skMethod, skConverter, skIterator}:
+      if arg.sons[i].sym.kind in {skProc, skFunc, skMethod, skConverter, skIterator}:
         copyCandidate(z, m)
         z.callee = arg.sons[i].typ
         if tfUnresolved in z.callee.flags: continue
@@ -2042,7 +2108,8 @@ proc prepareNamedParam(a: PNode) =
 proc arrayConstr(c: PContext, n: PNode): PType =
   result = newTypeS(tyArray, c)
   rawAddSon(result, makeRangeType(c, 0, 0, n.info))
-  addSonSkipIntLit(result, skipTypes(n.typ, {tyGenericInst, tyVar, tyOrdinal}))
+  addSonSkipIntLit(result, skipTypes(n.typ,
+      {tyGenericInst, tyVar, tyLent, tyOrdinal}))
 
 proc arrayConstr(c: PContext, info: TLineInfo): PType =
   result = newTypeS(tyArray, c)
@@ -2192,6 +2259,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
                                     n.sons[a], nOrig.sons[a])
           if arg == nil:
             m.state = csNoMatch
+            m.firstMismatch = f
             return
           if m.baseTypeMatch:
             #assert(container == nil)
@@ -2217,7 +2285,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
 proc semFinishOperands*(c: PContext, n: PNode) =
   # this needs to be called to ensure that after overloading resolution every
   # argument has been sem'checked:
-  for i in 1 .. <n.len:
+  for i in 1 ..< n.len:
     n.sons[i] = prepareOperand(c, n.sons[i])
 
 proc partialMatch*(c: PContext, n, nOrig: PNode, m: var TCandidate) =
@@ -2246,6 +2314,7 @@ proc matches*(c: PContext, n, nOrig: PNode, m: var TCandidate) =
         else:
           # no default value
           m.state = csNoMatch
+          m.firstMismatch = f
           break
       else:
         # use default value:
@@ -2287,7 +2356,8 @@ proc instTypeBoundOp*(c: PContext; dc: PSym; t: PType; info: TLineInfo;
     localError(info, errGenerated, "cannot instantiate '" & dc.name.s & "'")
   else:
     result = c.semGenerateInstance(c, dc, m.bindings, info)
-    assert sfFromGeneric in result.flags
+    if op == attachedDeepCopy:
+      assert sfFromGeneric in result.flags
 
 include suggest
 

compiler/suggest.nim

@@ -33,6 +33,7 @@
 # included from sigmatch.nim
 
 import algorithm, prefixmatches
+from wordrecg import wDeprecated
 
 when defined(nimsuggest):
   import passes, tables # importer
@@ -240,7 +241,7 @@ proc suggestField(c: PContext, s: PSym; f: PNode; info: TLineInfo; outputs: var
 
 proc getQuality(s: PSym): range[0..100] =
   if s.typ != nil and s.typ.len > 1:
-    var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyAlias})
+    var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyLent, tyAlias, tySink})
     if exp.kind == tyVarargs: exp = elemType(exp)
     if exp.kind in {tyExpr, tyStmt, tyGenericParam, tyAnything}: return 50
   return 100
@@ -309,7 +310,7 @@ proc typeFits(c: PContext, s: PSym, firstArg: PType): bool {.inline.} =
     let m = s.getModule()
     if m != nil and sfSystemModule in m.flags:
       if s.kind == skType: return
-      var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyAlias})
+      var exp = s.typ.sons[1].skipTypes({tyGenericInst, tyVar, tyLent, tyAlias, tySink})
       if exp.kind == tyVarargs: exp = elemType(exp)
       if exp.kind in {tyExpr, tyStmt, tyGenericParam, tyAnything}: return
     result = sigmatch.argtypeMatches(c, s.typ.sons[1], firstArg)
@@ -378,8 +379,8 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions)
       t = t.sons[0]
     suggestOperations(c, n, field, typ, outputs)
   else:
-    let orig = typ # skipTypes(typ, {tyGenericInst, tyAlias})
-    typ = skipTypes(typ, {tyGenericInst, tyVar, tyPtr, tyRef, tyAlias})
+    let orig = typ # skipTypes(typ, {tyGenericInst, tyAlias, tySink})
+    typ = skipTypes(typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink})
     if typ.kind == tyObject:
       var t = typ
       while true:
@@ -479,12 +480,23 @@ proc suggestSym*(info: TLineInfo; s: PSym; usageSym: var PSym; isDecl=true) {.in
         isDecl:
       suggestResult(symToSuggest(s, isLocal=false, ideOutline, info, 100, PrefixMatch.None, false, 0))
 
+proc warnAboutDeprecated(info: TLineInfo; s: PSym) =
+  if s.kind in routineKinds:
+    let n = s.ast[pragmasPos]
+    if n.kind != nkEmpty:
+      for it in n:
+        if whichPragma(it) == wDeprecated and it.safeLen == 2 and
+            it[1].kind in {nkStrLit..nkTripleStrLit}:
+          message(info, warnDeprecated, it[1].strVal & "; " & s.name.s)
+          return
+  message(info, warnDeprecated, s.name.s)
+
 proc markUsed(info: TLineInfo; s: PSym; usageSym: var PSym) =
   incl(s.flags, sfUsed)
   if s.kind == skEnumField and s.owner != nil:
     incl(s.owner.flags, sfUsed)
   if {sfDeprecated, sfError} * s.flags != {}:
-    if sfDeprecated in s.flags: message(info, warnDeprecated, s.name.s)
+    if sfDeprecated in s.flags: warnAboutDeprecated(info, s)
     if sfError in s.flags: localError(info, errWrongSymbolX, s.name.s)
   when defined(nimsuggest):
     suggestSym(info, s, usageSym, false)

compiler/transf.nim

@@ -21,9 +21,7 @@
 import
   intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os,
   idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread,
-  lambdalifting, sempass2, lowerings, lookups
-
-# implementation
+  lambdalifting, sempass2, lowerings, lookups, destroyer, liftlocals
 
 type
   PTransNode* = distinct PNode
@@ -45,7 +43,7 @@ type
     inlining: int            # > 0 if we are in inlining context (copy vars)
     nestedProcs: int         # > 0 if we are in a nested proc
     contSyms, breakSyms: seq[PSym]  # to transform 'continue' and 'break'
-    deferDetected, tooEarly: bool
+    deferDetected, tooEarly, needsDestroyPass: bool
   PTransf = ref TTransfContext
 
 proc newTransNode(a: PNode): PTransNode {.inline.} =
@@ -95,7 +93,7 @@ proc getCurrOwner(c: PTransf): PSym =
 
 proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode =
   let r = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info)
-  r.typ = typ #skipTypes(typ, {tyGenericInst, tyAlias})
+  r.typ = typ #skipTypes(typ, {tyGenericInst, tyAlias, tySink})
   incl(r.flags, sfFromGeneric)
   let owner = getCurrOwner(c)
   if owner.isIterator and not c.tooEarly:
@@ -121,7 +119,7 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
     if s.kind == skIterator:
       if c.tooEarly: return n
       else: return liftIterSym(n, getCurrOwner(c))
-    elif s.kind in {skProc, skConverter, skMethod} and not c.tooEarly:
+    elif s.kind in {skProc, skFunc, skConverter, skMethod} and not c.tooEarly:
       # top level .closure procs are still somewhat supported for 'Nake':
       return makeClosure(s, nil, n.info)
   #elif n.sym.kind in {skVar, skLet} and n.sym.typ.callConv == ccClosure:
@@ -233,7 +231,7 @@ proc freshLabels(c: PTransf, n: PNode; symMap: var TIdTable) =
     let x = PSym(idTableGet(symMap, n.sym))
     if x != nil: n.sym = x
   else:
-    for i in 0 .. <safeLen(n): freshLabels(c, n.sons[i], symMap)
+    for i in 0 ..< safeLen(n): freshLabels(c, n.sons[i], symMap)
 
 proc transformBlock(c: PTransf, n: PNode): PTransNode =
   var labl: PSym
@@ -277,7 +275,7 @@ proc transformWhile(c: PTransf; n: PNode): PTransNode =
     var body = newTransNode(n)
     for i in 0..n.len-2:
       body[i] = transform(c, n.sons[i])
-    body[<n.len] = transformLoopBody(c, n.sons[<n.len])
+    body[n.len-1] = transformLoopBody(c, n.sons[n.len-1])
     result[1] = body
     discard c.breakSyms.pop
 
@@ -333,7 +331,7 @@ proc transformYield(c: PTransf, n: PNode): PTransNode =
   # c.transCon.forStmt.len == 3 means that there is one for loop variable
   # and thus no tuple unpacking:
   if e.typ.isNil: return result # can happen in nimsuggest for unknown reasons
-  if skipTypes(e.typ, {tyGenericInst, tyAlias}).kind == tyTuple and
+  if skipTypes(e.typ, {tyGenericInst, tyAlias, tySink}).kind == tyTuple and
       c.transCon.forStmt.len != 3:
     e = skipConv(e)
     if e.kind == nkPar:
@@ -367,16 +365,22 @@ proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode =
       # addr ( nkConv ( deref ( x ) ) ) --> nkConv(x)
       n.sons[0].sons[0] = m.sons[0]
       result = PTransNode(n.sons[0])
+      if n.typ.skipTypes(abstractVar).kind != tyOpenArray:
+        PNode(result).typ = n.typ
   of nkHiddenStdConv, nkHiddenSubConv, nkConv:
     var m = n.sons[0].sons[1]
     if m.kind == a or m.kind == b:
       # addr ( nkConv ( deref ( x ) ) ) --> nkConv(x)
       n.sons[0].sons[1] = m.sons[0]
       result = PTransNode(n.sons[0])
+      if n.typ.skipTypes(abstractVar).kind != tyOpenArray:
+        PNode(result).typ = n.typ
   else:
     if n.sons[0].kind == a or n.sons[0].kind == b:
       # addr ( deref ( x )) --> x
       result = PTransNode(n.sons[0].sons[0])
+      if n.typ.skipTypes(abstractVar).kind != tyOpenArray:
+        PNode(result).typ = n.typ
 
 proc generateThunk(prc: PNode, dest: PType): PNode =
   ## Converts 'prc' into '(thunk, nil)' so that it's compatible with
@@ -502,7 +506,7 @@ proc putArgInto(arg: PNode, formal: PType): TPutArgInto =
       if putArgInto(arg.sons[i], formal) != paDirectMapping: return
     result = paDirectMapping
   else:
-    if skipTypes(formal, abstractInst).kind == tyVar: result = paVarAsgn
+    if skipTypes(formal, abstractInst).kind in {tyVar, tyLent}: result = paVarAsgn
     else: result = paFastAsgn
 
 proc findWrongOwners(c: PTransf, n: PNode) =
@@ -512,7 +516,7 @@ proc findWrongOwners(c: PTransf, n: PNode) =
       internalError(x.info, "bah " & x.sym.name.s & " " &
         x.sym.owner.name.s & " " & getCurrOwner(c).name.s)
   else:
-    for i in 0 .. <safeLen(n): findWrongOwners(c, n.sons[i])
+    for i in 0 ..< safeLen(n): findWrongOwners(c, n.sons[i])
 
 proc transformFor(c: PTransf, n: PNode): PTransNode =
   # generate access statements for the parameters (unless they are constant)
@@ -642,7 +646,7 @@ proc transformArrayAccess(c: PTransf, n: PNode): PTransNode =
     result = n.PTransNode
   else:
     result = newTransNode(n)
-    for i in 0 .. < n.len:
+    for i in 0 ..< n.len:
       result[i] = transform(c, skipConv(n.sons[i]))
 
 proc getMergeOp(n: PNode): PSym =
@@ -689,7 +693,7 @@ proc transformCall(c: PTransf, n: PNode): PTransNode =
           inc(j)
       add(result, a.PTransNode)
     if len(result) == 2: result = result[1]
-  elif magic in {mNBindSym, mTypeOf}:
+  elif magic in {mNBindSym, mTypeOf, mRunnableExamples}:
     # for bindSym(myconst) we MUST NOT perform constant folding:
     result = n.PTransNode
   elif magic == mProcCall:
@@ -712,7 +716,7 @@ proc transformExceptBranch(c: PTransf, n: PNode): PTransNode =
   result = transformSons(c, n)
   if n[0].isInfixAs():
     let excTypeNode = n[0][1]
-    let actions = newTransNode(nkStmtList, n[1].info, 2)
+    let actions = newTransNode(nkStmtListExpr, n[1], 2)
     # Generating `let exc = (excType)(getCurrentException())`
     # -> getCurrentException()
     let excCall = PTransNode(callCodegenProc("getCurrentException", ast.emptyNode))
@@ -746,7 +750,7 @@ proc dontInlineConstant(orig, cnst: PNode): bool {.inline.} =
 
 proc commonOptimizations*(c: PSym, n: PNode): PNode =
   result = n
-  for i in 0 .. < n.safeLen:
+  for i in 0 ..< n.safeLen:
     result.sons[i] = commonOptimizations(c, n.sons[i])
   var op = getMergeOp(n)
   if (op != nil) and (op.magic != mNone) and (sonsLen(n) >= 3):
@@ -782,11 +786,12 @@ proc transform(c: PTransf, n: PNode): PTransNode =
                   nkBlockStmt, nkBlockExpr}:
       oldDeferAnchor = c.deferAnchor
       c.deferAnchor = n
-
+  if n.typ != nil and tfHasAsgn in n.typ.flags:
+    c.needsDestroyPass = true
   case n.kind
   of nkSym:
     result = transformSym(c, n)
-  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit:
+  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom:
     # nothing to be done for leaves:
     result = PTransNode(n)
   of nkBracketExpr: result = transformArrayAccess(c, n)
@@ -909,7 +914,7 @@ proc processTransf(c: PTransf, n: PNode, owner: PSym): PNode =
   # Note: For interactive mode we cannot call 'passes.skipCodegen' and skip
   # this step! We have to rely that the semantic pass transforms too errornous
   # nodes into an empty node.
-  if c.fromCache or nfTransf in n.flags: return n
+  if c.rd != nil or nfTransf in n.flags: return n
   pushTransCon(c, newTransCon(owner))
   result = PNode(transform(c, n))
   popTransCon(c)
@@ -972,9 +977,12 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
     result = processTransf(c, result, prc)
     liftDefer(c, result)
     #result = liftLambdas(prc, result)
-    incl(result.flags, nfTransf)
     when useEffectSystem: trackProc(prc, result)
-    #if prc.name.s == "testbody":
+    result = liftLocalsIfRequested(prc, result)
+    if c.needsDestroyPass and newDestructors:
+      result = injectDestructorCalls(prc, result)
+    incl(result.flags, nfTransf)
+      #if prc.name.s == "testbody":
     #  echo renderTree(result)
 
 proc transformStmt*(module: PSym, n: PNode): PNode =
@@ -985,10 +993,12 @@ proc transformStmt*(module: PSym, n: PNode): PNode =
     result = processTransf(c, n, module)
     liftDefer(c, result)
     #result = liftLambdasForTopLevel(module, result)
-    incl(result.flags, nfTransf)
     when useEffectSystem: trackTopLevelStmt(module, result)
     #if n.info ?? "temp.nim":
     #  echo renderTree(result, {renderIds})
+    if c.needsDestroyPass and newDestructors:
+      result = injectDestructorCalls(module, result)
+    incl(result.flags, nfTransf)
 
 proc transformExpr*(module: PSym, n: PNode): PNode =
   if nfTransf in n.flags:
@@ -997,4 +1007,6 @@ proc transformExpr*(module: PSym, n: PNode): PNode =
     var c = openTransf(module, "")
     result = processTransf(c, n, module)
     liftDefer(c, result)
+    if c.needsDestroyPass and newDestructors:
+      result = injectDestructorCalls(module, result)
     incl(result.flags, nfTransf)

compiler/trees.nim

@@ -98,11 +98,11 @@ proc isDeepConstExpr*(n: PNode): bool =
   of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv:
     result = isDeepConstExpr(n.sons[1])
   of nkCurly, nkBracket, nkPar, nkObjConstr, nkClosure, nkRange:
-    for i in ord(n.kind == nkObjConstr) .. <n.len:
+    for i in ord(n.kind == nkObjConstr) ..< n.len:
       if not isDeepConstExpr(n.sons[i]): return false
     if n.typ.isNil: result = true
     else:
-      let t = n.typ.skipTypes({tyGenericInst, tyDistinct, tyAlias})
+      let t = n.typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink})
       if t.kind in {tyRef, tyPtr}: return false
       if t.kind != tyObject or not isCaseObj(t.n):
         result = true
@@ -118,7 +118,7 @@ proc isRange*(n: PNode): bool {.inline.} =
       result = true
 
 proc whichPragma*(n: PNode): TSpecialWord =
-  let key = if n.kind == nkExprColonExpr: n.sons[0] else: n
+  let key = if n.kind in nkPragmaCallKinds and n.len > 0: n.sons[0] else: n
   if key.kind == nkIdent: result = whichKeyword(key.ident)
 
 proc unnestStmts(n, result: PNode) =

compiler/types.nim

@@ -12,12 +12,10 @@
 import
   intsets, ast, astalgo, trees, msgs, strutils, platform, renderer
 
-proc firstOrd*(t: PType): BiggestInt
-proc lastOrd*(t: PType): BiggestInt
-proc lengthOrd*(t: PType): BiggestInt
 type
   TPreferedDesc* = enum
-    preferName, preferDesc, preferExported, preferModuleInfo, preferGenericArg
+    preferName, preferDesc, preferExported, preferModuleInfo, preferGenericArg,
+    preferTypeName
 
 proc typeToString*(typ: PType; prefer: TPreferedDesc = preferName): string
 template `$`*(typ: PType): string = typeToString(typ)
@@ -48,39 +46,26 @@ type
 
 proc equalParams*(a, b: PNode): TParamsEquality
   # returns whether the parameter lists of the procs a, b are exactly the same
-proc isOrdinalType*(t: PType): bool
-proc enumHasHoles*(t: PType): bool
 
 const
   # TODO: Remove tyTypeDesc from each abstractX and (where necessary)
   # replace with typedescX
   abstractPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyDistinct, tyOrdinal,
-                   tyTypeDesc, tyAlias, tyInferred}
+                   tyTypeDesc, tyAlias, tyInferred, tySink, tyLent}
   abstractVar* = {tyVar, tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc,
-                  tyAlias, tyInferred}
+                  tyAlias, tyInferred, tySink, tyLent}
   abstractRange* = {tyGenericInst, tyRange, tyDistinct, tyOrdinal, tyTypeDesc,
-                    tyAlias, tyInferred}
+                    tyAlias, tyInferred, tySink}
   abstractVarRange* = {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal,
-                       tyTypeDesc, tyAlias, tyInferred}
+                       tyTypeDesc, tyAlias, tyInferred, tySink}
   abstractInst* = {tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias,
-                   tyInferred}
+                   tyInferred, tySink}
   skipPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyTypeDesc, tyAlias,
-               tyInferred}
+               tyInferred, tySink, tyLent}
   # typedescX is used if we're sure tyTypeDesc should be included (or skipped)
   typedescPtrs* = abstractPtrs + {tyTypeDesc}
   typedescInst* = abstractInst + {tyTypeDesc}
 
-proc containsObject*(t: PType): bool
-proc containsGarbageCollectedRef*(typ: PType): bool
-proc containsHiddenPointer*(typ: PType): bool
-proc canFormAcycle*(typ: PType): bool
-proc isCompatibleToCString*(a: PType): bool
-proc getOrdValue*(n: PNode): BiggestInt
-proc computeSize*(typ: PType): BiggestInt
-proc getSize*(typ: PType): BiggestInt
-proc isPureObject*(typ: PType): bool
-proc invalidGenericInst*(f: PType): bool
-  # for debugging
 type
   TTypeFieldResult* = enum
     frNone,                   # type has no object type field
@@ -92,16 +77,16 @@ proc analyseObjectWithTypeField*(t: PType): TTypeFieldResult
   # made or intializing of the type field suffices or if there is no type field
   # at all in this type.
 
-proc invalidGenericInst(f: PType): bool =
+proc invalidGenericInst*(f: PType): bool =
   result = f.kind == tyGenericInst and lastSon(f) == nil
 
-proc isPureObject(typ: PType): bool =
+proc isPureObject*(typ: PType): bool =
   var t = typ
   while t.kind == tyObject and t.sons[0] != nil:
     t = t.sons[0].skipTypes(skipPtrs)
   result = t.sym != nil and sfPure in t.sym.flags
 
-proc getOrdValue(n: PNode): BiggestInt =
+proc getOrdValue*(n: PNode): BiggestInt =
   case n.kind
   of nkCharLit..nkUInt64Lit: result = n.intVal
   of nkNilLit: result = 0
@@ -116,14 +101,6 @@ proc isIntLit*(t: PType): bool {.inline.} =
 proc isFloatLit*(t: PType): bool {.inline.} =
   result = t.kind == tyFloat and t.n != nil and t.n.kind == nkFloatLit
 
-proc isCompatibleToCString(a: PType): bool =
-  if a.kind == tyArray:
-    if (firstOrd(a.sons[0]) == 0) and
-        (skipTypes(a.sons[0], {tyRange, tyGenericInst, tyAlias}).kind in
-            {tyInt..tyInt64, tyUInt..tyUInt64}) and
-        (a.sons[1].kind == tyChar):
-      result = true
-
 proc getProcHeader*(sym: PSym; prefer: TPreferedDesc = preferName): string =
   result = sym.owner.name.s & '.' & sym.name.s & '('
   var n = sym.typ.n
@@ -151,7 +128,7 @@ proc elemType*(t: PType): PType =
   else: result = t.lastSon
   assert(result != nil)
 
-proc isOrdinalType(t: PType): bool =
+proc isOrdinalType*(t: PType): bool =
   assert(t != nil)
   const
     # caution: uint, uint64 are no ordinal types!
@@ -159,7 +136,7 @@ proc isOrdinalType(t: PType): bool =
     parentKinds = {tyRange, tyOrdinal, tyGenericInst, tyAlias, tyDistinct}
   t.kind in baseKinds or (t.kind in parentKinds and isOrdinalType(t.sons[0]))
 
-proc enumHasHoles(t: PType): bool =
+proc enumHasHoles*(t: PType): bool =
   var b = t
   while b.kind in {tyRange, tyGenericInst, tyAlias}: b = b.sons[0]
   result = b.kind == tyEnum and tfEnumHasHoles in b.flags
@@ -252,7 +229,7 @@ proc searchTypeFor(t: PType, predicate: TTypePredicate): bool =
 proc isObjectPredicate(t: PType): bool =
   result = t.kind == tyObject
 
-proc containsObject(t: PType): bool =
+proc containsObject*(t: PType): bool =
   result = searchTypeFor(t, isObjectPredicate)
 
 proc isObjectWithTypeFieldPredicate(t: PType): bool =
@@ -297,7 +274,7 @@ proc isGCRef(t: PType): bool =
   result = t.kind in GcTypeKinds or
     (t.kind == tyProc and t.callConv == ccClosure)
 
-proc containsGarbageCollectedRef(typ: PType): bool =
+proc containsGarbageCollectedRef*(typ: PType): bool =
   # returns true if typ contains a reference, sequence or string (all the
   # things that are garbage-collected)
   result = searchTypeFor(typ, isGCRef)
@@ -312,7 +289,7 @@ proc containsTyRef*(typ: PType): bool =
 proc isHiddenPointer(t: PType): bool =
   result = t.kind in {tyString, tySequence}
 
-proc containsHiddenPointer(typ: PType): bool =
+proc containsHiddenPointer*(typ: PType): bool =
   # returns true if typ contains a string, table or sequence (all the things
   # that need to be copied deeply)
   result = searchTypeFor(typ, isHiddenPointer)
@@ -355,7 +332,7 @@ proc canFormAcycleAux(marker: var IntSet, typ: PType, startId: int): bool =
   of tyProc: result = typ.callConv == ccClosure
   else: discard
 
-proc canFormAcycle(typ: PType): bool =
+proc canFormAcycle*(typ: PType): bool =
   var marker = initIntSet()
   result = canFormAcycleAux(marker, typ, typ.id)
 
@@ -411,14 +388,14 @@ const
     "int", "int8", "int16", "int32", "int64",
     "float", "float32", "float64", "float128",
     "uint", "uint8", "uint16", "uint32", "uint64",
-    "unused0", "unused1",
-    "unused2", "varargs[$1]", "unused", "Error Type",
+    "opt", "sink",
+    "lent", "varargs[$1]", "unused", "Error Type",
     "BuiltInTypeClass", "UserTypeClass",
     "UserTypeClassInst", "CompositeTypeClass", "inferred",
     "and", "or", "not", "any", "static", "TypeFromExpr", "FieldAccessor",
     "void"]
 
-const preferToResolveSymbols = {preferName, preferModuleInfo, preferGenericArg}
+const preferToResolveSymbols = {preferName, preferTypeName, preferModuleInfo, preferGenericArg}
 
 template bindConcreteTypeToUserTypeClass*(tc, concrete: PType) =
   tc.sons.safeAdd concrete
@@ -444,7 +421,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
        sfAnon notin t.sym.flags:
     if t.kind == tyInt and isIntLit(t):
       result = t.sym.name.s & " literal(" & $t.n.intVal & ")"
-    elif prefer == preferName or t.sym.owner.isNil:
+    elif prefer in {preferName, preferTypeName} or t.sym.owner.isNil:
       result = t.sym.name.s
       if t.kind == tyGenericParam and t.sons != nil and t.sonsLen > 0:
         result.add ": "
@@ -521,7 +498,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
   of tyExpr:
     internalAssert t.len == 0
     result = "untyped"
-  of tyFromExpr, tyFieldAccessor:
+  of tyFromExpr:
     result = renderTree(t.n)
   of tyArray:
     if t.sons[0].kind == tyRange:
@@ -532,6 +509,8 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
           typeToString(t.sons[1]) & ']'
   of tySequence:
     result = "seq[" & typeToString(t.sons[0]) & ']'
+  of tyOpt:
+    result = "opt[" & typeToString(t.sons[0]) & ']'
   of tyOrdinal:
     result = "ordinal[" & typeToString(t.sons[0]) & ']'
   of tySet:
@@ -540,7 +519,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
     result = "openarray[" & typeToString(t.sons[0]) & ']'
   of tyDistinct:
     result = "distinct " & typeToString(t.sons[0],
-      if prefer == preferModuleInfo: preferModuleInfo else: preferName)
+      if prefer == preferModuleInfo: preferModuleInfo else: preferTypeName)
   of tyTuple:
     # we iterate over t.sons here, because t.n may be nil
     if t.n != nil:
@@ -554,12 +533,13 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
     elif sonsLen(t) == 0:
       result = "tuple[]"
     else:
-      result = "("
+      if prefer == preferTypeName: result = "("
+      else: result = "tuple of ("
       for i in countup(0, sonsLen(t) - 1):
         add(result, typeToString(t.sons[i]))
         if i < sonsLen(t) - 1: add(result, ", ")
       add(result, ')')
-  of tyPtr, tyRef, tyVar:
+  of tyPtr, tyRef, tyVar, tyLent:
     result = typeToStr[t.kind]
     if t.len >= 2:
       setLen(result, result.len-1)
@@ -587,7 +567,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
       add(result, typeToString(t.sons[i]))
       if i < sonsLen(t) - 1: add(result, ", ")
     add(result, ')')
-    if t.sons[0] != nil: add(result, ": " & typeToString(t.sons[0]))
+    if t.len > 0 and t.sons[0] != nil: add(result, ": " & typeToString(t.sons[0]))
     var prag = if t.callConv == ccDefault: "" else: CallingConvToStr[t.callConv]
     if tfNoSideEffect in t.flags:
       addSep(prag)
@@ -601,11 +581,13 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
     if len(prag) != 0: add(result, "{." & prag & ".}")
   of tyVarargs:
     result = typeToStr[t.kind] % typeToString(t.sons[0])
+  of tySink:
+    result = "sink " & typeToString(t.sons[0])
   else:
     result = typeToStr[t.kind]
   result.addTypeFlags(t)
 
-proc firstOrd(t: PType): BiggestInt =
+proc firstOrd*(t: PType): BiggestInt =
   case t.kind
   of tyBool, tyChar, tySequence, tyOpenArray, tyString, tyVarargs, tyProxy:
     result = 0
@@ -630,7 +612,7 @@ proc firstOrd(t: PType): BiggestInt =
     else:
       assert(t.n.sons[0].kind == nkSym)
       result = t.n.sons[0].sym.position
-  of tyGenericInst, tyDistinct, tyTypeDesc, tyFieldAccessor, tyAlias:
+  of tyGenericInst, tyDistinct, tyTypeDesc, tyAlias:
     result = firstOrd(lastSon(t))
   of tyOrdinal:
     if t.len > 0: result = firstOrd(lastSon(t))
@@ -639,7 +621,7 @@ proc firstOrd(t: PType): BiggestInt =
     internalError("invalid kind for first(" & $t.kind & ')')
     result = 0
 
-proc lastOrd(t: PType): BiggestInt =
+proc lastOrd*(t: PType; fixedUnsigned = false): BiggestInt =
   case t.kind
   of tyBool: result = 1
   of tyChar: result = 255
@@ -658,15 +640,18 @@ proc lastOrd(t: PType): BiggestInt =
   of tyInt64: result = 0x7FFFFFFFFFFFFFFF'i64
   of tyUInt:
     if platform.intSize == 4: result = 0xFFFFFFFF
+    elif fixedUnsigned: result = 0xFFFFFFFFFFFFFFFF'i64
     else: result = 0x7FFFFFFFFFFFFFFF'i64
   of tyUInt8: result = 0xFF
   of tyUInt16: result = 0xFFFF
   of tyUInt32: result = 0xFFFFFFFF
-  of tyUInt64: result = 0x7FFFFFFFFFFFFFFF'i64
+  of tyUInt64:
+    if fixedUnsigned: result = 0xFFFFFFFFFFFFFFFF'i64
+    else: result = 0x7FFFFFFFFFFFFFFF'i64
   of tyEnum:
     assert(t.n.sons[sonsLen(t.n) - 1].kind == nkSym)
     result = t.n.sons[sonsLen(t.n) - 1].sym.position
-  of tyGenericInst, tyDistinct, tyTypeDesc, tyFieldAccessor, tyAlias:
+  of tyGenericInst, tyDistinct, tyTypeDesc, tyAlias:
     result = lastOrd(lastSon(t))
   of tyProxy: result = 0
   of tyOrdinal:
@@ -676,7 +661,7 @@ proc lastOrd(t: PType): BiggestInt =
     internalError("invalid kind for last(" & $t.kind & ')')
     result = 0
 
-proc lengthOrd(t: PType): BiggestInt =
+proc lengthOrd*(t: PType): BiggestInt =
   case t.kind
   of tyInt64, tyInt32, tyInt: result = lastOrd(t)
   of tyDistinct: result = lengthOrd(t.sons[0])
@@ -764,7 +749,7 @@ proc equalParam(a, b: PSym): TParamsEquality =
 proc sameConstraints(a, b: PNode): bool =
   if isNil(a) and isNil(b): return true
   internalAssert a.len == b.len
-  for i in 1 .. <a.len:
+  for i in 1 ..< a.len:
     if not exprStructuralEquivalent(a[i].sym.constraint,
                                     b[i].sym.constraint):
       return false
@@ -988,11 +973,16 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
     if result and ExactGenericParams in c.flags:
       result = a.sym.position == b.sym.position
   of tyGenericInvocation, tyGenericBody, tySequence,
-     tyOpenArray, tySet, tyRef, tyPtr, tyVar,
-     tyArray, tyProc, tyVarargs, tyOrdinal, tyTypeClasses, tyFieldAccessor:
+     tyOpenArray, tySet, tyRef, tyPtr, tyVar, tyLent, tySink,
+     tyArray, tyProc, tyVarargs, tyOrdinal, tyTypeClasses, tyOpt:
     cycleCheck()
     if a.kind == tyUserTypeClass and a.n != nil: return a.n == b.n
-    result = sameChildrenAux(a, b, c) and sameFlags(a, b)
+    result = sameChildrenAux(a, b, c)
+    if result:
+      if IgnoreTupleFields in c.flags:
+        result = a.flags * {tfVarIsPtr} == b.flags * {tfVarIsPtr}
+      else:
+        result = sameFlags(a, b)
     if result and ExactGcSafety in c.flags:
       result = a.flags * {tfThread} == b.flags * {tfThread}
     if result and a.kind == tyProc:
@@ -1007,11 +997,12 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
     cycleCheck()
     result = sameTypeAux(a.lastSon, b.lastSon, c)
   of tyNone: result = false
-  of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("sameFlags")
+  of tyUnused, tyOptAsRef: internalError("sameFlags")
 
 proc sameBackendType*(x, y: PType): bool =
   var c = initSameTypeClosure()
   c.flags.incl IgnoreTupleFields
+  c.cmp = dcEqIgnoreDistinct
   result = sameTypeAux(x, y, c)
 
 proc compareTypes*(x, y: PType,
@@ -1070,11 +1061,12 @@ proc commonSuperclass*(a, b: PType): PType =
     y = y.sons[0]
 
 type
-  TTypeAllowedFlag = enum
+  TTypeAllowedFlag* = enum
     taField,
-    taHeap
+    taHeap,
+    taConcept
 
-  TTypeAllowedFlags = set[TTypeAllowedFlag]
+  TTypeAllowedFlags* = set[TTypeAllowedFlag]
 
 proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
                     flags: TTypeAllowedFlags = {}): PType
@@ -1089,7 +1081,7 @@ proc typeAllowedNode(marker: var IntSet, n: PNode, kind: TSymKind,
       of nkNone..nkNilLit:
         discard
       else:
-        if n.kind == nkRecCase and kind in {skProc, skConst}:
+        if n.kind == nkRecCase and kind in {skProc, skFunc, skConst}:
           return n[0].typ
         for i in countup(0, sonsLen(n) - 1):
           let it = n.sons[i]
@@ -1107,7 +1099,7 @@ proc matchType*(a: PType, pattern: openArray[tuple[k:TTypeKind, i:int]],
 
 proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
                     flags: TTypeAllowedFlags = {}): PType =
-  assert(kind in {skVar, skLet, skConst, skProc, skParam, skResult})
+  assert(kind in {skVar, skLet, skConst, skProc, skFunc, skParam, skResult})
   # if we have already checked the type, return true, because we stop the
   # evaluation if something is wrong:
   result = nil
@@ -1115,11 +1107,11 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
   if containsOrIncl(marker, typ.id): return
   var t = skipTypes(typ, abstractInst-{tyTypeDesc})
   case t.kind
-  of tyVar:
-    if kind in {skProc, skConst}: return t
+  of tyVar, tyLent:
+    if kind in {skProc, skFunc, skConst}: return t
     var t2 = skipTypes(t.sons[0], abstractInst-{tyTypeDesc})
     case t2.kind
-    of tyVar:
+    of tyVar, tyLent:
       if taHeap notin flags: result = t2 # ``var var`` is illegal on the heap
     of tyOpenArray:
       if kind != skParam: result = t
@@ -1142,9 +1134,12 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
   of tyVoid:
     if taField notin flags: result = t
   of tyTypeClasses:
-    if not (tfGenericTypeParam in t.flags or taField notin flags): result = t
+    if tfGenericTypeParam in t.flags or taConcept in flags: #or taField notin flags:
+      discard
+    elif kind notin {skParam, skResult}:
+      result = t
   of tyGenericBody, tyGenericParam, tyGenericInvocation,
-     tyNone, tyForward, tyFromExpr, tyFieldAccessor:
+     tyNone, tyForward, tyFromExpr:
     result = t
   of tyNil:
     if kind != skConst: result = t
@@ -1157,10 +1152,10 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
   of tyRange:
     if skipTypes(t.sons[0], abstractInst-{tyTypeDesc}).kind notin
         {tyChar, tyEnum, tyInt..tyFloat128, tyUInt8..tyUInt32}: result = t
-  of tyOpenArray, tyVarargs:
+  of tyOpenArray, tyVarargs, tySink:
     if kind != skParam: result = t
     else: result = typeAllowedAux(marker, t.sons[0], skVar, flags)
-  of tySequence:
+  of tySequence, tyOpt:
     if t.sons[0].kind != tyEmpty:
       result = typeAllowedAux(marker, t.sons[0], skVar, flags+{taHeap})
   of tyArray:
@@ -1176,7 +1171,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
       result = typeAllowedAux(marker, t.sons[i], kind, flags)
       if result != nil: break
   of tyObject, tyTuple:
-    if kind in {skProc, skConst} and
+    if kind in {skProc, skFunc, skConst} and
         t.kind == tyObject and t.sons[0] != nil: return t
     let flags = flags+{taField}
     for i in countup(0, sonsLen(t) - 1):
@@ -1188,17 +1183,74 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
     # for now same as error node; we say it's a valid type as it should
     # prevent cascading errors:
     result = nil
-  of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("typeAllowedAux")
+  of tyUnused, tyOptAsRef: internalError("typeAllowedAux")
 
-proc typeAllowed*(t: PType, kind: TSymKind): PType =
+proc typeAllowed*(t: PType, kind: TSymKind; flags: TTypeAllowedFlags = {}): PType =
   # returns 'nil' on success and otherwise the part of the type that is
   # wrong!
   var marker = initIntSet()
-  result = typeAllowedAux(marker, t, kind, {})
+  result = typeAllowedAux(marker, t, kind, flags)
 
 proc align(address, alignment: BiggestInt): BiggestInt =
   result = (address + (alignment - 1)) and not (alignment - 1)
 
+type
+  OptKind* = enum  ## What to map 'opt T' to internally.
+    oBool      ## opt[T] requires an additional 'bool' field
+    oNil       ## opt[T] has no overhead since 'nil'
+               ## is available
+    oEnum      ## We can use some enum value that is not yet
+               ## used for opt[T]
+    oPtr       ## opt[T] actually introduces a hidden pointer
+               ## in order for the type recursion to work
+
+proc optKind*(typ: PType): OptKind =
+  ## return true iff 'opt[T]' can be mapped to 'T' internally
+  ## because we have a 'nil' value available:
+  assert typ.kind == tyOpt
+  case typ.sons[0].skipTypes(abstractInst).kind
+  of tyRef, tyPtr, tyProc:
+    result = oNil
+  of tyArray, tyObject, tyTuple:
+    result = oPtr
+  of tyBool: result = oEnum
+  of tyEnum:
+    assert(typ.n.sons[0].kind == nkSym)
+    if typ.n.sons[0].sym.position != low(int):
+      result = oEnum
+    else:
+      result = oBool
+  else:
+    result = oBool
+
+proc optLowering*(typ: PType): PType =
+  case optKind(typ)
+  of oNil: result = typ.sons[0]
+  of oPtr:
+    result = newType(tyOptAsRef, typ.owner)
+    result.rawAddSon typ.sons[0]
+  of oBool:
+    result = newType(tyTuple, typ.owner)
+    result.rawAddSon newType(tyBool, typ.owner)
+    result.rawAddSon typ.sons[0]
+  of oEnum:
+    if lastOrd(typ) + 1 < `shl`(BiggestInt(1), 32):
+      result = newType(tyInt32, typ.owner)
+    else:
+      result = newType(tyInt64, typ.owner)
+
+proc optEnumValue*(typ: PType): BiggestInt =
+  assert typ.kind == tyOpt
+  assert optKind(typ) == oEnum
+  let elem = typ.sons[0].skipTypes(abstractInst).kind
+  if elem == tyBool:
+    result = 2
+  else:
+    assert elem == tyEnum
+    assert typ.n.sons[0].sym.position != low(int)
+    result = typ.n.sons[0].sym.position - 1
+
+
 const
   szNonConcreteType* = -3
   szIllegalRecursion* = -2
@@ -1279,7 +1331,7 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt =
     if typ.callConv == ccClosure: result = 2 * ptrSize
     else: result = ptrSize
     a = ptrSize
-  of tyNil, tyCString, tyString, tySequence, tyPtr, tyRef, tyVar, tyOpenArray:
+  of tyNil, tyCString, tyString, tySequence, tyPtr, tyRef, tyVar, tyLent, tyOpenArray:
     let base = typ.lastSon
     if base == typ or (base.kind == tyTuple and base.size==szIllegalRecursion):
       result = szIllegalRecursion
@@ -1353,13 +1405,21 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt =
   of tyStatic:
     result = if typ.n != nil: computeSizeAux(typ.lastSon, a)
              else: szUnknownSize
+  of tyOpt:
+    case optKind(typ)
+    of oBool: result = computeSizeAux(lastSon(typ), a) + 1
+    of oEnum:
+      if lastOrd(typ) + 1 < `shl`(BiggestInt(1), 32): result = 4
+      else: result = 8
+    of oNil: result = computeSizeAux(lastSon(typ), a)
+    of oPtr: result = ptrSize
   else:
     #internalError("computeSizeAux()")
     result = szUnknownSize
   typ.size = result
   typ.align = int16(a)
 
-proc computeSize(typ: PType): BiggestInt =
+proc computeSize*(typ: PType): BiggestInt =
   var a: BiggestInt = 1
   result = computeSizeAux(typ, a)
 
@@ -1368,7 +1428,7 @@ proc getReturnType*(s: PSym): PType =
   assert s.kind in skProcKinds
   result = s.typ.sons[0]
 
-proc getSize(typ: PType): BiggestInt =
+proc getSize*(typ: PType): BiggestInt =
   result = computeSize(typ)
   if result < 0: internalError("getSize: " & $typ.kind)
 
@@ -1466,7 +1526,7 @@ proc isCompileTimeOnly*(t: PType): bool {.inline.} =
 proc containsCompileTimeOnly*(t: PType): bool =
   if isCompileTimeOnly(t): return true
   if t.sons != nil:
-    for i in 0 .. <t.sonsLen:
+    for i in 0 ..< t.sonsLen:
       if t.sons[i] != nil and isCompileTimeOnly(t.sons[i]):
         return true
   return false
@@ -1563,7 +1623,7 @@ proc typeMismatch*(info: TLineInfo, formal, actual: PType) =
     let desc = typeToString(formal, preferDesc)
     let x = if named == desc: named else: named & " = " & desc
     var msg = msgKindToString(errTypeMismatch) &
-              typeToString(actual) & ") " &
+              typeToString(actual) & "> " &
               msgKindToString(errButExpectedX) % [x]
 
     if formal.kind == tyProc and actual.kind == tyProc:

compiler/typesrenderer.nim

@@ -20,7 +20,7 @@ proc renderPlainSymbolName*(n: PNode): string =
   result = ""
   case n.kind
   of nkPostfix, nkAccQuoted:
-    result = renderPlainSymbolName(n[<n.len])
+    result = renderPlainSymbolName(n[n.len-1])
   of nkIdent:
     result = n.ident.s
   of nkSym:
@@ -58,8 +58,8 @@ proc renderType(n: PNode): string =
       assert params.kind == nkFormalParams
       assert len(params) > 0
       result = "proc("
-      for i in 1 .. <len(params): result.add(renderType(params[i]) & ',')
-      result[<len(result)] = ')'
+      for i in 1 ..< len(params): result.add(renderType(params[i]) & ',')
+      result[len(result)-1] = ')'
     else:
       result = "proc"
   of nkIdentDefs:
@@ -67,18 +67,18 @@ proc renderType(n: PNode): string =
     let typePos = len(n) - 2
     let typeStr = renderType(n[typePos])
     result = typeStr
-    for i in 1 .. <typePos:
+    for i in 1 ..< typePos:
       assert n[i].kind == nkIdent
       result.add(',' & typeStr)
   of nkTupleTy:
     result = "tuple["
-    for i in 0 .. <len(n): result.add(renderType(n[i]) & ',')
-    result[<len(result)] = ']'
+    for i in 0 ..< len(n): result.add(renderType(n[i]) & ',')
+    result[len(result)-1] = ']'
   of nkBracketExpr:
     assert len(n) >= 2
     result = renderType(n[0]) & '['
-    for i in 1 .. <len(n): result.add(renderType(n[i]) & ',')
-    result[<len(result)] = ']'
+    for i in 1 ..< len(n): result.add(renderType(n[i]) & ',')
+    result[len(result)-1] = ']'
   else: result = ""
   assert(not result.isNil)
 
@@ -91,7 +91,7 @@ proc renderParamTypes(found: var seq[string], n: PNode) =
   ## generator does include the information.
   case n.kind
   of nkFormalParams:
-    for i in 1 .. <len(n): renderParamTypes(found, n[i])
+    for i in 1 ..< len(n): renderParamTypes(found, n[i])
   of nkIdentDefs:
     # These are parameter names + type + default value node.
     let typePos = len(n) - 2
@@ -102,7 +102,7 @@ proc renderParamTypes(found: var seq[string], n: PNode) =
       let typ = n[typePos+1].typ
       if not typ.isNil: typeStr = typeToString(typ, preferExported)
       if typeStr.len < 1: return
-    for i in 0 .. <typePos:
+    for i in 0 ..< typePos:
       found.add(typeStr)
   else:
     internalError(n.info, "renderParamTypes(found,n) with " & $n.kind)

compiler/vm.nim

@@ -19,7 +19,7 @@ import ast except getstr
 import
   strutils, astalgo, msgs, vmdef, vmgen, nimsets, types, passes,
   parser, vmdeps, idents, trees, renderer, options, transf, parseutils,
-  vmmarshal
+  vmmarshal, gorgeimpl
 
 from semfold import leValueConv, ordinalValToString
 from evaltempl import evalTemplate
@@ -66,7 +66,10 @@ proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) =
     stackTraceAux(c, x.next, x.comesFrom, recursionLimit-1)
     var info = c.debug[pc]
     # we now use the same format as in system/except.nim
-    var s = toFilename(info)
+    var s = substr(toFilename(info), 0)
+    # this 'substr' prevents a strange corruption. XXX This needs to be
+    # investigated eventually but first attempts to fix it broke everything
+    # see the araq-wip-fixed-writebarrier branch.
     var line = toLinenumber(info)
     if line > 0:
       add(s, '(')
@@ -319,7 +322,7 @@ proc opConv*(dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): bool =
       if x <% n.len and (let f = n.sons[x].sym; f.position == x):
         dest.node.strVal = if f.ast.isNil: f.name.s else: f.ast.strVal
       else:
-        for i in 0.. <n.len:
+        for i in 0..<n.len:
           if n.sons[i].kind != nkSym: internalError("opConv for enum")
           let f = n.sons[i].sym
           if f.position == x:
@@ -428,7 +431,7 @@ proc setLenSeq(c: PCtx; node: PNode; newLen: int; info: TLineInfo) =
   setLen(node.sons, newLen)
   if oldLen < newLen:
     # TODO: This is still not correct for tyPtr, tyRef default value
-    for i in oldLen .. <newLen:
+    for i in oldLen ..< newLen:
       node.sons[i] = newNodeI(typeKind, info)
 
 proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
@@ -508,7 +511,13 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         stackTrace(c, tos, pc, errIndexOutOfBounds)
       let idx = regs[rc].intVal.int
       let src = regs[rb].node
-      if src.kind notin {nkEmpty..nkNilLit} and idx <% src.len:
+      if src.kind in {nkStrLit..nkTripleStrLit}:
+        if idx <% src.strVal.len:
+          regs[ra].node = newNodeI(nkCharLit, c.debug[pc])
+          regs[ra].node.intVal = src.strVal[idx].ord
+        else:
+          stackTrace(c, tos, pc, errIndexOutOfBounds)
+      elif src.kind notin {nkEmpty..nkFloat128Lit} and idx <% src.len:
         regs[ra].node = src.sons[idx]
       else:
         stackTrace(c, tos, pc, errIndexOutOfBounds)
@@ -526,8 +535,14 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       # a[b] = c
       decodeBC(rkNode)
       let idx = regs[rb].intVal.int
-      if idx <% regs[ra].node.len:
-        putIntoNode(regs[ra].node.sons[idx], regs[rc])
+      let arr = regs[ra].node
+      if arr.kind in {nkStrLit..nkTripleStrLit}:
+        if idx <% arr.strVal.len:
+          arr.strVal[idx] = chr(regs[rc].intVal)
+        else:
+          stackTrace(c, tos, pc, errIndexOutOfBounds)
+      elif idx <% arr.len:
+        putIntoNode(arr.sons[idx], regs[rc])
       else:
         stackTrace(c, tos, pc, errIndexOutOfBounds)
     of opcLdObj:
@@ -641,8 +656,14 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcLenSeq:
       decodeBImm(rkInt)
       #assert regs[rb].kind == nkBracket
-      # also used by mNLen:
-      regs[ra].intVal = regs[rb].node.safeLen - imm
+      let high = (imm and 1) # discard flags
+      if (imm and nimNodeFlag) != 0:
+        # used by mNLen (NimNode.len)
+        regs[ra].intVal = regs[rb].node.safeLen - high
+      else:
+        # safeArrLen also return string node len
+        # used when string is passed as openArray in VM
+        regs[ra].intVal = regs[rb].node.safeArrLen - high
     of opcLenStr:
       decodeBImm(rkInt)
       assert regs[rb].kind == rkNode
@@ -758,9 +779,21 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       regs[ra].intVal = ord(regs[rb].intVal <% regs[rc].intVal)
     of opcEqRef:
       decodeBC(rkInt)
-      regs[ra].intVal = ord((regs[rb].node.kind == nkNilLit and
-                             regs[rc].node.kind == nkNilLit) or
-                             regs[rb].node == regs[rc].node)
+      if regs[rb].kind == rkNodeAddr:
+        if regs[rc].kind == rkNodeAddr:
+          regs[ra].intVal = ord(regs[rb].nodeAddr == regs[rc].nodeAddr)
+        else:
+          assert regs[rc].kind == rkNode
+          # we know these cannot be equal
+          regs[ra].intVal = ord(false)
+      elif regs[rc].kind == rkNodeAddr:
+        assert regs[rb].kind == rkNode
+        # we know these cannot be equal
+        regs[ra].intVal = ord(false)
+      else:
+        regs[ra].intVal = ord((regs[rb].node.kind == nkNilLit and
+                              regs[rc].node.kind == nkNilLit) or
+                              regs[rb].node == regs[rc].node)
     of opcEqNimrodNode:
       decodeBC(rkInt)
       regs[ra].intVal =
@@ -961,7 +994,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
           let node = regs[rb+i].regToNode
           node.info = c.debug[pc]
           macroCall.add(node)
-        let a = evalTemplate(macroCall, prc, genSymOwner)
+        var a = evalTemplate(macroCall, prc, genSymOwner)
+        if a.kind == nkStmtList and a.len == 1: a = a[0]
         a.recSetFlagIsRef
         ensureKind(rkNode)
         regs[ra].node = a
@@ -1056,7 +1090,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       regs[ra].node = newNodeI(nkBracket, c.debug[pc])
       regs[ra].node.typ = typ
       newSeq(regs[ra].node.sons, count)
-      for i in 0 .. <count:
+      for i in 0 ..< count:
         regs[ra].node.sons[i] = getNullValue(typ.sons[0], c.debug[pc])
     of opcNewStr:
       decodeB(rkNode)
@@ -1191,7 +1225,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       var u = regs[rb].node
       if u.kind notin {nkEmpty..nkNilLit}:
         # XXX can be optimized:
-        for i in 0.. <x.len: u.add(x.sons[i])
+        for i in 0..<x.len: u.add(x.sons[i])
       else:
         stackTrace(c, tos, pc, errGenerated, "cannot add to node kind: " & $u.kind)
       regs[ra].node = u
@@ -1445,7 +1479,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
           "request to create a NimNode of invalid kind")
       let cc = regs[rc].node
 
-      regs[ra].node = newNodeI(TNodeKind(int(k)),
+      let x = newNodeI(TNodeKind(int(k)),
         if cc.kind != nkNilLit:
           cc.info
         elif c.comesFromHeuristic.line > -1:
@@ -1454,7 +1488,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
           c.callsite[1].info
         else:
           c.debug[pc])
-      regs[ra].node.flags.incl nfIsRef
+      x.flags.incl nfIsRef
+      # prevent crashes in the compiler resulting from wrong macros:
+      if x.kind == nkIdent: x.ident = getIdent""
+      regs[ra].node = x
     of opcNCopyNimNode:
       decodeB(rkNode)
       regs[ra].node = copyNode(regs[rb].node)
@@ -1533,7 +1570,7 @@ proc execProc*(c: PCtx; sym: PSym; args: openArray[PNode]): PNode =
       if not isEmptyType(sym.typ.sons[0]) or sym.kind == skMacro:
         putIntoReg(tos.slots[0], getNullValue(sym.typ.sons[0], sym.info))
       # XXX We could perform some type checking here.
-      for i in 1.. <sym.typ.len:
+      for i in 1..<sym.typ.len:
         putIntoReg(tos.slots[i], args[i-1])
 
       result = rawExecute(c, start, tos).regToNode
@@ -1615,7 +1652,7 @@ proc evalConstExprAux(module: PSym; cache: IdentCache; prc: PSym, n: PNode,
   when debugEchoCode: c.echoCode start
   var tos = PStackFrame(prc: prc, comesFrom: 0, next: nil)
   newSeq(tos.slots, c.prc.maxSlots)
-  #for i in 0 .. <c.prc.maxSlots: tos.slots[i] = newNode(nkEmpty)
+  #for i in 0 ..< c.prc.maxSlots: tos.slots[i] = newNode(nkEmpty)
   result = rawExecute(c, start, tos).regToNode
   if result.info.line < 0: result.info = n.info
 
@@ -1648,7 +1685,7 @@ proc setupMacroParam(x: PNode, typ: PType): TFullReg =
 
 iterator genericParamsInMacroCall*(macroSym: PSym, call: PNode): (PSym, PNode) =
   let gp = macroSym.ast[genericParamsPos]
-  for i in 0 .. <gp.len:
+  for i in 0 ..< gp.len:
     let genericParam = gp[i].sym
     let posInCall = macroSym.typ.len + i
     yield (genericParam, call[posInCall])
@@ -1666,8 +1703,7 @@ proc evalMacroCall*(module: PSym; cache: IdentCache, n, nOrig: PNode,
   # arity here too:
   if sym.typ.len > n.safeLen and sym.typ.len > 1:
     globalError(n.info, "in call '$#' got $#, but expected $# argument(s)" % [
-        n.renderTree,
-        $ <n.safeLen, $ <sym.typ.len])
+        n.renderTree, $(n.safeLen-1), $(sym.typ.len-1)])
 
   setupGlobalCtx(module, cache)
   var c = globalCtx
@@ -1691,11 +1727,11 @@ proc evalMacroCall*(module: PSym; cache: IdentCache, n, nOrig: PNode,
   tos.slots[0].node = newNodeI(nkEmpty, n.info)
 
   # setup parameters:
-  for i in 1.. <sym.typ.len:
+  for i in 1..<sym.typ.len:
     tos.slots[i] = setupMacroParam(n.sons[i], sym.typ.sons[i])
 
   let gp = sym.ast[genericParamsPos]
-  for i in 0 .. <gp.len:
+  for i in 0 ..< gp.len:
     if sfImmediate notin sym.flags:
       let idx = sym.typ.len + i
       if idx < n.len:
@@ -1710,7 +1746,7 @@ proc evalMacroCall*(module: PSym; cache: IdentCache, n, nOrig: PNode,
       c.callsite = nil
       globalError(n.info, "static[T] or typedesc nor supported for .immediate macros")
   # temporary storage:
-  #for i in L .. <maxSlots: tos.slots[i] = newNode(nkEmpty)
+  #for i in L ..< maxSlots: tos.slots[i] = newNode(nkEmpty)
   result = rawExecute(c, start, tos).regToNode
   if result.info.line < 0: result.info = n.info
   if cyclicTree(result): globalError(n.info, errCyclicTree)

compiler/vmdef.nim

@@ -234,6 +234,9 @@ const
   slotSomeTemp* = slotTempUnknown
   relativeJumps* = {opcTJmp, opcFJmp, opcJmp, opcJmpBack}
 
+# flag is used to signal opcSeqLen if node is NimNode.
+const nimNodeFlag* = 16
+
 template opcode*(x: TInstr): TOpcode = TOpcode(x.uint32 and 0xff'u32)
 template regA*(x: TInstr): TRegister = TRegister(x.uint32 shr 8'u32 and 0xff'u32)
 template regB*(x: TInstr): TRegister = TRegister(x.uint32 shr 16'u32 and 0xff'u32)

compiler/vmdeps.nim

@@ -7,50 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-import ast, types, msgs, os, osproc, streams, options, idents, securehash
-
-proc readOutput(p: Process): (string, int) =
-  result[0] = ""
-  var output = p.outputStream
-  while not output.atEnd:
-    result[0].add(output.readLine)
-    result[0].add("\n")
-  if result[0].len > 0:
-    result[0].setLen(result[0].len - "\n".len)
-  result[1] = p.waitForExit
-
-proc opGorge*(cmd, input, cache: string, info: TLineInfo): (string, int) =
-  let workingDir = parentDir(info.toFullPath)
-  if cache.len > 0:# and optForceFullMake notin gGlobalOptions:
-    let h = secureHash(cmd & "\t" & input & "\t" & cache)
-    let filename = options.toGeneratedFile("gorge_" & $h, "txt")
-    var f: File
-    if open(f, filename):
-      result = (f.readAll, 0)
-      f.close
-      return
-    var readSuccessful = false
-    try:
-      var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStderrToStdout})
-      if input.len != 0:
-        p.inputStream.write(input)
-        p.inputStream.close()
-      result = p.readOutput
-      readSuccessful = true
-      writeFile(filename, result[0])
-    except IOError, OSError:
-      if not readSuccessful: result = ("", -1)
-  else:
-    try:
-      var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStderrToStdout})
-      if input.len != 0:
-        p.inputStream.write(input)
-        p.inputStream.close()
-      result = p.readOutput
-    except IOError, OSError:
-      result = ("", -1)
+import ast, types, msgs, os, streams, options, idents
 
 proc opSlurp*(file: string, info: TLineInfo, module: PSym): string =
   try:
@@ -84,7 +41,7 @@ proc mapTypeToBracketX(name: string; m: TMagic; t: PType; info: TLineInfo;
                        inst=false): PNode =
   result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
   result.add atomicTypeX(name, m, t, info)
-  for i in 0 .. < t.len:
+  for i in 0 ..< t.len:
     if t.sons[i] == nil:
       let void = atomicTypeX("void", mVoid, t, info)
       void.typ = newType(tyVoid, t.owner)
@@ -127,10 +84,10 @@ proc mapTypeToAstX(t: PType; info: TLineInfo;
 
   if inst:
     if t.sym != nil:  # if this node has a symbol
-      if allowRecursion:  # getTypeImpl behavior: turn off recursion
-        allowRecursion = false
-      else:  # getTypeInst behavior: return symbol
+      if not allowRecursion:  # getTypeInst behavior: return symbol
         return atomicType(t.sym)
+      #else:  # getTypeImpl behavior: turn off recursion
+      #  allowRecursion = false
 
   case t.kind
   of tyNone: result = atomicType("none", mNone)
@@ -162,24 +119,27 @@ proc mapTypeToAstX(t: PType; info: TLineInfo;
       result = atomicType("typeDesc", mTypeDesc)
   of tyGenericInvocation:
     result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
-    for i in 0 .. < t.len:
+    for i in 0 ..< t.len:
       result.add mapTypeToAst(t.sons[i], info)
-  of tyGenericInst, tyAlias:
+  of tyGenericInst:
     if inst:
       if allowRecursion:
         result = mapTypeToAstR(t.lastSon, info)
       else:
         result = newNodeX(nkBracketExpr)
-        result.add mapTypeToAst(t.lastSon, info)
-        for i in 1 .. < t.len-1:
+        #result.add mapTypeToAst(t.lastSon, info)
+        result.add mapTypeToAst(t[0], info)
+        for i in 1 ..< t.len-1:
           result.add mapTypeToAst(t.sons[i], info)
     else:
       result = mapTypeToAstX(t.lastSon, info, inst, allowRecursion)
   of tyGenericBody:
     if inst:
-      result = mapTypeToAstX(t.lastSon, info, inst, true)
+      result = mapTypeToAstR(t.lastSon, info)
     else:
       result = mapTypeToAst(t.lastSon, info)
+  of tyAlias:
+    result = mapTypeToAstX(t.lastSon, info, inst, allowRecursion)
   of tyOrdinal:
     result = mapTypeToAst(t.lastSon, info)
   of tyDistinct:
@@ -249,7 +209,10 @@ proc mapTypeToAstX(t: PType; info: TLineInfo;
     else:
       result = mapTypeToBracket("ref", mRef, t, info)
   of tyVar: result = mapTypeToBracket("var", mVar, t, info)
+  of tyLent: result = mapTypeToBracket("lent", mBuiltinType, t, info)
+  of tySink: result = mapTypeToBracket("sink", mBuiltinType, t, info)
   of tySequence: result = mapTypeToBracket("seq", mSeq, t, info)
+  of tyOpt: result = mapTypeToBracket("opt", mOpt, t, info)
   of tyProc:
     if inst:
       result = newNodeX(nkProcTy)
@@ -304,7 +267,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo;
   of tyNot: result = mapTypeToBracket("not", mNot, t, info)
   of tyAnything: result = atomicType("anything", mNone)
   of tyInferred: internalAssert false
-  of tyStatic, tyFromExpr, tyFieldAccessor:
+  of tyStatic, tyFromExpr:
     if inst:
       if t.n != nil: result = t.n.copyTree
       else: result = atomicType("void", mVoid)
@@ -313,7 +276,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo;
       result.add atomicType("static", mNone)
       if t.n != nil:
         result.add t.n.copyTree
-  of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("mapTypeToAstX")
+  of tyUnused, tyOptAsRef: internalError("mapTypeToAstX")
 
 proc opMapTypeToAst*(t: PType; info: TLineInfo): PNode =
   result = mapTypeToAstX(t, info, false, true)

compiler/vmgen.nim

@@ -30,7 +30,7 @@
 import
   strutils, ast, astalgo, types, msgs, renderer, vmdef,
   trees, intsets, rodread, magicsys, options, lowerings
-
+import platform
 from os import splitFile
 
 when hasFFI:
@@ -401,7 +401,7 @@ proc sameConstant*(a, b: PNode): bool =
 
 proc genLiteral(c: PCtx; n: PNode): int =
   # types do not matter here:
-  for i in 0 .. <c.constants.len:
+  for i in 0 ..< c.constants.len:
     if sameConstant(c.constants[i], n): return i
   result = rawGenLiteral(c, n)
 
@@ -430,7 +430,7 @@ proc genCase(c: PCtx; n: PNode; dest: var TDest) =
     c.gen(n.sons[0], tmp)
     # branch tmp, codeIdx
     # fjmp   elseLabel
-    for i in 1 .. <n.len:
+    for i in 1 ..< n.len:
       let it = n.sons[i]
       if it.len == 1:
         # else stmt:
@@ -460,7 +460,7 @@ proc genTry(c: PCtx; n: PNode; dest: var TDest) =
   c.gen(n.sons[0], dest)
   c.clearDest(n, dest)
   c.patch(elsePos)
-  for i in 1 .. <n.len:
+  for i in 1 ..< n.len:
     let it = n.sons[i]
     if it.kind != nkFinally:
       var blen = len(it)
@@ -518,7 +518,7 @@ proc genCall(c: PCtx; n: PNode; dest: var TDest) =
   let x = c.getTempRange(n.len, slotTempUnknown)
   # varargs need 'opcSetType' for the FFI support:
   let fntyp = skipTypes(n.sons[0].typ, abstractInst)
-  for i in 0.. <n.len:
+  for i in 0..<n.len:
     #if i > 0 and i < sonsLen(fntyp):
     #  let paramType = fntyp.n.sons[i]
     #  if paramType.typ.isCompileTimeOnly: continue
@@ -625,10 +625,10 @@ proc genUnaryABC(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
   c.gABC(n, opc, dest, tmp)
   c.freeTemp(tmp)
 
-proc genUnaryABI(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
+proc genUnaryABI(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; imm: BiggestInt=0) =
   let tmp = c.genx(n.sons[1])
   if dest < 0: dest = c.getTemp(n.typ)
-  c.gABI(n, opc, dest, tmp, 0)
+  c.gABI(n, opc, dest, tmp, imm)
   c.freeTemp(tmp)
 
 proc genBinaryABC(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
@@ -761,6 +761,49 @@ proc genCard(c: PCtx; n: PNode; dest: var TDest) =
   c.gABC(n, opcCard, dest, tmp)
   c.freeTemp(tmp)
 
+proc genIntCast(c: PCtx; n: PNode; dest: var TDest) =
+  const allowedIntegers = {tyInt..tyInt64, tyUInt..tyUInt64, tyChar}
+  var signedIntegers = {tyInt8..tyInt32}
+  var unsignedIntegers = {tyUInt8..tyUInt32, tyChar}
+  let src = n.sons[1].typ.skipTypes(abstractRange)#.kind
+  let dst = n.sons[0].typ.skipTypes(abstractRange)#.kind
+  let src_size = src.getSize
+
+  if platform.intSize < 8:
+    signedIntegers.incl(tyInt)
+    unsignedIntegers.incl(tyUInt)
+  if src_size == dst.getSize and src.kind in allowedIntegers and
+                                 dst.kind in allowedIntegers:
+    let tmp = c.genx(n.sons[1])
+    var tmp2 = c.getTemp(n.sons[1].typ)
+    let tmp3 = c.getTemp(n.sons[1].typ)
+    if dest < 0: dest = c.getTemp(n[0].typ)
+    proc mkIntLit(ival: int): int =
+      result = genLiteral(c, newIntTypeNode(nkIntLit, ival, getSysType(tyInt)))
+    if src.kind in unsignedIntegers and dst.kind in signedIntegers:
+      # cast unsigned to signed integer of same size
+      # signedVal = (unsignedVal xor offset) -% offset
+      let offset = 1 shl (src_size * 8 - 1)
+      c.gABx(n, opcLdConst, tmp2, mkIntLit(offset))
+      c.gABC(n, opcBitxorInt, tmp3, tmp, tmp2)
+      c.gABC(n, opcSubInt, dest, tmp3, tmp2)
+    elif src.kind in signedIntegers and dst.kind in unsignedIntegers:
+      # cast signed to unsigned integer of same size
+      # unsignedVal = (offset +% signedVal +% 1) and offset
+      let offset = (1 shl (src_size * 8))  - 1
+      c.gABx(n, opcLdConst, tmp2, mkIntLit(offset))
+      c.gABx(n, opcLdConst, dest, mkIntLit(offset+1))
+      c.gABC(n, opcAddu, tmp3, tmp, dest)
+      c.gABC(n, opcNarrowU, tmp3, TRegister(src_size*8))
+      c.gABC(n, opcBitandInt, dest, tmp3, tmp2)
+    else:
+      c.gABC(n, opcAsgnInt, dest, tmp)
+    c.freeTemp(tmp)
+    c.freeTemp(tmp2)
+    c.freeTemp(tmp3)
+  else:
+    globalError(n.info, errGenerated, "VM is only allowed to 'cast' between integers of same size")
+
 proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   case m
   of mAnd: c.genAndOr(n, opcFJmp, dest)
@@ -995,7 +1038,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     let n = n[1].skipConv
     let x = c.getTempRange(n.len, slotTempUnknown)
     internalAssert n.kind == nkBracket
-    for i in 0.. <n.len:
+    for i in 0..<n.len:
       var r: TRegister = x+i
       c.gen(n.sons[i], r)
     c.gABC(n, opcEcho, x, n.len)
@@ -1021,7 +1064,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     c.freeTemp(tmp)
   of mSlurp: genUnaryABC(c, n, dest, opcSlurp)
   of mStaticExec: genBinaryABCD(c, n, dest, opcGorge)
-  of mNLen: genUnaryABI(c, n, dest, opcLenSeq)
+  of mNLen: genUnaryABI(c, n, dest, opcLenSeq, nimNodeFlag)
   of mGetImpl: genUnaryABC(c, n, dest, opcGetImpl)
   of mNChild: genBinaryABC(c, n, dest, opcNChild)
   of mNSetChild, mNDel:
@@ -1130,6 +1173,8 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
       # produces a value
     else:
       globalError(n.info, "expandToAst requires a call expression")
+  of mRunnableExamples:
+    discard "just ignore any call to runnableExamples"
   else:
     # mGCref, mGCunref,
     globalError(n.info, "cannot generate code for: " & $m)
@@ -1243,7 +1288,7 @@ proc whichAsgnOpc(n: PNode): TOpcode =
     opcAsgnStr
   of tyFloat..tyFloat128:
     opcAsgnFloat
-  of tyRef, tyNil, tyVar:
+  of tyRef, tyNil, tyVar, tyLent, tyPtr:
     opcAsgnRef
   else:
     opcAsgnComplex
@@ -1289,7 +1334,7 @@ proc checkCanEval(c: PCtx; n: PNode) =
   if s.kind in {skVar, skTemp, skLet, skParam, skResult} and
       not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl:
     cannotEval(n)
-  elif s.kind in {skProc, skConverter, skMethod,
+  elif s.kind in {skProc, skFunc, skConverter, skMethod,
                   skIterator} and sfForward in s.flags:
     cannotEval(n)
 
@@ -1436,7 +1481,7 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
       cannotEval(n)
 
 template needsRegLoad(): untyped =
-  gfAddrOf notin flags and fitsRegister(n.typ.skipTypes({tyVar}))
+  gfAddrOf notin flags and fitsRegister(n.typ.skipTypes({tyVar, tyLent}))
 
 proc genArrAccess2(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
                    flags: TGenFlags) =
@@ -1508,7 +1553,7 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode =
     result = newNodeIT(nkFloatLit, info, t)
   of tyCString, tyString:
     result = newNodeIT(nkStrLit, info, t)
-  of tyVar, tyPointer, tyPtr, tySequence, tyExpr,
+  of tyVar, tyLent, tyPointer, tyPtr, tySequence, tyExpr,
      tyStmt, tyTypeDesc, tyStatic, tyRef, tyNil:
     result = newNodeIT(nkNilLit, info, t)
   of tyProc:
@@ -1537,6 +1582,8 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode =
       addSon(result, getNullValue(t.sons[i], info))
   of tySet:
     result = newNodeIT(nkCurly, info, t)
+  of tyOpt:
+    result = newNodeIT(nkNilLit, info, t)
   else:
     globalError(info, "cannot create null element for: " & $t.kind)
 
@@ -1643,7 +1690,7 @@ proc genObjConstr(c: PCtx, n: PNode, dest: var TDest) =
     c.gABx(n, opcNew, dest, c.genType(t.sons[0]))
   else:
     c.gABx(n, opcLdNull, dest, c.genType(n.typ))
-  for i in 1.. <n.len:
+  for i in 1..<n.len:
     let it = n.sons[i]
     if it.kind == nkExprColonExpr and it.sons[0].kind == nkSym:
       let idx = genField(it.sons[0])
@@ -1658,7 +1705,7 @@ proc genTupleConstr(c: PCtx, n: PNode, dest: var TDest) =
   if dest < 0: dest = c.getTemp(n.typ)
   c.gABx(n, opcLdNull, dest, c.genType(n.typ))
   # XXX x = (x.old, 22)  produces wrong code ... stupid self assignments
-  for i in 0.. <n.len:
+  for i in 0..<n.len:
     let it = n.sons[i]
     if it.kind == nkExprColonExpr:
       let idx = genField(it.sons[0])
@@ -1712,7 +1759,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     case s.kind
     of skVar, skForVar, skTemp, skLet, skParam, skResult:
       genRdVar(c, n, dest, flags)
-    of skProc, skConverter, skMacro, skTemplate, skMethod, skIterator:
+    of skProc, skFunc, skConverter, skMacro, skTemplate, skMethod, skIterator:
       # 'skTemplate' is only allowed for 'getAst' support:
       if procIsCallback(c, s): discard
       elif sfImportc in s.flags: c.importcSym(n.info, s)
@@ -1771,8 +1818,8 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
   of nkAddr, nkHiddenAddr: genAddrDeref(c, n, dest, opcAddrNode, flags)
   of nkIfStmt, nkIfExpr: genIf(c, n, dest)
   of nkWhenStmt:
-      # This is "when nimvm" node. Chose the first branch.
-      gen(c, n.sons[0].sons[1], dest)
+    # This is "when nimvm" node. Chose the first branch.
+    gen(c, n.sons[0].sons[1], dest)
   of nkCaseStmt: genCase(c, n, dest)
   of nkWhileStmt:
     unused(n, dest)
@@ -1794,7 +1841,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     for x in n: gen(c, x)
   of nkStmtListExpr:
     let L = n.len-1
-    for i in 0 .. <L: gen(c, n.sons[i])
+    for i in 0 ..< L: gen(c, n.sons[i])
     gen(c, n.sons[L], dest, flags)
   of nkPragmaBlock:
     gen(c, n.lastSon, dest, flags)
@@ -1808,7 +1855,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
   of nkVarSection, nkLetSection:
     unused(n, dest)
     genVarSection(c, n)
-  of declarativeDefs:
+  of declarativeDefs, nkMacroDef:
     unused(n, dest)
   of nkLambdaKinds:
     #let s = n.sons[namePos].sym
@@ -1840,9 +1887,11 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     if allowCast in c.features:
       genConv(c, n, n.sons[1], dest, opcCast)
     else:
-      globalError(n.info, errGenerated, "VM is not allowed to 'cast'")
+      genIntCast(c, n, dest)
   of nkTypeOfExpr:
     genTypeLit(c, n.typ, dest)
+  of nkComesFrom:
+    discard "XXX to implement for better stack traces"
   else:
     globalError(n.info, errGenerated, "cannot generate VM code for " & $n)
 
@@ -1880,7 +1929,7 @@ proc genExpr*(c: PCtx; n: PNode, requiresValue = true): int =
 proc genParams(c: PCtx; params: PNode) =
   # res.sym.position is already 0
   c.prc.slots[0] = (inUse: true, kind: slotFixedVar)
-  for i in 1.. <params.len:
+  for i in 1..<params.len:
     c.prc.slots[i] = (inUse: true, kind: slotFixedLet)
   c.prc.maxSlots = max(params.len, 1)
 
@@ -1893,7 +1942,7 @@ proc finalJumpTarget(c: PCtx; pc, diff: int) =
 
 proc genGenericParams(c: PCtx; gp: PNode) =
   var base = c.prc.maxSlots
-  for i in 0.. <gp.len:
+  for i in 0..<gp.len:
     var param = gp.sons[i].sym
     param.position = base + i # XXX: fix this earlier; make it consistent with templates
     c.prc.slots[base + i] = (inUse: true, kind: slotFixedLet)
@@ -1901,7 +1950,7 @@ proc genGenericParams(c: PCtx; gp: PNode) =
 
 proc optimizeJumps(c: PCtx; start: int) =
   const maxIterations = 10
-  for i in start .. <c.code.len:
+  for i in start ..< c.code.len:
     let opc = c.code[i].opcode
     case opc
     of opcTJmp, opcFJmp: