bump version to 0.19.6

(cherry picked from commit e54546bcff)

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,45 @@
+---
+name: Bug report
+about: You found an unexpected behaviour? Use this template.
+
+---
+
+<!-- Think about the title, twice. -->
+
+<!-- Summarize the Problem here, keep it simple. -->
+<!-- e.g. `echo` outputs the wrong string. -->
+
+
+### Example
+<!-- This should be a source code block. 
+```nim
+echo "Hello World!"
+```
+-->
+
+
+### Current Output
+<!--
+```
+Hola mundo!
+```
+-->
+
+
+### Expected Output
+<!--
+```
+Hello World!
+```
+-->
+
+
+### Possible Solution
+<!--- Have you found a possible solution? Post it here. -->
+
+
+### Additional Information
+<!--- For Example:
+   A link to a project where the issue is relevant.
+   A link to a related issue or discussion.
+-->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,23 @@
+---
+name: Feature request
+about: You want to suggest a new feature? Use this template.
+
+---
+
+### Summary
+<!--- Short summary of your proposed feature -->
+
+
+### Description
+<!--- Describe your solution, what problem does it fix? -->
+
+
+### Alternatives
+<!--- Are there any alternatives you've considered? -->
+
+
+### Additional Information
+<!--- For Example:
+   A link to a project where the issue is relevant.
+   A link to a related issue or discussion.
+   -->

.gitlab-ci.yml

@@ -53,8 +53,8 @@ test-windows:
   <<: *win_set_path_def
   script:
     - call ci\deps.bat
-    - nim c --taintMode:on tests\testament\tester
-    - tests\testament\tester.exe --pedantic all
+    - nim c --taintMode:on testament\tester
+    - testament\tester.exe --pedantic all
   tags:
     - windows
     - fast

.travis.yml

@@ -42,11 +42,11 @@ script:
   #- nimble install sdl1
   #- nimble install jester@#head -y
   #- nimble install niminst
-  - nim c --taintMode:on -d:nimCoroutines tests/testament/tester
-  - tests/testament/tester --pedantic all -d:nimCoroutines
+  - nim c --taintMode:on -d:nimCoroutines testament/tester
+  - testament/tester --pedantic all -d:nimCoroutines
   - nim c -o:bin/nimpretty nimpretty/nimpretty.nim
   - nim c -r nimpretty/tester.nim
-  - ./koch docs
+  - ./koch docs --git.commit:devel
   - ./koch csource
   - ./koch nimsuggest
   - nim c -r nimsuggest/tester

appveyor.yml

@@ -53,11 +53,11 @@ build_script:
 #  - nimble install opengl -y
 #  - nimble install sdl1 -y
 #  - nimble install jester@#head -y
-  - nim c --taintMode:on -d:nimCoroutines --os:genode -d:posix --compileOnly tests/testament/tester
-  - nim c --taintMode:on -d:nimCoroutines tests/testament/tester
+  - nim c --taintMode:on -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester
+  - nim c --taintMode:on -d:nimCoroutines testament/tester
 
 test_script:
-  - tests\testament\tester --pedantic all -d:nimCoroutines
+  - testament\tester --pedantic all -d:nimCoroutines
   - nim c -r nimdoc\tester
 #  - koch csource
 #  - koch zip

changelog.md

@@ -1,253 +1,12 @@
-## v0.19.X - XX/XX/2018
+## v0.19.2 - 2018-11-04
 
-### Changes affecting backwards compatibility
+This is a bugfix release of version 0.19.0. No new features or breaking changes.
+The most important bugfixes are:
 
-- The stdlib module ``future`` has been renamed to ``sugar``.
-- ``macros.callsite`` is now deprecated. Since the introduction of ``varargs``
-  parameters this became unnecessary.
-- Anonymous tuples with a single element can now be written as ``(1,)`` with a
-  trailing comma. The underlying AST is ``nnkTupleConstr(newLit 1)`` for this
-  example. ``nnkTupleConstr`` is a new node kind your macros need to be able
-  to deal with!
-- Indexing into a ``cstring`` for the JS target is now mapped
-  to ``charCodeAt``.
-- Assignments that would "slice" an object into its supertype are now prevented
-  at runtime. Use ``ref object`` with inheritance rather than ``object`` with
-  inheritance to prevent this issue.
-- The ``not nil`` type annotation now has to be enabled explicitly
-  via ``{.experimental: "notnil"}`` as we are still not pleased with how this
-  feature works with Nim's containers.
-- The parser now warns about inconsistent spacing around binary operators as
-  these can easily be confused with unary operators. This warning will likely
-  become an error in the future.
-- The ``'c`` and ``'C'`` suffix for octal literals is now deprecated to
-  bring the language in line with the standard library (e.g. ``parseOct``).
-- The dot style for import paths (e.g ``import path.to.module`` instead of
-  ``import path/to/module``) has been deprecated.
+- ``spawn`` can handle the empty seqs/strings that are internally
+  represented as ``nil``.
+- The most pressing bugs of the documentation generator have been fixed.
+- async streaming of the httpclient has been fixed.
 
-#### Breaking changes in the standard library
-
-- ``re.split`` for empty regular expressions now yields every character in
-  the string which is what other programming languages chose to do.
-- The returned tuple of ``system.instantiationInfo`` now has a third field
-  containing the column of the instantiation.
-
-- ``cookies.setCookie`` no longer assumes UTC for the expiration date.
-- ``strutils.formatEng`` does not distinguish between ``nil`` and ``""``
-  strings anymore for its ``unit`` parameter. Instead the space is controlled
-  by a new parameter ``useUnitSpace``.
-
-- The ``times.parse`` and ``times.format`` procs have been rewritten.
-  The proc signatures are the same so it should generally not break anything.
-  However, the new implementation is a bit stricter, which is a breaking change.
-  For example ``parse("2017-01-01 foo", "yyyy-MM-dd")`` will now raise an error.
-
-- ``proc `-`*(a, b: Time): int64`` in the ``times`` module has changed return type
-  to ``times.Duration`` in order to support higher time resolutions.
-  The proc is no longer deprecated.
-
-- The ``times.Timezone`` is now an immutable ref-type that must be initialized
-  with an explicit constructor (``newTimezone``).
-
-- ``posix.Timeval.tv_sec`` has changed type to ``posix.Time``.
-
-- ``math.`mod` `` for floats now behaves the same as ``mod`` for integers
-  (previously it used floor division like Python). Use ``math.floorMod`` for the old behavior.
-
-- For string inputs, ``unicode.isUpper`` and ``unicode.isLower`` now require a
-  second mandatory parameter ``skipNonAlpha``.
-
-- For string inputs, ``strutils.isUpperAscii`` and ``strutils.isLowerAscii`` now
-  require a second mandatory parameter ``skipNonAlpha``.
-
-- ``osLastError`` is now marked with ``sideEffect``
-- The procs ``parseHexInt`` and ``parseOctInt`` now fail on empty strings
-  and strings containing only valid prefixes, e.g. "0x" for hex integers.
-
-- ``terminal.setCursorPos`` and ``terminal.setCursorXPos`` now work correctly
-  with 0-based coordinates on POSIX (previously, you needed to use
-  1-based coordinates on POSIX for correct behaviour; the Windows behaviour
-  was always correct).
-
-- ``lineInfoObj`` now returns absolute path instead of project path.
-  It's used by ``lineInfo``, ``check``, ``expect``, ``require``, etc.
-
-- ``net.sendTo`` no longer returns an int and now raises an ``OSError``.
-- `threadpool`'s `await` and derivatives have been renamed to `blockUntil`
-  to avoid confusions with `await` from the `async` macro.
-
-
-#### Breaking changes in the compiler
-
-- The undocumented ``#? braces`` parsing mode was removed.
-- The undocumented PHP backend was removed.
-- The default location of ``nimcache`` for the native code targets was
-  changed. Read [the compiler user guide](https://nim-lang.org/docs/nimc.html#generated-c-code-directory)
-  for more information.
-
-### Library additions
-
-- ``re.split`` now also supports the ``maxsplit`` parameter for consistency
-  with ``strutils.split``.
-- Added ``system.toOpenArray`` in order to support zero-copy slicing
-  operations. This is currently not yet available for the JavaScript target.
-- Added ``getCurrentDir``, ``findExe``, ``cpDir`` and  ``mvDir`` procs to
-  ``nimscript``.
-- The ``times`` module now supports up to nanosecond time resolution when available.
-- Added the type ``times.Duration`` for representing fixed durations of time.
-- Added the proc ``times.convert`` for converting between different time units,
-  e.g days to seconds.
-- Added the proc ``algorithm.binarySearch[T, K]`` with the ```cmp``` parameter.
-- Added the proc ``algorithm.upperBound``.
-- Added inverse hyperbolic functions, ``math.arcsinh``, ``math.arccosh`` and ``math.arctanh`` procs.
-- Added cotangent, secant and cosecant procs ``math.cot``, ``math.sec`` and ``math.csc``; and their hyperbolic, inverse and inverse hyperbolic functions, ``math.coth``, ``math.sech``, ``math.csch``, ``math.arccot``, ``math.arcsec``, ``math.arccsc``, ``math.arccoth``, ``math.arcsech`` and ``math.arccsch`` procs.
-- Added the procs ``math.floorMod`` and ``math.floorDiv`` for floor based integer division.
-- Added the procs ``rationals.`div```, ``rationals.`mod```, ``rationals.floorDiv`` and ``rationals.floorMod`` for rationals.
-- Added the proc ``math.prod`` for product of elements in openArray.
-- Added the proc ``parseBinInt`` to parse a binary integer from a string, which returns the value.
-- ``parseOct`` and ``parseBin`` in parseutils now also support the ``maxLen`` argument similar to ``parseHexInt``.
-- Added the proc ``flush`` for memory mapped files.
-- Added the ``MemMapFileStream``.
-- Added a simple interpreting event parser template ``eventParser`` to the ``pegs`` module.
-- Added ``macros.copyLineInfo`` to copy lineInfo from other node.
-- Added ``system.ashr`` an arithmetic right shift for integers.
-
-### Library changes
-
-- ``macros.astGenRepr``, ``macros.lispRepr`` and ``macros.treeRepr``
-  now escapes the content of string literals consistently.
-- ``macros.NimSym`` and ``macros.NimIdent`` is now deprecated in favor
-  of the more general ``NimNode``.
-- ``macros.getImpl`` now includes the pragmas of types, instead of omitting them.
-- ``macros.hasCustomPragma`` and ``macros.getCustomPragmaVal`` now
-  also support ``ref`` and ``ptr`` types, pragmas on types and variant
-  fields.
-- ``system.SomeReal`` is now called ``SomeFloat`` for consistency and
-  correctness.
-- ``algorithm.smartBinarySearch`` and ``algorithm.binarySearch`` is
-  now joined in ``binarySearch``. ``smartbinarySearch`` is now
-  deprecated.
-- The `terminal` module now exports additional procs for generating ANSI color
-  codes as strings.
-- Added the parameter ``val`` for the ``CritBitTree[int].inc`` proc.
-- An exception raised from a ``test`` block of ``unittest`` now shows its type in
-  error message.
-- The ``compiler/nimeval`` API was rewritten to simplify the "compiler as an
-  API". Using the Nim compiler and its VM as a scripting engine has never been
-  easier. See ``tests/compilerapi/tcompilerapi.nim`` for an example of how to
-  use the Nim VM in a native Nim application.
-- Added the parameter ``val`` for the ``CritBitTree[T].incl`` proc.
-- The proc ``tgamma`` was renamed to ``gamma``. ``tgamma`` is deprecated.
-- The ``pegs`` module now exports getters for the fields of its ``Peg`` and ``NonTerminal``
-  object types. ``Peg``s with child nodes now have the standard ``items`` and ``pairs``
-  iterators.
-- The ``accept`` socket procedure defined in the ``net`` module can now accept
-  a nil socket.
-
-### Language additions
-
-- Dot calls combined with explicit generic instantiations can now be written
-  as ``x.y[:z]`` which is transformed into ``y[z](x)`` by the parser.
-- ``func`` is now an alias for ``proc {.noSideEffect.}``.
-- In order to make ``for`` loops and iterators more flexible to use Nim now
-  supports so called "for-loop macros". See
-  the [manual](manual.html#macros-for-loop-macros) for more details.
-  This feature enables a Python-like generic ``enumerate`` implementation.
-
-- Case statements can now be rewritten via macros. See the [manual](manual.html#macros-case-statement-macros) for more information.
-  This feature enables custom pattern matchers.
-
-
-- the `typedesc` special type has been renamed to just `type`.
-- `static` and `type` are now also modifiers similar to `ref` and `ptr`.
-  They denote the special types `static[T]` and `type[T]`.
-- Forcing compile-time evaluation with `static` now supports specifying
-  the desired target type (as a concrete type or as a type class)
-- The `type` operator now supports checking that the supplied expression
-  matches an expected type constraint.
-
-### Language changes
-
-- The `importcpp` pragma now allows importing the listed fields of generic
-  C++ types. Support for numeric parameters have also been added through
-  the use of `static[T]` types.
-  (#6415)
-
-- Native C++ exceptions can now be imported with `importcpp` pragma.
-  Imported exceptions can be raised and caught just like Nim exceptions.
-  More details in language manual.
-
-- ``nil`` for strings/seqs is finally gone. Instead the default value for
-  these is ``"" / @[]``. Use ``--nilseqs:on`` for a transition period.
-
-- Accessing the binary zero terminator in Nim's native strings
-  is now invalid. Internally a Nim string still has the trailing zero for
-  zero-copy interoperability with ``cstring``. Compile your code with the
-  new switch ``--laxStrings:on`` if you need a transition period.
-
-- The command syntax now supports keyword arguments after the first comma.
-
-- Thread-local variables can now be declared inside procs. This implies all
-  the effects of the ``global`` pragma.
-
-- Nim now supports the ``except`` clause in the export statement.
-
-- Range float types, example ``range[0.0 .. Inf]``. More details in language manual.
-- The ``{.this.}`` pragma has been deprecated. It never worked within generics and
-  we found the resulting code harder to read than the more explicit ``obj.field``
-  syntax.
-- "Memory regions" for pointer types have been deprecated, they were hardly used
-  anywhere. Note that this has **nothing** to do with the ``--gc:regions`` switch
-  of managing memory.
-
-- The exception hierarchy was slightly reworked, ``SystemError`` was renamed to
-  ``CatchableError`` and is the new base class for any exception that is guaranteed to
-  be catchable. This change should have minimal impact on most existing Nim code.
-
-
-### Tool changes
-
-- ``jsondoc2`` has been renamed ``jsondoc``, similar to how ``doc2`` was renamed
-  ``doc``. The old ``jsondoc`` can still be invoked with ``jsondoc0``.
-
-### Compiler changes
-
-- The VM's instruction count limit was raised to 3 million instructions in
-  order to support more complex computations at compile-time.
-
-- Support for hot code reloading has been implemented for the JavaScript
-  target. To use it, compile your code with `--hotCodeReloading:on` and use a
-  helper library such as LiveReload or BrowserSync.
-
-- A new compiler option `--cppCompileToNamespace` puts the generated C++ code
-  into the namespace "Nim" in order to avoid naming conflicts with existing
-  C++ code. This is done for all Nim code - internal and exported.
-
-- Added ``macros.getProjectPath`` and ``ospaths.putEnv`` procs to Nim's virtual
-  machine.
-
-- The ``deadCodeElim`` option is now always turned on and the switch has no
-  effect anymore, but is recognized for backwards compatibility.
-
-- ``experimental`` is now a pragma / command line switch that can enable specific
-  language extensions, it is not an all-or-nothing switch anymore.
-
-- Nintendo Switch was added as a new platform target. See [the compiler user guide](https://nim-lang.org/docs/nimc.html)
-  for more info.
-
-- macros.bindSym now capable to accepts not only literal string or string constant expression.
-  bindSym enhancement make it also can accepts computed string or ident node inside macros /
-  compile time functions / static blocks. Only in templates / regular code it retains it's old behavior.
-  This new feature can be accessed via {.experimental: "dynamicBindSym".} pragma/switch.
-
-- On Posix systems the global system wide configuration is now put under ``/etc/nim/nim.cfg``,
-  it used to be ``/etc/nim.cfg``. Usually it does not exist, however.
-
-- On Posix systems the user configuration is now looked under ``$XDG_CONFIG_HOME/nim/nim.cfg``
-  (if ``XDG_CONFIG_HOME`` is not defined, then under ``~/.config/nim/nim.cfg``). It used to be
-  ``$XDG_CONFIG_DIR/nim.cfg`` (and ``~/.config/nim.cfg``).
-
-  Similarly, on Windows, the user configuration is now looked under ``%APPDATA%/nim/nim.cfg``.
-  This used to be ``%APPDATA%/nim.cfg``.
 
 ### Bugfixes

changelogs/changelog_0_18_1.md

@@ -0,0 +1,663 @@
+## v0.18.0 - 01/03/2018
+
+### Changes affecting backwards compatibility
+
+#### Breaking changes in the standard library
+
+- The ``[]`` proc for strings now raises an ``IndexError`` exception when
+  the specified slice is out of bounds. See issue
+  [#6223](https://github.com/nim-lang/Nim/issues/6223) for more details.
+  You can use ``substr(str, start, finish)`` to get the old behaviour back,
+  see [this commit](https://github.com/nim-lang/nimbot/commit/98cc031a27ea89947daa7f0bb536bcf86462941f) for an example.
+
+- ``strutils.split`` and ``strutils.rsplit`` with an empty string and a
+  separator now returns that empty string.
+  See issue [#4377](https://github.com/nim-lang/Nim/issues/4377).
+
+- Arrays of char cannot be converted to ``cstring`` anymore, pointers to
+  arrays of char can! This means ``$`` for arrays can finally exist
+  in ``system.nim`` and do the right thing. This means ``$myArrayOfChar`` changed
+  its behaviour! Compile with ``-d:nimNoArrayToString`` to see where to fix your
+  code.
+
+- `reExtended` is no longer default for the `re` constructor in the `re`
+  module.
+
+- The behavior of ``$`` has been changed for all standard library collections. The
+  collection-to-string implementations now perform proper quoting and escaping of
+  strings and chars.
+
+- `newAsyncSocket` taking an `AsyncFD` now runs `setBlocking(false)` on the
+  fd.
+
+- ``mod`` and bitwise ``and`` do not produce ``range`` subtypes anymore. This
+  turned out to be more harmful than helpful and the language is simpler
+  without this special typing rule.
+
+- ``formatFloat``/``formatBiggestFloat`` now support formatting floats with zero
+  precision digits. The previous ``precision = 0`` behavior (default formatting)
+  is now available via ``precision = -1``.
+
+- Moved from stdlib into Nimble packages:
+  - [``basic2d``](https://github.com/nim-lang/basic2d)
+    _deprecated: use ``glm``, ``arraymancer``, ``neo``, or another package instead_
+  - [``basic3d``](https://github.com/nim-lang/basic3d)
+    _deprecated: use ``glm``, ``arraymancer``, ``neo``, or another package instead_
+  - [``gentabs``](https://github.com/lcrees/gentabs)
+  - [``libuv``](https://github.com/lcrees/libuv)
+  - [``numeric``](https://github.com/lcrees/polynumeric)
+  - [``poly``](https://github.com/lcrees/polynumeric)
+  - [``pdcurses``](https://github.com/lcrees/pdcurses)
+  - [``romans``](https://github.com/lcrees/romans)
+  - [``libsvm``](https://github.com/nim-lang/libsvm_legacy)
+  - [``joyent_http_parser``](https://github.com/nim-lang/joyent_http_parser)
+
+- Proc [toCountTable](https://nim-lang.org/docs/tables.html#toCountTable,openArray[A])
+  now produces a `CountTable` with values correspoding to the number of occurrences
+  of the key in the input. It used to produce a table with all values set to `1`.
+
+  Counting occurrences in a sequence used to be:
+
+  ```nim
+  let mySeq = @[1, 2, 1, 3, 1, 4]
+  var myCounter = initCountTable[int]()
+
+  for item in mySeq:
+    myCounter.inc item
+  ```
+
+  Now, you can simply do:
+
+  ```nim
+  let
+    mySeq = @[1, 2, 1, 3, 1, 4]
+    myCounter = mySeq.toCountTable()
+  ```
+
+- If you use ``--dynlibOverride:ssl`` with OpenSSL 1.0.x, you now have to
+  define ``openssl10`` symbol (``-d:openssl10``). By default OpenSSL 1.1.x is
+  assumed.
+
+- ``newNativeSocket`` is now named ``createNativeSocket``.
+
+- ``newAsyncNativeSocket`` is now named ``createAsyncNativeSocket``
+  and it no longer raises an OS error but returns an ``osInvalidSocket`` when
+  creation fails.
+
+- The ``securehash`` module is now deprecated. Instead import ``std / sha1``.
+
+- The ``readPasswordFromStdin`` proc has been moved from the ``rdstdin``
+  to the ``terminal`` module, thus it does not depend on linenoise anymore.
+
+#### Breaking changes in the compiler
+
+- ``\n`` is now only the single line feed character like in most
+  other programming languages. The new platform specific newline escape sequence is
+  written as ``\p``. This change only affects the Windows platform.
+
+- The overloading rules changed slightly so that constrained generics are
+  preferred over unconstrained generics. (Bug #6526)
+
+- We changed how array accesses "from backwards" like ``a[^1]`` or ``a[0..^1]`` are
+  implemented. These are now implemented purely in ``system.nim`` without compiler
+  support. There is a new "heterogenous" slice type ``system.HSlice`` that takes 2
+  generic parameters which can be ``BackwardsIndex`` indices. ``BackwardsIndex`` is
+  produced by ``system.^``.
+  This means if you overload ``[]`` or ``[]=`` you need to ensure they also work
+  with ``system.BackwardsIndex`` (if applicable for the accessors).
+
+- The parsing rules of ``if`` expressions were changed so that multiple
+  statements are allowed in the branches. We found few code examples that
+  now fail because of this change, but here is one:
+
+```nim
+t[ti] = if exp_negative: '-' else: '+'; inc(ti)
+```
+
+This now needs to be written as:
+
+```nim
+t[ti] = (if exp_negative: '-' else: '+'); inc(ti)
+```
+
+- The experimental overloading of the dot ``.`` operators now take
+  an ``untyped``` parameter as the field name, it used to be
+  a ``static[string]``. You can use ``when defined(nimNewDot)`` to make
+  your code work with both old and new Nim versions.
+  See [special-operators](https://nim-lang.org/docs/manual.html#special-operators)
+  for more information.
+
+- ``yield`` (or ``await`` which is mapped to ``yield``) never worked reliably
+  in an array, seq or object constructor and is now prevented at compile-time.
+
+### Library additions
+
+- **Added ``sequtils.mapLiterals`` for easier construction of array and tuple literals.**
+
+- Added ``system.runnableExamples`` to make examples in Nim's documentation easier
+  to write and test. The examples are tested as the last step of
+  ``nim doc``.
+
+- Implemented ``getIoHandler`` proc in the ``asyncdispatch`` module that allows
+  you to retrieve the underlying IO Completion Port or ``Selector[AsyncData]``
+  object in the specified dispatcher.
+
+- For string formatting / interpolation a new module
+  called [strformat](https://nim-lang.org/docs/strformat.html) has been added
+  to the stdlib.
+
+- The `ReadyKey` type in the selectors module now contains an ``errorCode``
+  field to help distinguish between ``Event.Error`` events.
+
+- Implemented an `accept` proc that works on a `SocketHandle` in
+  ``nativesockets``.
+
+- Added ``algorithm.rotateLeft``.
+
+- Added ``typetraits.$`` as an alias for ``typetraits.name``.
+
+- Added ``system.getStackTraceEntries`` that allows you to access the stack
+  trace in a structured manner without string parsing.
+
+- Added ``parseutils.parseSaturatedNatural``.
+
+- Added ``macros.unpackVarargs``.
+
+- Added support for asynchronous programming for the JavaScript backend using
+  the `asyncjs` module.
+
+- Added true color support for some terminals. Example:
+```nim
+import colors, terminal
+
+const Nim = "Efficient and expressive programming."
+
+var
+  fg = colYellow
+  bg = colBlue
+  int = 1.0
+
+enableTrueColors()
+
+for i in 1..15:
+  styledEcho bgColor, bg, fgColor, fg, Nim, resetStyle
+  int -= 0.01
+  fg = intensity(fg, int)
+
+setForegroundColor colRed
+setBackgroundColor colGreen
+styledEcho "Red on Green.", resetStyle
+```
+
+### Library changes
+
+- ``echo`` now works with strings that contain ``\0`` (the binary zero is not
+  shown) and ``nil`` strings are equal to empty strings.
+
+- JSON: Deprecated `getBVal`, `getFNum`, and `getNum` in favour of
+  `getBool`, `getFloat`, `getBiggestInt`. A new `getInt` procedure was also
+  added.
+
+- ``rationals.toRational`` now uses an algorithm based on continued fractions.
+  This means its results are more precise and it can't run into an infinite loop
+  anymore.
+
+- ``os.getEnv`` now takes an optional ``default`` parameter that tells ``getEnv``
+  what to return if the environment variable does not exist.
+
+- The ``random`` procs in ``random.nim`` have all been deprecated. Instead use
+  the new ``rand`` procs. The module now exports the state of the random
+  number generator as type ``Rand`` so multiple threads can easily use their
+  own random number generators that do not require locking. For more information
+  about this rename see issue [#6934](https://github.com/nim-lang/Nim/issues/6934)
+
+- ``writeStackTrace`` is now proclaimed to have no IO effect (even though it does)
+  so that it is more useful for debugging purposes.
+
+- ``db_mysql`` module: ``DbConn`` is now a ``distinct`` type that doesn't expose the
+  details of the underlying ``PMySQL`` type.
+
+- ``parseopt2`` is now deprecated, use ``parseopt`` instead.
+
+### Language additions
+
+- It is now possible to forward declare object types so that mutually
+  recursive types can be created across module boundaries. See
+  [package level objects](https://nim-lang.org/docs/manual.html#package-level-objects)
+  for more information.
+
+- Added support for casting between integers of same bitsize in VM (compile time and nimscript).
+  This allows to, among other things, reinterpret signed integers as unsigned.
+
+- Custom pragmas are now supported using pragma ``pragma``, please see language
+  manual for details.
+
+- Standard library modules can now also be imported via the ``std`` pseudo-directory.
+  This is useful in order to distinguish between standard library and nimble package
+  imports:
+
+  ```nim
+  import std / [strutils, os, osproc]
+  import someNimblePackage / [strutils, os]
+  ```
+
+### Language changes
+
+- The **unary** ``<`` is now deprecated, for ``.. <`` use ``..<`` for other usages
+  use the ``pred`` proc.
+
+- Bodies of ``for`` loops now get their own scope:
+
+```nim
+# now compiles:
+for i in 0..4:
+  let i = i + 1
+  echo i
+```
+
+- To make Nim even more robust the system iterators ``..`` and ``countup``
+  now only accept a single generic type ``T``. This means the following code
+  doesn't die with an "out of range" error anymore:
+
+```nim
+var b = 5.Natural
+var a = -5
+for i in a..b:
+  echo i
+```
+
+- ``atomic`` and ``generic`` are no longer keywords in Nim. ``generic`` used to be
+  an alias for ``concept``, ``atomic`` was not used for anything.
+
+- The memory manager now uses a variant of the TLSF algorithm that has much
+  better memory fragmentation behaviour. According
+  to [http://www.gii.upv.es/tlsf/](http://www.gii.upv.es/tlsf/) the maximum
+  fragmentation measured is lower than 25%. As a nice bonus ``alloc`` and
+  ``dealloc`` became O(1) operations.
+
+- The compiler is now more consistent in its treatment of ambiguous symbols:
+  Types that shadow procs and vice versa are marked as ambiguous (bug #6693).
+
+- codegenDecl pragma now works for the JavaScript backend. It returns an empty
+  string for function return type placeholders.
+
+- Extra semantic checks for procs with noreturn pragma: return type is not allowed,
+  statements after call to noreturn procs are no longer allowed.
+
+- Noreturn proc calls and raising exceptions branches are now skipped during common type
+  deduction in ``if`` and ``case`` expressions. The following code snippets now compile:
+  ```nim
+  import strutils
+  let str = "Y"
+  let a = case str:
+    of "Y": true
+    of "N": false
+    else: raise newException(ValueError, "Invalid boolean")
+  let b = case str:
+    of nil, "": raise newException(ValueError, "Invalid boolean")
+    elif str.startsWith("Y"): true
+    elif str.startsWith("N"): false
+    else: false
+  let c = if str == "Y": true
+    elif str == "N": false
+    else:
+      echo "invalid bool"
+      quit("this is the end")
+  ```
+
+- Pragmas now support call syntax, for example: ``{.exportc"myname".}`` and
+  ``{.exportc("myname").}``
+
+- The ``deprecated`` pragma now supports a user-definable warning message for procs.
+
+  ```nim
+  proc bar {.deprecated: "use foo instead".} =
+    return
+
+  bar()
+  ```
+
+### Tool changes
+
+- The ``nim doc`` command is now an alias for ``nim doc2``, the second version of
+  the documentation generator. The old version 1 can still be accessed
+  via the new ``nim doc0`` command.
+
+- Nim's ``rst2html`` command now supports the testing of code snippets via an RST
+  extension that we called ``:test:``::
+
+  ```rst
+  .. code-block:: nim
+      :test:
+    # shows how the 'if' statement works
+    if true: echo "yes"
+  ```
+
+### Compiler changes
+
+### Bugfixes
+
+- Fixed "ReraiseError when using try/except within finally block"
+  ([#5871](https://github.com/nim-lang/Nim/issues/5871))
+- Fixed "Range type inference leads to counter-intuitive behvaiour"
+  ([#5854](https://github.com/nim-lang/Nim/issues/5854))
+- Fixed "JSON % operator can fail in extern procs with dynamic types"
+  ([#6385](https://github.com/nim-lang/Nim/issues/6385))
+- Fixed ""intVal is not accessible" in VM"
+  ([#6083](https://github.com/nim-lang/Nim/issues/6083))
+- Fixed "Add excl for OrderedSet"
+  ([#2467](https://github.com/nim-lang/Nim/issues/2467))
+- Fixed "newSeqOfCap actually doesn't reserve memory"
+  ([#6403](https://github.com/nim-lang/Nim/issues/6403))
+- Fixed "[Regression] Nim segfaults"
+  ([#6435](https://github.com/nim-lang/Nim/issues/6435))
+- Fixed "Seq assignment is slower than expected"
+  ([#6433](https://github.com/nim-lang/Nim/issues/6433))
+- Fixed "json module issues with empty dicts and lists"
+  ([#6438](https://github.com/nim-lang/Nim/issues/6438))
+- Fixed "mingw installed via finish.exe fails to link if Nim located in path with whitespace"
+  ([#6452](https://github.com/nim-lang/Nim/issues/6452))
+- Fixed "unittest.check does not perform short-circuit evaluation"
+  ([#5784](https://github.com/nim-lang/Nim/issues/5784))
+- Fixed "Error while concatenating an array of chars."
+  ([#5861](https://github.com/nim-lang/Nim/issues/5861))
+- Fixed "range initialization: [ProveInit] hint: Cannot prove that"
+  ([#6474](https://github.com/nim-lang/Nim/issues/6474))
+- Fixed "scanf can call procs with side-effects multiple times"
+  ([#6487](https://github.com/nim-lang/Nim/issues/6487))
+- Fixed "gcsafe detection problem"
+  ([#5620](https://github.com/nim-lang/Nim/issues/5620))
+- Fixed "C++ codegen: `mitems` generates invalid code."
+  ([#4910](https://github.com/nim-lang/Nim/issues/4910))
+- Fixed "strange runtime behavior on macOS"
+  ([#6496](https://github.com/nim-lang/Nim/issues/6496))
+- Fixed "stdtmpl: invalid indentation after a line ending in question mark"
+  ([#5070](https://github.com/nim-lang/Nim/issues/5070))
+- Fixed "Windows: NAN troubles on c backend"
+  ([#6511](https://github.com/nim-lang/Nim/issues/6511))
+- Fixed "lib/nim/system/cellsets.nim(33, 31) Error: type mismatch while attempting to compile for 16bit CPUs"
+  ([#3558](https://github.com/nim-lang/Nim/issues/3558))
+- Fixed "Can't compile dynlib with ``-d:useNimRtl`` and ``--threads:on``"
+  ([#5143](https://github.com/nim-lang/Nim/issues/5143))
+- Fixed "var s = @[0,1,2,...] can generate thousand of single assignments in C code"
+  ([#5007](https://github.com/nim-lang/Nim/issues/5007))
+- Fixed "`echo` discards everything after a null character"
+  ([#1137](https://github.com/nim-lang/Nim/issues/1137))
+- Fixed "Turn off reExtended by default"
+  ([#5627](https://github.com/nim-lang/Nim/issues/5627))
+- Fixed "Bad Links in docs/backends.html"
+  ([#5914](https://github.com/nim-lang/Nim/issues/5914))
+- Fixed "Index out of bounds error in db_postgres when executing non parameter-substituted queries containing "?""
+  ([#6571](https://github.com/nim-lang/Nim/issues/6571))
+- Fixed "Please add pipe2 support to posix stdlib"
+  ([#6553](https://github.com/nim-lang/Nim/issues/6553))
+- Fixed "Return semantics vary depending on return style"
+  ([#6422](https://github.com/nim-lang/Nim/issues/6422))
+- Fixed "parsecsv.open reports SIGSEGV when calling 'open' on missing file"
+  ([#6148](https://github.com/nim-lang/Nim/issues/6148))
+- Fixed "VCC: Nim generates non-compilable code for system.nim"
+  ([#6606](https://github.com/nim-lang/Nim/issues/6606))
+- Fixed "Generic subtype matches worse than a generic"
+  ([#6526](https://github.com/nim-lang/Nim/issues/6526))
+- Fixed "formatFloat inconsistent scientific notation"
+  ([#6589](https://github.com/nim-lang/Nim/issues/6589))
+- Fixed "Generated c code calls function twice"
+  ([#6292](https://github.com/nim-lang/Nim/issues/6292))
+- Fixed "Range type inference leads to counter-intuitive behvaiour"
+  ([#5854](https://github.com/nim-lang/Nim/issues/5854))
+- Fixed "New backward indexing is too limited"
+  ([#6631](https://github.com/nim-lang/Nim/issues/6631))
+- Fixed "Table usage in a macro (SIGSEGV: Illegal storage access.)"
+  ([#1860](https://github.com/nim-lang/Nim/issues/1860))
+- Fixed "Incorrect deprecation error"
+  ([#6634](https://github.com/nim-lang/Nim/issues/6634))
+- Fixed "Wrong indices in arrays not starting with 0"
+  ([#6675](https://github.com/nim-lang/Nim/issues/6675))
+- Fixed "if expressions"
+  ([#6609](https://github.com/nim-lang/Nim/issues/6609))
+- Fixed "BackwardsIndex: converter + `[]` + unrelated type[^1]: lib/system.nim(3536, 3) Error"
+  ([#6692](https://github.com/nim-lang/Nim/issues/6692))
+- Fixed "BackwardsIndex: converter + `[]` + unrelated type[^1]: lib/system.nim(3536, 3) Error"
+  ([#6692](https://github.com/nim-lang/Nim/issues/6692))
+- Fixed "js backend 0.17.3: array bounds check for non zero based arrays is buggy"
+  ([#6532](https://github.com/nim-lang/Nim/issues/6532))
+- Fixed "HttpClient's new API doesn't work through a proxy for https URLs"
+  ([#6685](https://github.com/nim-lang/Nim/issues/6685))
+- Fixed "isServing isn't declared and isn't compiling"
+  ([#6707](https://github.com/nim-lang/Nim/issues/6707))
+- Fixed "[Regression] value out of range"
+  ([#6710](https://github.com/nim-lang/Nim/issues/6710))
+
+- Fixed "Error when using `multisync` macro"
+  ([#6708](https://github.com/nim-lang/Nim/issues/6708))
+
+- Fixed "formatFloat inconsistent scientific notation"
+  ([#6589](https://github.com/nim-lang/Nim/issues/6589))
+- Fixed "Using : (constructor arguments) for passing values to functions with default arguments causes a compiler crash."
+  ([#6765](https://github.com/nim-lang/Nim/issues/6765))
+- Fixed "In-place object initialization leads to vcc incompatible code"
+  ([#6757](https://github.com/nim-lang/Nim/issues/6757))
+- Fixed "Improve parseCookies doc"
+  ([#5721](https://github.com/nim-lang/Nim/issues/5721))
+- Fixed "Parser regression with nested do notation inside conditional"
+  ([#6166](https://github.com/nim-lang/Nim/issues/6166))
+- Fixed "Request for better error message"
+  ([#6776](https://github.com/nim-lang/Nim/issues/6776))
+- Fixed "Testament tester does not execute test with `exitcode` only"
+  ([#6775](https://github.com/nim-lang/Nim/issues/6775))
+- Fixed "JS integer division off by one"
+  ([#6753](https://github.com/nim-lang/Nim/issues/6753))
+- Fixed "Regression: cannot prove not nil"
+  ([#5781](https://github.com/nim-lang/Nim/issues/5781))
+- Fixed "SIGSEGV: Illegal storage access. (Attempt to read from nil?) in generic proc"
+  ([#6073](https://github.com/nim-lang/Nim/issues/6073))
+- Fixed "Request for better error message"
+  ([#6776](https://github.com/nim-lang/Nim/issues/6776))
+- Fixed "Nim #head: sorting via reference hangs compiler"
+  ([#6724](https://github.com/nim-lang/Nim/issues/6724))
+- Fixed "Cannot cast pointer to char in cpp"
+  ([#5979](https://github.com/nim-lang/Nim/issues/5979))
+- Fixed "asynchttpserver replies with several errors on single request"
+  ([#6386](https://github.com/nim-lang/Nim/issues/6386))
+- Fixed "object variants superclass trigger bad codegen"
+  ([#5521](https://github.com/nim-lang/Nim/issues/5521))
+- Fixed "JS integer division off by one"
+  ([#6753](https://github.com/nim-lang/Nim/issues/6753))
+- Fixed "js backend compiler crash with tables indexed by certain types"
+  ([#6568](https://github.com/nim-lang/Nim/issues/6568))
+- Fixed "Jsgen bug with is"
+  ([#6445](https://github.com/nim-lang/Nim/issues/6445))
+- Fixed "Subrange definition with ..<"
+  ([#6788](https://github.com/nim-lang/Nim/issues/6788))
+- Fixed "fields not initialized: array with enum index type as object field."
+  ([#6682](https://github.com/nim-lang/Nim/issues/6682))
+- Fixed "Can not delete data in table when table's data type is kind of "not nil""
+  ([#6555](https://github.com/nim-lang/Nim/issues/6555))
+- Fixed "tables.nim: Cannot prove that 'n' is initialized"
+  ([#6121](https://github.com/nim-lang/Nim/issues/6121))
+- Fixed "issues with 'not nil' applied to a closure proc"
+  ([#6489](https://github.com/nim-lang/Nim/issues/6489))
+- Fixed "`not nil` not working in some cases"
+  ([#4686](https://github.com/nim-lang/Nim/issues/4686))
+- Fixed "Cannot prove '@[v]' is not nil"
+  ([#3993](https://github.com/nim-lang/Nim/issues/3993))
+
+- Fixed "Feature: support TCP_NODELAY in net.sockets"
+  ([#6795](https://github.com/nim-lang/Nim/issues/6795))
+- Fixed "Code that makes the compiler throw an error message and then hangs"
+  ([#6820](https://github.com/nim-lang/Nim/issues/6820))
+- Fixed "Code that makes the compiler throw an error message and then hangs"
+  ([#6820](https://github.com/nim-lang/Nim/issues/6820))
+- Fixed "Inconsistent behavior with sequence and string slicing"
+  ([#6223](https://github.com/nim-lang/Nim/issues/6223))
+- Fixed "Wrong behavior of "split" (proc and iterator)"
+  ([#4377](https://github.com/nim-lang/Nim/issues/4377))
+- Fixed "[Documentation] Invalid module name: [foo, bar]"
+  ([#6831](https://github.com/nim-lang/Nim/issues/6831))
+- Fixed "The destructor is not called for temporary objects"
+  ([#4214](https://github.com/nim-lang/Nim/issues/4214))
+- Fixed "Destructors does not work with implicit items iterator in for loop"
+  ([#985](https://github.com/nim-lang/Nim/issues/985))
+- Fixed "Error in template when using the type of the parameter inside it"
+  ([#6756](https://github.com/nim-lang/Nim/issues/6756))
+- Fixed "should json.to() respect parent attributes?"
+  ([#5856](https://github.com/nim-lang/Nim/issues/5856))
+- Fixed "json 'to' macro can not marshalize into tuples"
+  ([#6095](https://github.com/nim-lang/Nim/issues/6095))
+- Fixed "json.to fails with seq[T]"
+  ([#6604](https://github.com/nim-lang/Nim/issues/6604))
+- Fixed "json.to() is not worth using compared to marshal.to[T]"
+  ([#5848](https://github.com/nim-lang/Nim/issues/5848))
+- Fixed "Memory not being released in time, running out of memory"
+  ([#6031](https://github.com/nim-lang/Nim/issues/6031))
+- Fixed "[Regression] Bad C codegen for generic code"
+  ([#6889](https://github.com/nim-lang/Nim/issues/6889))
+- Fixed "rollingFileLogger deletes file on every start."
+  ([#6264](https://github.com/nim-lang/Nim/issues/6264))
+- Fixed "Remove/deprecate securehash module."
+  ([#6033](https://github.com/nim-lang/Nim/issues/6033))
+- Fixed "[bug or not] object construction for seq[T] failed without space after colon"
+  ([#5999](https://github.com/nim-lang/Nim/issues/5999))
+- Fixed "issues with the random module"
+  ([#4726](https://github.com/nim-lang/Nim/issues/4726))
+- Fixed "Reassigning local var to seq of objects results in nil element in Object's seq field"
+  ([#668](https://github.com/nim-lang/Nim/issues/668))
+- Fixed "Compilation error with "newseq[string]""
+  ([#6726](https://github.com/nim-lang/Nim/issues/6726))
+- Fixed "await inside array/dict literal produces invalid code - Part 2"
+  ([#6626](https://github.com/nim-lang/Nim/issues/6626))
+- Fixed "terminal.eraseline() gives OverflowError on Windows"
+  ([#6931](https://github.com/nim-lang/Nim/issues/6931))
+- Fixed "[Regression] `sequtils.any` conflicts with `system.any`"
+  ([#6932](https://github.com/nim-lang/Nim/issues/6932))
+- Fixed "C++ codegen: `mitems` generates invalid code."
+  ([#4910](https://github.com/nim-lang/Nim/issues/4910))
+- Fixed "seq.mitems produces invalid cpp codegen"
+  ([#6892](https://github.com/nim-lang/Nim/issues/6892))
+- Fixed "Concepts regression"
+  ([#6108](https://github.com/nim-lang/Nim/issues/6108))
+- Fixed "Generic iterable concept with array crashes compiler"
+  ([#6277](https://github.com/nim-lang/Nim/issues/6277))
+- Fixed "C code generation "‘a’ is a pointer; did you mean to use ‘->’?""
+  ([#6462](https://github.com/nim-lang/Nim/issues/6462))
+- Fixed "`--NimblePath` fails if a `-` in path which is not followed by a number"
+  ([#6949](https://github.com/nim-lang/Nim/issues/6949))
+- Fixed ""not registered in the selector" in asyncfile.close() for something that clearly was registered"
+  ([#6906](https://github.com/nim-lang/Nim/issues/6906))
+- Fixed "strange frexp behavior"
+  ([#6353](https://github.com/nim-lang/Nim/issues/6353))
+
+- Fixed "noreturn branches of case statements shouldn't contribute to type"
+  ([#6885](https://github.com/nim-lang/Nim/issues/6885))
+- Fixed "Type inference for 'if' statements changed"
+  ([#6980](https://github.com/nim-lang/Nim/issues/6980))
+- Fixed "newest asyncdispatch recursion"
+  ([#6100](https://github.com/nim-lang/Nim/issues/6100))
+- Fixed "Ambiguous identifier between set type and proc"
+  ([#6965](https://github.com/nim-lang/Nim/issues/6965))
+
+- Fixed "Inconsistent behavior with sequence and string slicing"
+  ([#6223](https://github.com/nim-lang/Nim/issues/6223))
+
+- Fixed "Unsupported OpenSSL library imported dynamically"
+  ([#5000](https://github.com/nim-lang/Nim/issues/5000))
+- Fixed "`nim check` segfaults"
+  ([#6972](https://github.com/nim-lang/Nim/issues/6972))
+- Fixed "GC deadlock"
+  ([#6988](https://github.com/nim-lang/Nim/issues/6988))
+- Fixed "Create a seq without memory initialization"
+  ([#6401](https://github.com/nim-lang/Nim/issues/6401))
+- Fixed "Fix bug for getch on Windows while using the arrow keys"
+  ([#6966](https://github.com/nim-lang/Nim/issues/6966))
+- Fixed "runnableExamples doesn't work in templates"
+  ([#7018](https://github.com/nim-lang/Nim/issues/7018))
+- Fixed "runnableExamples doesn't work with untyped statement blocks"
+  ([#7019](https://github.com/nim-lang/Nim/issues/7019))
+
+- Fixed "Critical bug in parseBiggestFloat"
+  ([#7060](https://github.com/nim-lang/Nim/issues/7060))
+- Fixed "[RFC] strformat.% should be gone"
+  ([#7078](https://github.com/nim-lang/Nim/issues/7078))
+- Fixed "compiler crash on simple macro"
+  ([#7093](https://github.com/nim-lang/Nim/issues/7093))
+- Fixed "Make newlines sane again"
+  ([#7089](https://github.com/nim-lang/Nim/issues/7089))
+- Fixed "JS - Unicode enum string representation issue"
+  ([#6741](https://github.com/nim-lang/Nim/issues/6741))
+- Fixed "Strange behaviour of 0.17.3 (working ok in 0.17.2)"
+  ([#6989](https://github.com/nim-lang/Nim/issues/6989))
+- Fixed "Strange behaviour of 0.17.3 (working ok in 0.17.2)"
+  ([#6989](https://github.com/nim-lang/Nim/issues/6989))
+- Fixed "Compiler crash: try expression with infix as"
+  ([#7116](https://github.com/nim-lang/Nim/issues/7116))
+- Fixed "nimsuggest crash"
+  ([#7140](https://github.com/nim-lang/Nim/issues/7140))
+- Fixed "[RFC] Reintroduce readChar"
+  ([#7072](https://github.com/nim-lang/Nim/issues/7072))
+- Fixed "Copyright line needs updating"
+  ([#7129](https://github.com/nim-lang/Nim/issues/7129))
+- Fixed "-0.0 doesn't result in negative zero in VM"
+  ([#7079](https://github.com/nim-lang/Nim/issues/7079))
+- Fixed "Windows large filesize"
+  ([#7121](https://github.com/nim-lang/Nim/issues/7121))
+- Fixed "Securehash is not parsimonious with MD5 and other hash modules"
+  ([#6961](https://github.com/nim-lang/Nim/issues/6961))
+- Fixed "os.findExe() shouldn't look in current directory on posix, unless exe has a /"
+  ([#6939](https://github.com/nim-lang/Nim/issues/6939))
+- Fixed "`compiles(...)` with `fatal` pragma causes compiler to exit early"
+  ([#7080](https://github.com/nim-lang/Nim/issues/7080))
+- Fixed "NPE when compile macro that returns concrete value"
+  ([#5450](https://github.com/nim-lang/Nim/issues/5450))
+- Fixed "Using a variable of type `int | float` causes internal compiler error"
+  ([#6946](https://github.com/nim-lang/Nim/issues/6946))
+- Fixed "Unsigned integers could not be used as array indexes."
+  ([#7153](https://github.com/nim-lang/Nim/issues/7153))
+- Fixed "countdown with uint causes underflow"
+  ([#4220](https://github.com/nim-lang/Nim/issues/4220))
+- Fixed "Inconsistent method call syntax"
+  ([#7200](https://github.com/nim-lang/Nim/issues/7200))
+- Fixed "Impossible to create an empty const array"
+  ([#6853](https://github.com/nim-lang/Nim/issues/6853))
+- Fixed "Strange UINT handling"
+  ([#3985](https://github.com/nim-lang/Nim/issues/3985))
+- Fixed "Bad codegen when passing arg that is part of return value destination"
+  ([#6960](https://github.com/nim-lang/Nim/issues/6960))
+- Fixed "No info about gcsafety in error message when global var is accessed in async proc"
+  ([#6186](https://github.com/nim-lang/Nim/issues/6186))
+- Fixed "Resolving package vs. local import ambiguities"
+  ([#2819](https://github.com/nim-lang/Nim/issues/2819))
+- Fixed "Internal error with type() operator"
+  ([#3711](https://github.com/nim-lang/Nim/issues/3711))
+- Fixed "newAsyncSocket should raise an OS error plus other inconsistencies"
+  ([#4995](https://github.com/nim-lang/Nim/issues/4995))
+- Fixed "mapLiterals fails with negative values"
+  ([#7215](https://github.com/nim-lang/Nim/issues/7215))
+- Fixed "fmWrite doesn't truncate file with openAsync, unlike open()"
+  ([#5531](https://github.com/nim-lang/Nim/issues/5531))
+- Fixed "Move libsvm to an external nimble module"
+  ([#5786](https://github.com/nim-lang/Nim/issues/5786))
+- Fixed "Prevent acceptAddr gotcha with newSocket"
+  ([#7227](https://github.com/nim-lang/Nim/issues/7227))
+- Fixed "strtabs.getOrDefault is inconsistent with tables.getOrDefault"
+  ([#4265](https://github.com/nim-lang/Nim/issues/4265))
+
+- Fixed "Code falling through into exception handler when no exception thrown."
+  ([#7232](https://github.com/nim-lang/Nim/issues/7232))
+- Fixed "the new generic inference rules are broken"
+  ([#7247](https://github.com/nim-lang/Nim/issues/7247))
+- Fixed "Odd `..<` regression"
+  ([#6992](https://github.com/nim-lang/Nim/issues/6992))
+- Fixed "Different proc type inferred from default parameter"
+  ([#4659](https://github.com/nim-lang/Nim/issues/4659))
+- Fixed "Different proc type inferred from default parameter"
+  ([#4659](https://github.com/nim-lang/Nim/issues/4659))
+- Fixed "Testament sometimes ignores test failures"
+  ([#7236](https://github.com/nim-lang/Nim/issues/7236))
+- Fixed "New Allocator Fails On >=4GB Requests"
+  ([#7120](https://github.com/nim-lang/Nim/issues/7120))
+- Fixed "User pragmas hide effect specifications from sempass2"
+  ([#7216](https://github.com/nim-lang/Nim/issues/7216))
+- Fixed "C++: SIGABRT instead of IndexError for out-of-bounds"
+  ([#6512](https://github.com/nim-lang/Nim/issues/6512))
+- Fixed "An uncaught exception in cpp mode doesn't show the exception name/msg"
+  ([#6431](https://github.com/nim-lang/Nim/issues/6431))

changelogs/changelog_0_19_0.md

@@ -0,0 +1,253 @@
+## v0.19.X - XX/XX/2018
+
+### Changes affecting backwards compatibility
+
+- The stdlib module ``future`` has been renamed to ``sugar``.
+- ``macros.callsite`` is now deprecated. Since the introduction of ``varargs``
+  parameters this became unnecessary.
+- Anonymous tuples with a single element can now be written as ``(1,)`` with a
+  trailing comma. The underlying AST is ``nnkTupleConstr(newLit 1)`` for this
+  example. ``nnkTupleConstr`` is a new node kind your macros need to be able
+  to deal with!
+- Indexing into a ``cstring`` for the JS target is now mapped
+  to ``charCodeAt``.
+- Assignments that would "slice" an object into its supertype are now prevented
+  at runtime. Use ``ref object`` with inheritance rather than ``object`` with
+  inheritance to prevent this issue.
+- The ``not nil`` type annotation now has to be enabled explicitly
+  via ``{.experimental: "notnil"}`` as we are still not pleased with how this
+  feature works with Nim's containers.
+- The parser now warns about inconsistent spacing around binary operators as
+  these can easily be confused with unary operators. This warning will likely
+  become an error in the future.
+- The ``'c`` and ``'C'`` suffix for octal literals is now deprecated to
+  bring the language in line with the standard library (e.g. ``parseOct``).
+- The dot style for import paths (e.g ``import path.to.module`` instead of
+  ``import path/to/module``) has been deprecated.
+
+#### Breaking changes in the standard library
+
+- ``re.split`` for empty regular expressions now yields every character in
+  the string which is what other programming languages chose to do.
+- The returned tuple of ``system.instantiationInfo`` now has a third field
+  containing the column of the instantiation.
+
+- ``cookies.setCookie`` no longer assumes UTC for the expiration date.
+- ``strutils.formatEng`` does not distinguish between ``nil`` and ``""``
+  strings anymore for its ``unit`` parameter. Instead the space is controlled
+  by a new parameter ``useUnitSpace``.
+
+- The ``times.parse`` and ``times.format`` procs have been rewritten.
+  The proc signatures are the same so it should generally not break anything.
+  However, the new implementation is a bit stricter, which is a breaking change.
+  For example ``parse("2017-01-01 foo", "yyyy-MM-dd")`` will now raise an error.
+
+- ``proc `-`*(a, b: Time): int64`` in the ``times`` module has changed return type
+  to ``times.Duration`` in order to support higher time resolutions.
+  The proc is no longer deprecated.
+
+- The ``times.Timezone`` is now an immutable ref-type that must be initialized
+  with an explicit constructor (``newTimezone``).
+
+- ``posix.Timeval.tv_sec`` has changed type to ``posix.Time``.
+
+- ``math.`mod` `` for floats now behaves the same as ``mod`` for integers
+  (previously it used floor division like Python). Use ``math.floorMod`` for the old behavior.
+
+- For string inputs, ``unicode.isUpper`` and ``unicode.isLower`` now require a
+  second mandatory parameter ``skipNonAlpha``.
+
+- For string inputs, ``strutils.isUpperAscii`` and ``strutils.isLowerAscii`` now
+  require a second mandatory parameter ``skipNonAlpha``.
+
+- ``osLastError`` is now marked with ``sideEffect``
+- The procs ``parseHexInt`` and ``parseOctInt`` now fail on empty strings
+  and strings containing only valid prefixes, e.g. "0x" for hex integers.
+
+- ``terminal.setCursorPos`` and ``terminal.setCursorXPos`` now work correctly
+  with 0-based coordinates on POSIX (previously, you needed to use
+  1-based coordinates on POSIX for correct behaviour; the Windows behaviour
+  was always correct).
+
+- ``lineInfoObj`` now returns absolute path instead of project path.
+  It's used by ``lineInfo``, ``check``, ``expect``, ``require``, etc.
+
+- ``net.sendTo`` no longer returns an int and now raises an ``OSError``.
+- `threadpool`'s `await` and derivatives have been renamed to `blockUntil`
+  to avoid confusions with `await` from the `async` macro.
+
+
+#### Breaking changes in the compiler
+
+- The undocumented ``#? braces`` parsing mode was removed.
+- The undocumented PHP backend was removed.
+- The default location of ``nimcache`` for the native code targets was
+  changed. Read [the compiler user guide](https://nim-lang.org/docs/nimc.html#generated-c-code-directory)
+  for more information.
+
+### Library additions
+
+- ``re.split`` now also supports the ``maxsplit`` parameter for consistency
+  with ``strutils.split``.
+- Added ``system.toOpenArray`` in order to support zero-copy slicing
+  operations. This is currently not yet available for the JavaScript target.
+- Added ``getCurrentDir``, ``findExe``, ``cpDir`` and  ``mvDir`` procs to
+  ``nimscript``.
+- The ``times`` module now supports up to nanosecond time resolution when available.
+- Added the type ``times.Duration`` for representing fixed durations of time.
+- Added the proc ``times.convert`` for converting between different time units,
+  e.g days to seconds.
+- Added the proc ``algorithm.binarySearch[T, K]`` with the ```cmp``` parameter.
+- Added the proc ``algorithm.upperBound``.
+- Added inverse hyperbolic functions, ``math.arcsinh``, ``math.arccosh`` and ``math.arctanh`` procs.
+- Added cotangent, secant and cosecant procs ``math.cot``, ``math.sec`` and ``math.csc``; and their hyperbolic, inverse and inverse hyperbolic functions, ``math.coth``, ``math.sech``, ``math.csch``, ``math.arccot``, ``math.arcsec``, ``math.arccsc``, ``math.arccoth``, ``math.arcsech`` and ``math.arccsch`` procs.
+- Added the procs ``math.floorMod`` and ``math.floorDiv`` for floor based integer division.
+- Added the procs ``rationals.`div```, ``rationals.`mod```, ``rationals.floorDiv`` and ``rationals.floorMod`` for rationals.
+- Added the proc ``math.prod`` for product of elements in openArray.
+- Added the proc ``parseBinInt`` to parse a binary integer from a string, which returns the value.
+- ``parseOct`` and ``parseBin`` in parseutils now also support the ``maxLen`` argument similar to ``parseHexInt``.
+- Added the proc ``flush`` for memory mapped files.
+- Added the ``MemMapFileStream``.
+- Added a simple interpreting event parser template ``eventParser`` to the ``pegs`` module.
+- Added ``macros.copyLineInfo`` to copy lineInfo from other node.
+- Added ``system.ashr`` an arithmetic right shift for integers.
+
+### Library changes
+
+- ``macros.astGenRepr``, ``macros.lispRepr`` and ``macros.treeRepr``
+  now escapes the content of string literals consistently.
+- ``macros.NimSym`` and ``macros.NimIdent`` is now deprecated in favor
+  of the more general ``NimNode``.
+- ``macros.getImpl`` now includes the pragmas of types, instead of omitting them.
+- ``macros.hasCustomPragma`` and ``macros.getCustomPragmaVal`` now
+  also support ``ref`` and ``ptr`` types, pragmas on types and variant
+  fields.
+- ``system.SomeReal`` is now called ``SomeFloat`` for consistency and
+  correctness.
+- ``algorithm.smartBinarySearch`` and ``algorithm.binarySearch`` is
+  now joined in ``binarySearch``. ``smartbinarySearch`` is now
+  deprecated.
+- The `terminal` module now exports additional procs for generating ANSI color
+  codes as strings.
+- Added the parameter ``val`` for the ``CritBitTree[int].inc`` proc.
+- An exception raised from a ``test`` block of ``unittest`` now shows its type in
+  error message.
+- The ``compiler/nimeval`` API was rewritten to simplify the "compiler as an
+  API". Using the Nim compiler and its VM as a scripting engine has never been
+  easier. See ``tests/compilerapi/tcompilerapi.nim`` for an example of how to
+  use the Nim VM in a native Nim application.
+- Added the parameter ``val`` for the ``CritBitTree[T].incl`` proc.
+- The proc ``tgamma`` was renamed to ``gamma``. ``tgamma`` is deprecated.
+- The ``pegs`` module now exports getters for the fields of its ``Peg`` and ``NonTerminal``
+  object types. ``Peg``s with child nodes now have the standard ``items`` and ``pairs``
+  iterators.
+- The ``accept`` socket procedure defined in the ``net`` module can now accept
+  a nil socket.
+
+### Language additions
+
+- Dot calls combined with explicit generic instantiations can now be written
+  as ``x.y[:z]`` which is transformed into ``y[z](x)`` by the parser.
+- ``func`` is now an alias for ``proc {.noSideEffect.}``.
+- In order to make ``for`` loops and iterators more flexible to use Nim now
+  supports so called "for-loop macros". See
+  the [manual](manual.html#macros-for-loop-macros) for more details.
+  This feature enables a Python-like generic ``enumerate`` implementation.
+
+- Case statements can now be rewritten via macros. See the [manual](manual.html#macros-case-statement-macros) for more information.
+  This feature enables custom pattern matchers.
+
+
+- the `typedesc` special type has been renamed to just `type`.
+- `static` and `type` are now also modifiers similar to `ref` and `ptr`.
+  They denote the special types `static[T]` and `type[T]`.
+- Forcing compile-time evaluation with `static` now supports specifying
+  the desired target type (as a concrete type or as a type class)
+- The `type` operator now supports checking that the supplied expression
+  matches an expected type constraint.
+
+### Language changes
+
+- The `importcpp` pragma now allows importing the listed fields of generic
+  C++ types. Support for numeric parameters have also been added through
+  the use of `static[T]` types.
+  (#6415)
+
+- Native C++ exceptions can now be imported with `importcpp` pragma.
+  Imported exceptions can be raised and caught just like Nim exceptions.
+  More details in language manual.
+
+- ``nil`` for strings/seqs is finally gone. Instead the default value for
+  these is ``"" / @[]``. Use ``--nilseqs:on`` for a transition period.
+
+- Accessing the binary zero terminator in Nim's native strings
+  is now invalid. Internally a Nim string still has the trailing zero for
+  zero-copy interoperability with ``cstring``. Compile your code with the
+  new switch ``--laxStrings:on`` if you need a transition period.
+
+- The command syntax now supports keyword arguments after the first comma.
+
+- Thread-local variables can now be declared inside procs. This implies all
+  the effects of the ``global`` pragma.
+
+- Nim now supports the ``except`` clause in the export statement.
+
+- Range float types, example ``range[0.0 .. Inf]``. More details in language manual.
+- The ``{.this.}`` pragma has been deprecated. It never worked within generics and
+  we found the resulting code harder to read than the more explicit ``obj.field``
+  syntax.
+- "Memory regions" for pointer types have been deprecated, they were hardly used
+  anywhere. Note that this has **nothing** to do with the ``--gc:regions`` switch
+  of managing memory.
+
+- The exception hierarchy was slightly reworked, ``SystemError`` was renamed to
+  ``CatchableError`` and is the new base class for any exception that is guaranteed to
+  be catchable. This change should have minimal impact on most existing Nim code.
+
+
+### Tool changes
+
+- ``jsondoc2`` has been renamed ``jsondoc``, similar to how ``doc2`` was renamed
+  ``doc``. The old ``jsondoc`` can still be invoked with ``jsondoc0``.
+
+### Compiler changes
+
+- The VM's instruction count limit was raised to 3 million instructions in
+  order to support more complex computations at compile-time.
+
+- Support for hot code reloading has been implemented for the JavaScript
+  target. To use it, compile your code with `--hotCodeReloading:on` and use a
+  helper library such as LiveReload or BrowserSync.
+
+- A new compiler option `--cppCompileToNamespace` puts the generated C++ code
+  into the namespace "Nim" in order to avoid naming conflicts with existing
+  C++ code. This is done for all Nim code - internal and exported.
+
+- Added ``macros.getProjectPath`` and ``ospaths.putEnv`` procs to Nim's virtual
+  machine.
+
+- The ``deadCodeElim`` option is now always turned on and the switch has no
+  effect anymore, but is recognized for backwards compatibility.
+
+- ``experimental`` is now a pragma / command line switch that can enable specific
+  language extensions, it is not an all-or-nothing switch anymore.
+
+- Nintendo Switch was added as a new platform target. See [the compiler user guide](https://nim-lang.org/docs/nimc.html)
+  for more info.
+
+- macros.bindSym now capable to accepts not only literal string or string constant expression.
+  bindSym enhancement make it also can accepts computed string or ident node inside macros /
+  compile time functions / static blocks. Only in templates / regular code it retains it's old behavior.
+  This new feature can be accessed via {.experimental: "dynamicBindSym".} pragma/switch.
+
+- On Posix systems the global system wide configuration is now put under ``/etc/nim/nim.cfg``,
+  it used to be ``/etc/nim.cfg``. Usually it does not exist, however.
+
+- On Posix systems the user configuration is now looked under ``$XDG_CONFIG_HOME/nim/nim.cfg``
+  (if ``XDG_CONFIG_HOME`` is not defined, then under ``~/.config/nim/nim.cfg``). It used to be
+  ``$XDG_CONFIG_DIR/nim.cfg`` (and ``~/.config/nim.cfg``).
+
+  Similarly, on Windows, the user configuration is now looked under ``%APPDATA%/nim/nim.cfg``.
+  This used to be ``%APPDATA%/nim.cfg``.
+
+### Bugfixes

changelogs/changelog_X_XX_X.md

@@ -0,0 +1,27 @@
+## v0.XX.0 - XX/XX/2018
+
+### Changes affecting backwards compatibility
+
+- Example item: ``Foo`` changed to ``Bar``.
+
+#### Breaking changes in the standard library
+
+
+#### Breaking changes in the compiler
+
+### Library additions
+
+### Library changes
+
+
+### Language additions
+
+
+### Language changes
+
+
+### Tool changes
+
+### Compiler changes
+
+### Bugfixes

changelogs/readme.md

@@ -0,0 +1,61 @@
+# Update changelog
+After each release, call:
+```
+git mv changelog.md changelogs/changelog_0_19_0.md # use correct version
+cp changelogs/changelog_X_XX_X.md changelog.md
+git add changelog.md
+```
+
+## Recent changelogs are saved here (with their git history)
+
+## Older changelogs are stored in https://github.com/nim-lang/website
+
+## source files:
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2009-12-21-version-086-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2010-03-14-version-088-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2010-10-20-version-0810-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2011-07-10-version-0812-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2012-02-09-version-0814-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2012-09-23-version-090-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2013-05-20-version-092-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2014-04-21-version-094-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2014-10-19-version-096-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2014-12-29-version-0102-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2015-04-30-version-0110-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2015-05-04-version-0112-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2015-10-27-version-0120-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2016-01-18-version-0130-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2016-06-07-version-0140-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2016-06-09-version-0142-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2016-09-30-version-0150-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2016-10-23-version-0152-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2017-01-08-version-0160-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2017-05-17-version-0170-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2017-09-07-version-0172-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2018-03-01-version-0180-released.md
+* https://github.com/nim-lang/website/blob/master/jekyll/_posts/2018-09-26-version-0190-released.md
+
+## urls:
+* https://nim-lang.org/blog/2009/12/21/version-086-released.html
+* https://nim-lang.org/blog/2010/03/14/version-088-released.html
+* https://nim-lang.org/blog/2010/10/20/version-0810-released.html
+* https://nim-lang.org/blog/2011/07/10/version-0812-released.html
+* https://nim-lang.org/blog/2012/02/09/version-0814-released.html
+* https://nim-lang.org/blog/2012/09/23/version-090-released.html
+* https://nim-lang.org/blog/2013/05/20/version-092-released.html
+* https://nim-lang.org/blog/2014/04/21/version-094-released.html
+* https://nim-lang.org/blog/2014/10/19/version-096-released.html
+* https://nim-lang.org/blog/2014/12/29/version-0102-released.html
+* https://nim-lang.org/blog/2015/04/30/version-0110-released.html
+* https://nim-lang.org/blog/2015/05/04/version-0112-released.html
+* https://nim-lang.org/blog/2015/10/27/version-0120-released.html
+* https://nim-lang.org/blog/2016/01/18/version-0130-released.html
+* https://nim-lang.org/blog/2016/06/07/version-0140-released.html
+* https://nim-lang.org/blog/2016/06/09/version-0142-released.html
+* https://nim-lang.org/blog/2016/09/30/version-0150-released.html
+* https://nim-lang.org/blog/2016/10/23/version-0152-released.html
+* https://nim-lang.org/blog/2017/01/08/version-0160-released.html
+* https://nim-lang.org/blog/2017/05/17/version-0170-released.html
+* https://nim-lang.org/blog/2017/09/07/version-0172-released.html
+* https://nim-lang.org/blog/2018/03/01/version-0180-released.html
+* https://nim-lang.org/blog/2018/09/26/version-0190-released.html

ci/nsis_build.bat

@@ -1,4 +1,4 @@
-REM - Run the full testsuite;  tests\testament\tester all
+REM - Run the full testsuite;  testament\tester all
 
 REM - Uncomment the list of changes in news.txt
 REM - write a news ticker entry

compiler/aliases.nim

@@ -181,9 +181,16 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
       else: discard
     of nkObjConstr:
       result = arNo
-      for i in 1..<b.len:
+      for i in 1 ..< b.len:
         let res = isPartOf(a, b[i][1])
         if res != arNo:
           result = res
           if res == arYes: break
+    of nkCall:
+      result = arNo
+      for i in 1 ..< b.len:
+        let res = isPartOf(a, b[i])
+        if res != arNo:
+          result = res
+          if res == arYes: break
     else: discard

compiler/ast.nim

@@ -990,7 +990,7 @@ const
   miscPos* = 5  # used for undocumented and hacky stuff
   bodyPos* = 6       # position of body; use rodread.getBody() instead!
   resultPos* = 7
-  dispatcherPos* = 8 # caution: if method has no 'result' it can be position 7!
+  dispatcherPos* = 8
 
   nkCallKinds* = {nkCall, nkInfix, nkPrefix, nkPostfix,
                   nkCommand, nkCallStrLit, nkHiddenCallConv}
@@ -1711,8 +1711,7 @@ proc toVar*(typ: PType): PType =
 proc toRef*(typ: PType): PType =
   ## If ``typ`` is a tyObject then it is converted into a `ref <typ>` and
   ## returned. Otherwise ``typ`` is simply returned as-is.
-  result = typ
-  if typ.kind == tyObject:
+  if typ.skipTypes({tyAlias, tyGenericInst}).kind == tyObject:
     result = newType(tyRef, typ.owner)
     rawAddSon(result, typ)
 
@@ -1720,19 +1719,19 @@ proc toObject*(typ: PType): PType =
   ## If ``typ`` is a tyRef then its immediate son is returned (which in many
   ## cases should be a ``tyObject``).
   ## Otherwise ``typ`` is simply returned as-is.
-  result = typ
-  if result.kind == tyRef:
-    result = result.lastSon
+  let t = typ.skipTypes({tyAlias, tyGenericInst})
+  if t.kind == tyRef: t.lastSon
+  else: typ
 
 proc isException*(t: PType): bool =
   # check if `y` is object type and it inherits from Exception
   assert(t != nil)
 
-  if t.kind != tyObject:
+  if t.kind notin {tyObject, tyGenericInst}:
     return false
 
   var base = t
-  while base != nil:
+  while base != nil and base.kind in {tyRef, tyObject, tyGenericInst}:
     if base.sym != nil and base.sym.magic == mException:
       return true
     base = base.lastSon

compiler/ccgexprs.nim

@@ -31,10 +31,18 @@ proc intLiteral(i: BiggestInt): Rope =
     result = ~"(IL64(-9223372036854775807) - IL64(1))"
 
 proc genLiteral(p: BProc, n: PNode, ty: PType): Rope =
-  if ty == nil: internalError(p.config, n.info, "genLiteral: ty is nil")
   case n.kind
   of nkCharLit..nkUInt64Lit:
-    case skipTypes(ty, abstractVarRange).kind
+    var k: TTypeKind
+    if ty != nil:
+      k = skipTypes(ty, abstractVarRange).kind
+    else:
+      case n.kind
+      of nkCharLit: k = tyChar
+      of nkUInt64Lit: k = tyUInt64
+      of nkInt64Lit: k = tyInt64
+      else: k = tyNil # don't go into the case variant that uses 'ty'
+    case k
     of tyChar, tyNil:
       result = intLiteral(n.intVal)
     of tyBool:
@@ -46,8 +54,8 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope =
       result = "(($1) $2)" % [getTypeDesc(p.module,
           ty), intLiteral(n.intVal)]
   of nkNilLit:
-    let t = skipTypes(ty, abstractVarRange)
-    if t.kind == tyProc and t.callConv == ccClosure:
+    let k = if ty == nil: tyPointer else: skipTypes(ty, abstractVarRange).kind
+    if k == tyProc and skipTypes(ty, abstractVarRange).callConv == ccClosure:
       let id = nodeTableTestOrSet(p.module.dataCache, n, p.module.labels)
       result = p.module.tmpBase & rope(id)
       if id == p.module.labels:
@@ -59,7 +67,9 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope =
     else:
       result = rope("NIM_NIL")
   of nkStrLit..nkTripleStrLit:
-    case skipTypes(ty, abstractVarRange + {tyStatic, tyUserTypeClass, tyUserTypeClassInst}).kind
+    let k = if ty == nil: tyString
+            else: skipTypes(ty, abstractVarRange + {tyStatic, tyUserTypeClass, tyUserTypeClassInst}).kind
+    case k
     of tyNil:
       result = genNilStringLiteral(p.module, n.info)
     of tyString:
@@ -1976,8 +1986,8 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       frmt = "$1 = $2->len;$n"
     lineCg(p, cpsStmts, frmt, tmp.r, rdLoc(a))
     putIntoDest(p, d, e, tmp.r)
-  of mGCref: unaryStmt(p, e, d, "#nimGCref($1);$n")
-  of mGCunref: unaryStmt(p, e, d, "#nimGCunref($1);$n")
+  of mGCref: unaryStmt(p, e, d, "if ($1) { #nimGCref($1); }$n")
+  of mGCunref: unaryStmt(p, e, d, "if ($1) { #nimGCunref($1); }$n")
   of mSetLengthStr: genSetLengthStr(p, e, d)
   of mSetLengthSeq: genSetLengthSeq(p, e, d)
   of mIncl, mExcl, mCard, mLtSet, mLeSet, mEqSet, mMulSet, mPlusSet, mMinusSet,
@@ -2394,15 +2404,8 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
     if ex.kind != nkEmpty:
       genLineDir(p, n)
       var a: TLoc
-      if ex.kind in nkCallKinds and (ex[0].kind != nkSym or
-                                     ex[0].sym.magic == mNone):
-        # bug #6037: do not assign to a temp in C++ mode:
-        incl a.flags, lfSingleUse
-        genCall(p, ex, a)
-        if lfSingleUse notin a.flags:
-          line(p, cpsStmts, a.r & ";\L")
-      else:
-        initLocExpr(p, ex, a)
+      initLocExprSingleUse(p, ex, a)
+      line(p, cpsStmts, "(void)(" & a.r & ");\L")
   of nkAsmStmt: genAsmStmt(p, n)
   of nkTryStmt:
     if p.module.compileToCpp and optNoCppExceptions notin p.config.globalOptions:

compiler/ccgstmts.nim

@@ -307,11 +307,13 @@ proc genSingleVar(p: BProc, a: PNode) =
 
 proc genClosureVar(p: BProc, a: PNode) =
   var immediateAsgn = a.sons[2].kind != nkEmpty
+  var v: TLoc
+  initLocExpr(p, a.sons[0], v)
+  genLineDir(p, a)
   if immediateAsgn:
-    var v: TLoc
-    initLocExpr(p, a.sons[0], v)
-    genLineDir(p, a)
     loadInto(p, a.sons[0], a.sons[2], v)
+  else:
+    constructLoc(p, v)
 
 proc genVarStmt(p: BProc, n: PNode) =
   for it in n.sons:

compiler/ccgtypes.nim

@@ -165,7 +165,6 @@ proc mapType(conf: ConfigRef; typ: PType): TCTypeKind =
     of tySet:
       if mapSetType(conf, base) == ctArray: result = ctPtrToArray
       else: result = ctPtr
-    # XXX for some reason this breaks the pegs module
     else: result = ctPtr
   of tyPointer: result = ctPtr
   of tySequence: result = ctNimSeq
@@ -968,9 +967,9 @@ proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope;
 
 proc discriminatorTableName(m: BModule, objtype: PType, d: PSym): Rope =
   # bugfix: we need to search the type that contains the discriminator:
-  var objtype = objtype
+  var objtype = objtype.skipTypes(abstractPtrs)
   while lookupInRecord(objtype.n, d.name) == nil:
-    objtype = objtype.sons[0]
+    objtype = objtype.sons[0].skipTypes(abstractPtrs)
   if objtype.sym == nil:
     internalError(m.config, d.info, "anonymous obj with discriminator")
   result = "NimDT_$1_$2" % [rope($hashType(objtype)), rope(d.name.s.mangle)]

compiler/cgen.nim

@@ -18,7 +18,7 @@ import
 
 import strutils except `%` # collides with ropes.`%`
 
-from modulegraphs import ModuleGraph
+from modulegraphs import ModuleGraph, PPassContext
 from lineinfos import
   warnGcMem, errXMustBeCompileTime, hintDependency, errGenerated, errCannotOpenFile
 import dynlib
@@ -567,7 +567,12 @@ proc loadDynamicLib(m: BModule, lib: PLib) =
       var p = newProc(nil, m)
       p.options = p.options - {optStackTrace, optEndb}
       var dest: TLoc
-      initLocExpr(p, lib.path, dest)
+      initLoc(dest, locTemp, lib.path, OnStack)
+      dest.r = getTempName(m)
+      appcg(m, m.s[cfsDynLibInit],"$1 $2;$n",
+           [getTypeDesc(m, lib.path.typ), rdLoc(dest)])
+      expr(p, lib.path, dest)
+
       add(m.s[cfsVars], p.s(cpsLocals))
       add(m.s[cfsDynLibInit], p.s(cpsInit))
       add(m.s[cfsDynLibInit], p.s(cpsStmts))
@@ -1630,6 +1635,9 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) =
   # deps are allowed (and the system module is processed in the wrong
   # order anyway)
   g.config = config
+  let (outDir, _, _) = splitFile(config.outfile)
+  if not outDir.isEmpty:
+    createDir(outDir)
   if g.generatedHeader != nil: finishModule(g.generatedHeader)
   while g.forwardedProcsCounter > 0:
     for m in cgenModules(g):

compiler/cgendata.nim

@@ -13,7 +13,7 @@ import
   ast, astalgo, ropes, passes, options, intsets, platform, sighashes,
   tables, ndi, lineinfos, pathutils
 
-from modulegraphs import ModuleGraph
+from modulegraphs import ModuleGraph, PPassContext
 
 type
   TLabel* = Rope              # for the C generator a label is just a rope
@@ -132,7 +132,7 @@ type
                             # nimtvDeps is VERY hard to cache because it's
                             # not a list of IDs nor can it be made to be one.
 
-  TCGen = object of TPassContext # represents a C source file
+  TCGen = object of PPassContext # represents a C source file
     s*: TCFileSections        # sections of the C file
     flags*: set[Codegenflag]
     module*: PSym

compiler/cgmeth.nim

@@ -37,10 +37,9 @@ proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
 
 proc getDispatcher*(s: PSym): PSym =
   ## can return nil if is has no dispatcher.
-  let dispn = lastSon(s.ast)
-  if dispn.kind == nkSym:
-    let disp = dispn.sym
-    if sfDispatcher in disp.flags: result = disp
+  if dispatcherPos < s.ast.len:
+    result = s.ast[dispatcherPos].sym
+    doAssert sfDispatcher in result.flags
 
 proc methodCall*(n: PNode; conf: ConfigRef): PNode =
   result = n
@@ -99,13 +98,14 @@ proc sameMethodBucket(a, b: PSym): MethodResult =
         return No
 
 proc attachDispatcher(s: PSym, dispatcher: PNode) =
-  var L = s.ast.len-1
-  var x = s.ast.sons[L]
-  if x.kind == nkSym and sfDispatcher in x.sym.flags:
+  if dispatcherPos < s.ast.len:
     # we've added a dispatcher already, so overwrite it
-    s.ast.sons[L] = dispatcher
+    s.ast.sons[dispatcherPos] = dispatcher
   else:
-    s.ast.add(dispatcher)
+    setLen(s.ast.sons, dispatcherPos+1)
+    if s.ast[resultPos] == nil:
+      s.ast[resultPos] = newNodeI(nkEmpty, s.info)
+    s.ast.sons[dispatcherPos] = dispatcher
 
 proc createDispatcher(s: PSym): PSym =
   var disp = copySym(s)
@@ -165,7 +165,7 @@ proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
     case sameMethodBucket(disp, s)
     of Yes:
       add(g.methods[i].methods, s)
-      attachDispatcher(s, lastSon(disp.ast))
+      attachDispatcher(s, disp.ast[dispatcherPos])
       fixupDispatcher(s, disp, g.config)
       #echo "fixup ", disp.name.s, " ", disp.id
       when useEffectSystem: checkMethodEffects(g, disp, s)

compiler/closureiters.nim

@@ -462,10 +462,17 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       result.typ = n.typ
 
       for i in 0 ..< n.len:
-        if n[i].kind == nkStmtListExpr:
+        case n[i].kind
+        of nkExprColonExpr:
+          if n[i][1].kind == nkStmtListExpr:
+            let (st, ex) = exprToStmtList(n[i][1])
+            result.add(st)
+            n[i][1] = ex
+        of nkStmtListExpr:
           let (st, ex) = exprToStmtList(n[i])
           result.add(st)
           n[i] = ex
+        else: discard
       result.add(n)
 
   of nkIfStmt, nkIfExpr:
@@ -852,15 +859,8 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
       discard
 
     of nkStmtList, nkStmtListExpr:
-      assert(isEmptyType(n.typ), "nkStmtListExpr not lowered")
-
       result = addGotoOut(result, gotoOut)
       for i in 0 ..< n.len:
-        if n[i].hasYieldsInExpressions:
-          # Lower nkStmtListExpr nodes inside `n[i]` first
-          var ns = false
-          n[i] = ctx.lowerStmtListExprs(n[i], ns)
-
         if n[i].hasYields:
           # Create a new split
           let go = newNodeI(nkGotoState, n[i].info)
@@ -1009,7 +1009,7 @@ proc stateFromGotoState(n: PNode): int =
   assert(n.kind == nkGotoState)
   result = n[0].intVal.int
 
-proc tranformStateAssignments(ctx: var Ctx, n: PNode): PNode =
+proc transformStateAssignments(ctx: var Ctx, n: PNode): PNode =
   # This transforms 3 patterns:
   ########################## 1
   # yield e
@@ -1051,7 +1051,7 @@ proc tranformStateAssignments(ctx: var Ctx, n: PNode): PNode =
       result.add(retStmt)
     else:
       for i in 0 ..< n.len:
-        n[i] = ctx.tranformStateAssignments(n[i])
+        n[i] = ctx.transformStateAssignments(n[i])
 
   of nkSkip:
     discard
@@ -1071,7 +1071,7 @@ proc tranformStateAssignments(ctx: var Ctx, n: PNode): PNode =
 
   else:
     for i in 0 ..< n.len:
-      n[i] = ctx.tranformStateAssignments(n[i])
+      n[i] = ctx.transformStateAssignments(n[i])
 
 proc skipStmtList(ctx: Ctx; n: PNode): PNode =
   result = n
@@ -1220,18 +1220,20 @@ proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   # while true:
   #   block :stateLoop:
   #     gotoState :state
+  #     local vars decl (if needed)
   #     body # Might get wrapped in try-except
   let loopBody = newNodeI(nkStmtList, n.info)
   result = newTree(nkWhileStmt, newSymNode(ctx.g.getSysSym(n.info, "true")), loopBody)
   result.info = n.info
 
+  let localVars = newNodeI(nkStmtList, n.info)
   if not ctx.stateVarSym.isNil:
     let varSect = newNodeI(nkVarSection, n.info)
     addVar(varSect, newSymNode(ctx.stateVarSym))
-    loopBody.add(varSect)
+    localVars.add(varSect)
 
     if not ctx.tempVars.isNil:
-      loopBody.add(ctx.tempVars)
+      localVars.add(ctx.tempVars)
 
   let blockStmt = newNodeI(nkBlockStmt, n.info)
   blockStmt.add(newSymNode(ctx.stateLoopLabel))
@@ -1240,7 +1242,7 @@ proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   gs.add(ctx.newStateAccess())
   gs.add(ctx.g.newIntLit(n.info, ctx.states.len - 1))
 
-  var blockBody = newTree(nkStmtList, gs, n)
+  var blockBody = newTree(nkStmtList, gs, localVars, n)
   if ctx.hasExceptions:
     blockBody = ctx.wrapIntoTryExcept(blockBody)
 
@@ -1292,13 +1294,18 @@ proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
     # should folllow the same logic.
     ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), fn, fn.info)
     ctx.stateVarSym.typ = g.createClosureIterStateType(fn)
-
   ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), fn, fn.info)
-  let n = n.toStmtList
+  var n = n.toStmtList
 
   discard ctx.newState(n, nil)
   let gotoOut = newTree(nkGotoState, g.newIntLit(n.info, -1))
 
+  var ns = false
+  n = ctx.lowerStmtListExprs(n, ns)
+
+  if n.hasYieldsInExpressions():
+    internalError(ctx.g.config, "yield in expr not lowered")
+
   # Splitting transformation
   discard ctx.transformClosureIteratorBody(n, gotoOut)
 
@@ -1314,7 +1321,7 @@ proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
     result.add(s)
     result.add(body)
 
-  result = ctx.tranformStateAssignments(result)
+  result = ctx.transformStateAssignments(result)
   result = ctx.wrapIntoStateLoop(result)
 
   # echo "TRANSFORM TO STATES: "

compiler/commands.nim

@@ -48,14 +48,14 @@ const
       "Copyright (c) 2006-" & copyrightYear & " by Andreas Rumpf\n"
 
 const
-  Usage = slurp"../doc/basicopt.txt".replace("//", "")
+  Usage = slurp"../doc/basicopt.txt".replace(" //", " ")
   FeatureDesc = block:
     var x = ""
     for f in low(Feature)..high(Feature):
       if x.len > 0: x.add "|"
       x.add $f
     x
-  AdvancedUsage = slurp"../doc/advopt.txt".replace("//", "") % FeatureDesc
+  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", " ") % FeatureDesc
 
 proc getCommandLineDesc(conf: ConfigRef): string =
   result = (HelpMessage % [VersionAsString, platform.OS[conf.target.hostOS].name,
@@ -769,6 +769,7 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser;
     # nim filename.nims  is the same as "nim e filename.nims":
     if p.key.endswith(".nims"):
       config.command = "e"
+      incl(config.globalOptions, optWasNimscript)
       config.projectName = unixToNativePath(p.key)
       config.arguments = cmdLineRest(p)
       result = true

compiler/depends.nim

@@ -13,10 +13,10 @@ import
   os, options, ast, astalgo, msgs, ropes, idents, passes, modulepaths,
   pathutils
 
-from modulegraphs import ModuleGraph
+from modulegraphs import ModuleGraph, PPassContext
 
 type
-  TGen = object of TPassContext
+  TGen = object of PPassContext
     module: PSym
     config: ConfigRef
     graph: ModuleGraph

compiler/docgen.nim

@@ -40,6 +40,7 @@ type
                     # already. See bug #3655
     destFile*: AbsoluteFile
     thisDir*: AbsoluteDir
+    examples: string
 
   PDoc* = ref TDocumentor ## Alias to type less.
 
@@ -226,18 +227,30 @@ proc genComment(d: PDoc, n: PNode): string =
                                toLinenumber(n.info), toColumn(n.info),
                                dummyHasToc, d.options, d.conf), result)
 
-proc genRecComment(d: PDoc, n: PNode): Rope =
+proc genRecCommentAux(d: PDoc, n: PNode): Rope =
   if n == nil: return nil
   result = genComment(d, n).rope
   if result == nil:
-    if n.kind notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}:
+    if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef,
+                  nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn}:
+      # notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}:
       for i in countup(0, len(n)-1):
-        result = genRecComment(d, n.sons[i])
+        result = genRecCommentAux(d, n.sons[i])
         if result != nil: return
   else:
     when defined(nimNoNilSeqs): n.comment = ""
     else: n.comment = nil
 
+proc genRecComment(d: PDoc, n: PNode): Rope =
+  if n == nil: return nil
+  result = genComment(d, n).rope
+  if result == nil:
+    if n.kind in {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef,
+                  nkMacroDef, nkTemplateDef, nkConverterDef}:
+      result = genRecCommentAux(d, n[bodyPos])
+    else:
+      result = genRecCommentAux(d, n)
+
 proc getPlainDocstring(n: PNode): string =
   ## Gets the plain text docstring of a node non destructively.
   ##
@@ -345,6 +358,14 @@ proc testExample(d: PDoc; ex: PNode) =
       "_examples" & $d.exampleCounter & ".nim"))
   #let nimcache = outp.changeFileExt"" & "_nimcache"
   renderModule(ex, d.filename, outp.string, conf = d.conf)
+  d.examples.add "import r\"" & outp.string & "\"\n"
+
+proc runAllExamples(d: PDoc) =
+  if d.examples.len == 0: return
+  let outputDir = d.conf.getNimcacheDir / RelativeDir"runnableExamples"
+  let outp = outputDir / RelativeFile(extractFilename(d.filename.changeFileExt"" &
+      "_examples.nim"))
+  writeFile(outp, d.examples)
   let backend = if isDefined(d.conf, "js"): "js"
                 elif isDefined(d.conf, "cpp"): "cpp"
                 elif isDefined(d.conf, "objc"): "objc"
@@ -367,6 +388,9 @@ proc extractImports(n: PNode; result: PNode) =
   for i in 0..<n.safeLen: extractImports(n[i], result)
 
 proc prepareExamples(d: PDoc; n: PNode) =
+  var docComment = newTree(nkCommentStmt)
+  let loc = d.conf.toFileLineCol(n.info)
+  docComment.comment = "autogenerated by docgen from " & loc
   var runnableExamples = newTree(nkStmtList,
       newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
   runnableExamples.info = n.info
@@ -413,27 +437,6 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope) =
 proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
   getAllRunnableExamplesRec(d, n, n, dest)
 
-when false:
-  proc findDocComment(n: PNode): PNode =
-    if n == nil: return nil
-    if not isNil(n.comment) and startsWith(n.comment, "##"): return n
-    for i in countup(0, safeLen(n)-1):
-      result = findDocComment(n.sons[i])
-      if result != nil: return
-
-  proc extractDocComment*(s: PSym, d: PDoc): string =
-    let n = findDocComment(s.ast)
-    result = ""
-    if not n.isNil:
-      if not d.isNil:
-        var dummyHasToc: bool
-        renderRstToOut(d[], parseRst(n.comment, toFilename(d.conf, n.info),
-                                     toLinenumber(n.info), toColumn(n.info),
-                                     dummyHasToc, d.options + {roSkipPounds}),
-                       result)
-      else:
-        result = n.comment.substr(2).replace("\n##", "\n").strip
-
 proc isVisible(d: PDoc; n: PNode): bool =
   result = false
   if n.kind == nkPostfix:
@@ -723,7 +726,6 @@ proc exportSym(d: PDoc; s: PSym) =
             rope changeFileExt(external, "html")])
 
 proc generateDoc*(d: PDoc, n, orig: PNode) =
-  if orig.info.fileIndex != n.info.fileIndex: return
   case n.kind
   of nkCommentStmt: add(d.modDesc, genComment(d, n))
   of nkProcDef:
@@ -771,13 +773,13 @@ proc generateDoc*(d: PDoc, n, orig: PNode) =
 proc add(d: PDoc; j: JsonNode) =
   if j != nil: d.jArray.add j
 
-proc generateJson*(d: PDoc, n: PNode) =
+proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
   case n.kind
   of nkCommentStmt:
-    if startsWith(n.comment, "##"):
-      let stripped = n.comment.substr(2).strip
-      d.add %{ "comment": %stripped, "line": %n.info.line.int,
-               "col": %n.info.col }
+    if includeComments:
+      d.add %*{"comment": genComment(d, n)}
+    else:
+      add(d.modDesc, genComment(d, n))
   of nkProcDef:
     when useEffectSystem: documentRaises(d.cache, n)
     d.add genJsonItem(d, n, n.sons[namePos], skProc)
@@ -805,11 +807,11 @@ proc generateJson*(d: PDoc, n: PNode) =
                 succ(skType, ord(n.kind)-ord(nkTypeSection)))
   of nkStmtList:
     for i in countup(0, sonsLen(n) - 1):
-      generateJson(d, n.sons[i])
+      generateJson(d, n.sons[i], includeComments)
   of nkWhenStmt:
     # generate documentation for the first branch only:
     if not checkForFalse(n.sons[0].sons[0]):
-      generateJson(d, lastSon(n.sons[0]))
+      generateJson(d, lastSon(n.sons[0]), includeComments)
   else: discard
 
 proc genTagsItem(d: PDoc, n, nameNode: PNode, k: TSymKind): string =
@@ -891,7 +893,7 @@ proc genOutFile(d: PDoc): Rope =
     setIndexTerm(d[], external, "", title)
   else:
     # Modules get an automatic title for the HTML, but no entry in the index.
-    title = "Module " & extractFilename(changeFileExt(d.filename, ""))
+    title = extractFilename(changeFileExt(d.filename, ""))
 
   let bodyname = if d.hasToc and not d.isPureRst: "doc.body_toc_group"
                  elif d.hasToc: "doc.body_toc"
@@ -923,6 +925,7 @@ proc generateIndex*(d: PDoc) =
     writeIndexFile(d[], dest.string)
 
 proc writeOutput*(d: PDoc, useWarning = false) =
+  runAllExamples(d)
   var content = genOutFile(d)
   if optStdout in d.conf.globalOptions:
     writeRope(stdout, content)
@@ -935,8 +938,13 @@ proc writeOutput*(d: PDoc, useWarning = false) =
         outfile.string)
 
 proc writeOutputJson*(d: PDoc, useWarning = false) =
+  runAllExamples(d)
+  var modDesc: string
+  for desc in d.modDesc:
+    modDesc &= desc
   let content = %*{"orig": d.filename,
     "nimble": getPackageName(d.conf, d.filename),
+    "moduleDescription": modDesc,
     "entries": d.jArray}
   if optStdout in d.conf.globalOptions:
     write(stdout, $content)
@@ -946,7 +954,9 @@ proc writeOutputJson*(d: PDoc, useWarning = false) =
       write(f, $content)
       close(f)
     else:
-      discard "fixme: error report"
+      localError(d.conf, newLineInfo(d.conf, AbsoluteFile d.filename, -1, -1),
+                 warnUser, "unable to open file \"" & d.destFile.string &
+                 "\" for writing")
 
 proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
   var ast = parseFile(conf.projectMainIdx, cache, conf)

compiler/docgen2.nim

@@ -14,10 +14,10 @@ import
   os, options, ast, astalgo, msgs, ropes, idents, passes, docgen, lineinfos,
   pathutils
 
-from modulegraphs import ModuleGraph
+from modulegraphs import ModuleGraph, PPassContext
 
 type
-  TGen = object of TPassContext
+  TGen = object of PPassContext
     doc: PDoc
     module: PSym
   PGen = ref TGen
@@ -55,20 +55,26 @@ proc processNodeJson(c: PPassContext, n: PNode): PNode =
   result = n
   var g = PGen(c)
   if shouldProcess(g):
-    generateJson(g.doc, n)
+    generateJson(g.doc, n, false)
 
-proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
+template myOpenImpl(ext: untyped) {.dirty.} =
   var g: PGen
   new(g)
   g.module = module
   var d = newDocumentor(AbsoluteFile toFullPath(graph.config, FileIndex module.position),
-      graph.cache, graph.config)
+      graph.cache, graph.config, ext)
   d.hasToc = true
   g.doc = d
   result = g
 
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
+  myOpenImpl(HtmlExt)
+
+proc myOpenJson(graph: ModuleGraph; module: PSym): PPassContext =
+  myOpenImpl(JsonExt)
+
 const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
-const docgen2JsonPass* = makePass(open = myOpen, process = processNodeJson,
+const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
                                   close = closeJson)
 
 proc finishDoc2Pass*(project: string) =

compiler/evaltempl.nim

@@ -17,6 +17,7 @@ type
   TemplCtx = object
     owner, genSymOwner: PSym
     instLines: bool   # use the instantiation lines numbers
+    isDeclarative: bool
     mapping: TIdTable # every gensym'ed symbol needs to be mapped to some
                       # new symbol
     config: ConfigRef
@@ -54,11 +55,30 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
       result.add copyNode(c, templ, actual)
   of nkNone..nkIdent, nkType..nkNilLit: # atom
     result.add copyNode(c, templ, actual)
+  of nkCommentStmt:
+    # for the documentation generator we don't keep documentation comments
+    # in the AST that would confuse it (bug #9432), but only if we are not in a
+    # "declarative" context (bug #9235).
+    if c.isDeclarative:
+      var res = copyNode(c, templ, actual)
+      for i in countup(0, sonsLen(templ) - 1):
+        evalTemplateAux(templ.sons[i], actual, c, res)
+      result.add res
+    else:
+      result.add newNodeI(nkEmpty, templ.info)
   else:
+    var isDeclarative = false
+    if templ.kind in {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef,
+                      nkMacroDef, nkTemplateDef, nkConverterDef, nkTypeSection,
+                      nkVarSection, nkLetSection, nkConstSection} and
+        not c.isDeclarative:
+      c.isDeclarative = true
+      isDeclarative = true
     var res = copyNode(c, templ, actual)
     for i in countup(0, sonsLen(templ) - 1):
       evalTemplateAux(templ.sons[i], actual, c, res)
     result.add res
+    if isDeclarative: c.isDeclarative = false
 
 const
   errWrongNumberOfArguments = "wrong number of arguments"

compiler/gorgeimpl.nim

@@ -24,11 +24,11 @@ proc readOutput(p: Process): (string, int) =
 
 proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (string, int) =
   let workingDir = parentDir(toFullPath(conf, info))
-  if cache.len > 0:# and optForceFullMake notin gGlobalOptions:
+  if cache.len > 0:
     let h = secureHash(cmd & "\t" & input & "\t" & cache)
     let filename = toGeneratedFile(conf, AbsoluteFile("gorge_" & $h), "txt").string
     var f: File
-    if open(f, filename):
+    if optForceFullMake notin conf.globalOptions and open(f, filename):
       result = (f.readAll, 0)
       f.close
       return

compiler/importer.nim

@@ -92,6 +92,7 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
         rawImportSymbol(c, e)
         e = nextIdentIter(it, fromMod.tab)
     else: rawImportSymbol(c, s)
+    suggestSym(c.config, n.info, s, c.graph.usageSym, false)
 
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
   var i: TTabIter

compiler/installer.ini

@@ -69,6 +69,7 @@ Files: "doc"
 Files: "doc/html"
 Files: "tools"
 Files: "nimpretty"
+Files: "testament"
 Files: "nimsuggest"
 Files: "nimsuggest/tests/*.nim"
 Files: "web/website.ini"

compiler/jsgen.nim

@@ -34,10 +34,10 @@ import
   times, ropes, math, passes, ccgutils, wordrecg, renderer,
   intsets, cgmeth, lowerings, sighashes, lineinfos, rodutils, pathutils
 
-from modulegraphs import ModuleGraph
+from modulegraphs import ModuleGraph, PPassContext
 
 type
-  TJSGen = object of TPassContext
+  TJSGen = object of PPassContext
     module: PSym
     graph: ModuleGraph
     config: ConfigRef
@@ -476,6 +476,7 @@ proc binaryUintExpr(p: PProc, n: PNode, r: var TCompRes, op: string,
     r.res = "$1 = (($1 $2 $3) $4)" % [x.rdLoc, rope op, y.rdLoc, trimmer]
   else:
     r.res = "(($1 $2 $3) $4)" % [x.rdLoc, rope op, y.rdLoc, trimmer]
+  r.kind = resExpr
 
 proc ternaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
   var x, y, z: TCompRes
@@ -1706,13 +1707,13 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
   of mHigh:
     unaryExpr(p, n, r, "", "($1 != null ? ($1.length-1) : -1)")
   of mInc:
-    if n[1].typ.skipTypes(abstractRange).kind in tyUInt .. tyUInt64:
+    if n[1].typ.skipTypes(abstractRange).kind in {tyUInt..tyUInt64}:
       binaryUintExpr(p, n, r, "+", true)
     else:
       if optOverflowCheck notin p.options: binaryExpr(p, n, r, "", "$1 += $2")
       else: binaryExpr(p, n, r, "addInt", "$1 = addInt($1, $2)")
   of ast.mDec:
-    if n[1].typ.skipTypes(abstractRange).kind in tyUInt .. tyUInt64:
+    if n[1].typ.skipTypes(abstractRange).kind in {tyUInt..tyUInt64}:
       binaryUintExpr(p, n, r, "-", true)
     else:
       if optOverflowCheck notin p.options: binaryExpr(p, n, r, "", "$1 -= $2")
@@ -2282,6 +2283,9 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
         else: AbsoluteFile(getCurrentDir() / m.config.outFile.string)
       else:
         changeFileExt(completeCFilePath(m.config, AbsoluteFile f), ext)
+    let (outDir, _, _) = splitFile(outfile)
+    if not outDir.isEmpty:
+      createDir(outDir)
     discard writeRopeIfNotEqual(genHeader() & code, outfile)
     for obj, content in items(globals.classes):
       genClass(m.config, obj, content, ext)

compiler/lambdalifting.nim

@@ -277,8 +277,9 @@ proc freshVarForClosureIter*(g: ModuleGraph; s, owner: PSym): PNode =
 proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
   let s = n.sym
   if illegalCapture(s):
-    localError(g.config, n.info, "illegal capture '$1' of type <$2> which is declared here: $3" %
-      [s.name.s, typeToString(s.typ), g.config$s.info])
+    localError(g.config, n.info,
+      ("'$1' is of type <$2> which cannot be captured as it would violate memory" &
+       " safety, declared here: $3") % [s.name.s, typeToString(s.typ), g.config$s.info])
   elif owner.typ.callConv notin {ccClosure, ccDefault}:
     localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
       [s.name.s, owner.name.s, CallingConvToStr[owner.typ.callConv]])

compiler/layouter.nim

@@ -28,10 +28,10 @@ type
     config: ConfigRef
     fid: FileIndex
     lastTok: TTokType
-    inquote: bool
+    inquote, lastTokWasTerse: bool
     semicolons: SemicolonKind
-    col, lastLineNumber, lineSpan, indentLevel, indWidth: int
-    nested: int
+    col, lastLineNumber, lineSpan, indentLevel, indWidth*: int
+    keepIndents*: int
     doIndentMore*: int
     content: string
     indentStack: seq[int]
@@ -41,9 +41,10 @@ type
 proc openEmitter*(em: var Emitter, cache: IdentCache;
                   config: ConfigRef, fileIdx: FileIndex) =
   let fullPath = Absolutefile config.toFullPath(fileIdx)
-  em.indWidth = getIndentWidth(fileIdx, llStreamOpen(fullPath, fmRead),
-                               cache, config)
-  if em.indWidth == 0: em.indWidth = 2
+  if em.indWidth == 0:
+    em.indWidth = getIndentWidth(fileIdx, llStreamOpen(fullPath, fmRead),
+                                cache, config)
+    if em.indWidth == 0: em.indWidth = 2
   em.config = config
   em.fid = fileIdx
   em.lastTok = tkInvalid
@@ -52,11 +53,16 @@ proc openEmitter*(em: var Emitter, cache: IdentCache;
   em.content = newStringOfCap(16_000)
   em.indentStack = newSeqOfCap[int](30)
   em.indentStack.add 0
+  em.lastLineNumber = 1
 
 proc closeEmitter*(em: var Emitter) =
+  if fileExists(em.config.outFile) and readFile(em.config.outFile.string) == em.content:
+    discard "do nothing, see #9499"
+    return
   var f = llStreamOpen(em.config.outFile, fmWrite)
   if f == nil:
     rawMessage(em.config, errGenerated, "cannot open file: " & em.config.outFile.string)
+    return
   f.llStreamWrite em.content
   llStreamClose(f)
 
@@ -106,15 +112,16 @@ proc softLinebreak(em: var Emitter, lit: string) =
       for i in 1..em.indentLevel+moreIndent(em): wr(" ")
     else:
       # search backwards for a good split position:
-      for a in em.altSplitPos:
+      for a in mitems(em.altSplitPos):
         if a > em.fixedUntil:
           var spaces = 0
           while a+spaces < em.content.len and em.content[a+spaces] == ' ':
             inc spaces
           if spaces > 0: delete(em.content, a, a+spaces-1)
-          let ws = "\L" & repeat(' ',em.indentLevel+moreIndent(em))
           em.col = em.content.len - a
+          let ws = "\L" & repeat(' ', em.indentLevel+moreIndent(em))
           em.content.insert(ws, a)
+          a = -1
           break
 
 proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
@@ -134,6 +141,22 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
         for i in 1 .. LineCommentColumn - em.col: wr(" ")
     wr lit
 
+  if tok.tokType == tkComment and tok.literal.startsWith("#!nimpretty"):
+    case tok.literal
+    of "#!nimpretty off":
+      inc em.keepIndents
+      wr("\L")
+      em.lastLineNumber = tok.line + 1
+    of "#!nimpretty on":
+      dec em.keepIndents
+      em.lastLineNumber = tok.line
+    wr("\L")
+    #for i in 1 .. tok.indent: wr " "
+    wr tok.literal
+    em.col = 0
+    em.lineSpan = 0
+    return
+
   var preventComment = false
   if tok.tokType == tkComment and tok.line == em.lastLineNumber and tok.indent >= 0:
     # we have an inline comment so handle it before the indentation token:
@@ -142,7 +165,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     em.fixedUntil = em.content.high
 
   elif tok.indent >= 0:
-    if em.lastTok in (splitters + oprSet):
+    if em.lastTok in (splitters + oprSet) or em.keepIndents > 0:
       em.indentLevel = tok.indent
     else:
       if tok.indent > em.indentStack[^1]:
@@ -171,12 +194,13 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
       wr(" ")
     em.fixedUntil = em.content.high
 
+  var lastTokWasTerse = false
   case tok.tokType
   of tokKeywordLow..tokKeywordHigh:
     if endsInAlpha(em):
       wr(" ")
     elif not em.inquote and not endsInWhite(em) and
-        em.lastTok notin openPars:
+        em.lastTok notin openPars and not em.lastTokWasTerse:
       #and tok.tokType in oprSet
       wr(" ")
 
@@ -212,14 +236,19 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
      tkBracketDotRi,
      tkCurlyDotRi,
      tkParDotRi,
-     tkColonColon, tkDot:
+     tkColonColon:
+    wr(TokTypeToStr[tok.tokType])
+  of tkDot:
+    lastTokWasTerse = true
     wr(TokTypeToStr[tok.tokType])
   of tkEquals:
     if not em.inquote and not em.endsInWhite: wr(" ")
     wr(TokTypeToStr[tok.tokType])
     if not em.inquote: wr(" ")
   of tkOpr, tkDotDot:
-    if tok.strongSpaceA == 0 and tok.strongSpaceB == 0:
+    if (tok.strongSpaceA == 0 and tok.strongSpaceB == 0) or em.inquote:
+      # bug #9504: remember to not spacify a keyword:
+      lastTokWasTerse = true
       # if not surrounded by whitespace, don't produce any whitespace either:
       wr(tok.ident.s)
     else:
@@ -229,8 +258,8 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
         tok.strongSpaceB == 0 and tok.strongSpaceA > 0
 
       if not isUnary(tok):
-        wr(" ")
         rememberSplit(splitBinary)
+        wr(" ")
   of tkAccent:
     if not em.inquote and endsInAlpha(em): wr(" ")
     wr(TokTypeToStr[tok.tokType])
@@ -253,6 +282,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
     wr lit
 
   em.lastTok = tok.tokType
+  em.lastTokWasTerse = lastTokWasTerse
   em.lastLineNumber = tok.line + em.lineSpan
   em.lineSpan = 0
 

compiler/lexer.nim

@@ -224,7 +224,7 @@ proc openLexer*(lex: var TLexer, fileIdx: FileIndex, inputstream: PLLStream;
                  cache: IdentCache; config: ConfigRef) =
   openBaseLexer(lex, inputstream)
   lex.fileIdx = fileidx
-  lex.indentAhead = - 1
+  lex.indentAhead = -1
   lex.currLineIndent = 0
   inc(lex.lineNumber, inputstream.lineOffset)
   lex.cache = cache
@@ -740,11 +740,17 @@ proc handleCRLF(L: var TLexer, pos: int): int =
     result = nimlexbase.handleLF(L, pos)
   else: result = pos
 
-proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
+type
+  StringMode = enum
+    normal,
+    raw,
+    generalized
+
+proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
   var pos = L.bufpos
   var buf = L.buf                 # put `buf` in a register
   var line = L.lineNumber         # save linenumber for better error message
-  tokenBegin(tok, pos)
+  tokenBegin(tok, pos - ord(mode == raw))
   inc pos # skip "
   if buf[pos] == '\"' and buf[pos+1] == '\"':
     tok.tokType = tkTripleStrLit # long string literal:
@@ -784,12 +790,12 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
         inc(pos)
   else:
     # ordinary string literal
-    if rawMode: tok.tokType = tkRStrLit
+    if mode != normal: tok.tokType = tkRStrLit
     else: tok.tokType = tkStrLit
     while true:
       var c = buf[pos]
       if c == '\"':
-        if rawMode and buf[pos+1] == '\"':
+        if mode != normal and buf[pos+1] == '\"':
           inc(pos, 2)
           add(tok.literal, '"')
         else:
@@ -800,7 +806,7 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
         tokenEndIgnore(tok, pos)
         lexMessage(L, errGenerated, "closing \" expected")
         break
-      elif (c == '\\') and not rawMode:
+      elif (c == '\\') and mode == normal:
         L.bufpos = pos
         getEscapedChar(L, tok)
         pos = L.bufpos
@@ -1064,8 +1070,10 @@ proc skip(L: var TLexer, tok: var TToken) =
   tok.strongSpaceA = 0
   when defined(nimpretty):
     var hasComment = false
+    var commentIndent = L.currLineIndent
     tok.commentOffsetA = L.offsetBase + pos
     tok.commentOffsetB = tok.commentOffsetA
+    tok.line = -1
   while true:
     case buf[pos]
     of ' ':
@@ -1076,9 +1084,6 @@ proc skip(L: var TLexer, tok: var TToken) =
       inc(pos)
     of CR, LF:
       tokenEndPrevious(tok, pos)
-      when defined(nimpretty):
-        # we are not yet in a comment, so update the comment token's line information:
-        if not hasComment: inc tok.line
       pos = handleCRLF(L, pos)
       buf = L.buf
       var indent = 0
@@ -1087,13 +1092,19 @@ proc skip(L: var TLexer, tok: var TToken) =
           inc(pos)
           inc(indent)
         elif buf[pos] == '#' and buf[pos+1] == '[':
-          when defined(nimpretty): hasComment = true
+          when defined(nimpretty):
+            hasComment = true
+            if tok.line < 0:
+              tok.line = L.lineNumber
+              commentIndent = indent
           skipMultiLineComment(L, tok, pos+2, false)
           pos = L.bufpos
           buf = L.buf
         else:
           break
       tok.strongSpaceA = 0
+      when defined(nimpretty):
+        if buf[pos] == '#' and tok.line < 0: commentIndent = indent
       if buf[pos] > ' ' and (buf[pos] != '#' or buf[pos+1] == '#'):
         tok.indent = indent
         L.currLineIndent = indent
@@ -1101,14 +1112,20 @@ proc skip(L: var TLexer, tok: var TToken) =
     of '#':
       # do not skip documentation comment:
       if buf[pos+1] == '#': break
-      when defined(nimpretty): hasComment = true
+      when defined(nimpretty):
+        hasComment = true
+        if tok.line < 0:
+          tok.line = L.lineNumber
+
       if buf[pos+1] == '[':
         skipMultiLineComment(L, tok, pos+2, false)
         pos = L.bufpos
         buf = L.buf
       else:
         tokenBegin(tok, pos)
-        while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos)
+        while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
+          when defined(nimpretty): tok.literal.add buf[pos]
+          inc(pos)
         tokenEndIgnore(tok, pos+1)
         when defined(nimpretty):
           tok.commentOffsetB = L.offsetBase + pos + 1
@@ -1120,6 +1137,7 @@ proc skip(L: var TLexer, tok: var TToken) =
     if hasComment:
       tok.commentOffsetB = L.offsetBase + pos - 1
       tok.tokType = tkComment
+      tok.indent = commentIndent
     if gIndentationWidth <= 0:
       gIndentationWidth = tok.indent
 
@@ -1168,7 +1186,7 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
     of 'r', 'R':
       if L.buf[L.bufpos + 1] == '\"':
         inc(L.bufpos)
-        getString(L, tok, true)
+        getString(L, tok, raw)
       else:
         getSymbol(L, tok)
     of '(':
@@ -1246,9 +1264,9 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
         lexMessage(L, errGenerated, "invalid token: " & c & " (\\" & $(ord(c)) & ')')
     of '\"':
       # check for generalized raw string literal:
-      var rawMode = L.bufpos > 0 and L.buf[L.bufpos-1] in SymChars
-      getString(L, tok, rawMode)
-      if rawMode:
+      let mode = if L.bufpos > 0 and L.buf[L.bufpos-1] in SymChars: generalized else: normal
+      getString(L, tok, mode)
+      if mode == generalized:
         # tkRStrLit -> tkGStrLit
         # tkTripleStrLit -> tkGTripleStrLit
         inc(tok.tokType, 2)

compiler/lineinfos.nim

@@ -41,7 +41,7 @@ type
     hintLineTooLong, hintXDeclaredButNotUsed, hintConvToBaseNotNeeded,
     hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled,
     hintProcessing, hintCodeBegin, hintCodeEnd, hintConf, hintPath,
-    hintConditionAlwaysTrue, hintName, hintPattern,
+    hintConditionAlwaysTrue, hintConditionAlwaysFalse, hintName, hintPattern,
     hintExecuting, hintLinking, hintDependency,
     hintSource, hintPerformance, hintStackTrace, hintGCStats,
     hintGlobalVar,
@@ -107,6 +107,7 @@ const
     hintConf: "used config file '$1'",
     hintPath: "added path: '$1'",
     hintConditionAlwaysTrue: "condition is always true: '$1'",
+    hintConditionAlwaysFalse: "condition is always false: '$1'",
     hintName: "name should be: '$1'",
     hintPattern: "$1",
     hintExecuting: "$1",
@@ -140,7 +141,7 @@ const
     "Success", "SuccessX", "CC", "LineTooLong",
     "XDeclaredButNotUsed", "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded",
     "ExprAlwaysX", "QuitCalled", "Processing", "CodeBegin", "CodeEnd", "Conf",
-    "Path", "CondTrue", "Name", "Pattern", "Exec", "Link", "Dependency",
+    "Path", "CondTrue", "CondFalse", "Name", "Pattern", "Exec", "Link", "Dependency",
     "Source", "Performance", "StackTrace", "GCStats", "GlobalVar",
     "User", "UserRaw", "ExtendedContext",
   ]

compiler/main.nim

@@ -265,6 +265,7 @@ proc mainCommand*(graph: ModuleGraph) =
     conf.cmd = cmdInteractive
     commandInteractive(graph)
   of "e":
+    incl conf.globalOptions, optWasNimscript
     commandEval(graph, mainCommandArg(conf))
   of "nop", "help":
     # prevent the "success" message:

compiler/modulegraphs.nim

@@ -62,6 +62,19 @@ type
     cacheSeqs*: Table[string, PNode] # state that is shared to suppor the 'macrocache' API
     cacheCounters*: Table[string, BiggestInt]
     cacheTables*: Table[string, BTree[string, PNode]]
+    passes*: seq[TPass]
+
+  TPassContext* = object of RootObj # the pass's context
+  PPassContext* = ref TPassContext
+
+  TPassOpen* = proc (graph: ModuleGraph; module: PSym): PPassContext {.nimcall.}
+  TPassClose* = proc (graph: ModuleGraph; p: PPassContext, n: PNode): PNode {.nimcall.}
+  TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}
+
+  TPass* = tuple[open: TPassOpen,
+                 process: TPassProcess,
+                 close: TPassClose,
+                 isFrontend: bool]
 
 proc hash*(x: FileIndex): Hash {.borrow.}
 

compiler/msgs.nim

@@ -190,10 +190,13 @@ template toFullPath*(conf: ConfigRef; info: TLineInfo): string =
 proc toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
   if info.fileIndex.int32 < 0:
     result = "???"
-  elif optListFullPaths in conf.globalOptions:
-    result = conf.m.fileInfos[info.fileIndex.int32].fullPath.string
+    return
+  let absPath = conf.m.fileInfos[info.fileIndex.int32].fullPath.string
+  if optListFullPaths in conf.globalOptions:
+    result = absPath
   else:
-    result = conf.m.fileInfos[info.fileIndex.int32].projPath.string
+    let relPath = conf.m.fileInfos[info.fileIndex.int32].projPath.string
+    result = if relPath.count("..") > 2: absPath else: relPath
 
 proc toLinenumber*(info: TLineInfo): int {.inline.} =
   result = int info.line
@@ -414,7 +417,7 @@ proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
 proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
   if i.fileIndex.int32 < 0: return ""
 
-  if not optPreserveOrigSource(conf) and conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
+  if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
     try:
       for line in lines(toFullPath(conf, i)):
         addSourceLine conf, i.fileIndex, line.string

compiler/nim.nim

@@ -54,7 +54,8 @@ proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
     of cmdArgument:
       if processArgument(pass, p, argsCount, config): break
   if pass == passCmd2:
-    if optRun notin config.globalOptions and config.arguments.len > 0 and config.command.normalize != "run":
+    if {optRun, optWasNimscript} * config.globalOptions == {} and
+        config.arguments.len > 0 and config.command.normalize notin ["run", "e"]:
       rawMessage(config, errGenerated, errArgsNeedRunOption)
 
 proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =

compiler/options.nim

@@ -45,7 +45,7 @@ type                          # please make sure we have under 32 options
   TOptions* = set[TOption]
   TGlobalOption* = enum       # **keep binary compatible**
     gloptNone, optForceFullMake,
-    optDeadCodeElimUnused,    # deprecated, always on
+    optWasNimscript,
     optListCmd, optCompileOnly, optNoLinking,
     optCDebug,                # turn on debugging information
     optGenDynLib,             # generate a dynamic library

compiler/parser.nim

@@ -46,7 +46,7 @@ type
     inSemiStmtList*: int
     emptyNode: PNode
     when defined(nimpretty2):
-      em: Emitter
+      em*: Emitter
 
   SymbolMode = enum
     smNormal, smAllowNil, smAfterDot
@@ -286,7 +286,7 @@ proc checkBinary(p: TParser) {.inline.} =
 #|
 #| prefixOperator = operator
 #|
-#| optInd = COMMENT?
+#| optInd = COMMENT? IND?
 #| optPar = (IND{>} | IND{=})?
 #|
 #| simpleExpr = arrowExpr (OP0 optInd arrowExpr)* pragma?
@@ -1037,6 +1037,7 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
   addSon(result, p.emptyNode) # return type
   when defined(nimpretty2):
     inc p.em.doIndentMore
+    inc p.em.keepIndents
   let hasParLe = p.tok.tokType == tkParLe and p.tok.indent < 0
   if hasParLe:
     getTok(p)
@@ -1073,6 +1074,7 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
     result = p.emptyNode
   when defined(nimpretty2):
     dec p.em.doIndentMore
+    dec p.em.keepIndents
 
 proc optPragmas(p: var TParser): PNode =
   if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)):
@@ -1578,7 +1580,6 @@ proc parseTry(p: var TParser; isExpr: bool): PNode =
     colcom(p, b)
     addSon(b, parseStmt(p))
     addSon(result, b)
-    if b.kind == nkFinally: break
   if b == nil: parMessage(p, "expected 'except'")
 
 proc parseExceptBlock(p: var TParser, kind: TNodeKind): PNode =

compiler/passaux.nim

@@ -12,10 +12,10 @@
 import
   strutils, ast, astalgo, passes, idents, msgs, options, idgen, lineinfos
 
-from modulegraphs import ModuleGraph
+from modulegraphs import ModuleGraph, PPassContext
 
 type
-  VerboseRef = ref object of TPassContext
+  VerboseRef = ref object of PPassContext
     config: ConfigRef
 
 proc verboseOpen(graph: ModuleGraph; s: PSym): PPassContext =

compiler/passes.nim

@@ -16,21 +16,8 @@ import
   nimsets, syntaxes, times, idgen, modulegraphs, reorder, rod,
   lineinfos, pathutils
 
-
 type
-  TPassContext* = object of RootObj # the pass's context
-
-  PPassContext* = ref TPassContext
-
-  TPassOpen* = proc (graph: ModuleGraph; module: PSym): PPassContext {.nimcall.}
-  TPassClose* = proc (graph: ModuleGraph; p: PPassContext, n: PNode): PNode {.nimcall.}
-  TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}
-
-  TPass* = tuple[open: TPassOpen, process: TPassProcess, close: TPassClose,
-                 isFrontend: bool]
-
   TPassData* = tuple[input: PNode, closeOutput: PNode]
-  TPasses* = openArray[TPass]
 
 # a pass is a tuple of procedure vars ``TPass.close`` may produce additional
 # nodes. These are passed to the other close procedures.
@@ -57,16 +44,12 @@ const
 type
   TPassContextArray = array[0..maxPasses - 1, PPassContext]
 
-var
-  gPasses: array[0..maxPasses - 1, TPass]
-  gPassesLen*: int
-
 proc clearPasses*(g: ModuleGraph) =
-  gPassesLen = 0
+  g.passes.setLen(0)
 
 proc registerPass*(g: ModuleGraph; p: TPass) =
-  gPasses[gPassesLen] = p
-  inc(gPassesLen)
+  internalAssert g.config, g.passes.len < maxPasses
+  g.passes.add(p)
 
 proc carryPass*(g: ModuleGraph; p: TPass, module: PSym;
                 m: TPassData): TPassData =
@@ -76,7 +59,7 @@ proc carryPass*(g: ModuleGraph; p: TPass, module: PSym;
                        else: m.closeOutput
 
 proc carryPasses*(g: ModuleGraph; nodes: PNode, module: PSym;
-                  passes: TPasses) =
+                  passes: openArray[TPass]) =
   var passdata: TPassData
   passdata.input = nodes
   for pass in passes:
@@ -84,23 +67,23 @@ proc carryPasses*(g: ModuleGraph; nodes: PNode, module: PSym;
 
 proc openPasses(g: ModuleGraph; a: var TPassContextArray;
                 module: PSym) =
-  for i in countup(0, gPassesLen - 1):
-    if not isNil(gPasses[i].open):
-      a[i] = gPasses[i].open(g, module)
+  for i in countup(0, g.passes.len - 1):
+    if not isNil(g.passes[i].open):
+      a[i] = g.passes[i].open(g, module)
     else: a[i] = nil
 
 proc closePasses(graph: ModuleGraph; a: var TPassContextArray) =
   var m: PNode = nil
-  for i in countup(0, gPassesLen - 1):
-    if not isNil(gPasses[i].close): m = gPasses[i].close(graph, a[i], m)
+  for i in countup(0, graph.passes.len - 1):
+    if not isNil(graph.passes[i].close): m = graph.passes[i].close(graph, a[i], m)
     a[i] = nil                # free the memory here
 
-proc processTopLevelStmt(n: PNode, a: var TPassContextArray): bool =
+proc processTopLevelStmt(graph: ModuleGraph, n: PNode, a: var TPassContextArray): bool =
   # this implements the code transformation pipeline
   var m = n
-  for i in countup(0, gPassesLen - 1):
-    if not isNil(gPasses[i].process):
-      m = gPasses[i].process(a[i], m)
+  for i in countup(0, graph.passes.len - 1):
+    if not isNil(graph.passes[i].process):
+      m = graph.passes[i].process(a[i], m)
       if isNil(m): return false
   result = true
 
@@ -111,19 +94,19 @@ proc resolveMod(conf: ConfigRef; module, relativeTo: string): FileIndex =
   else:
     result = fileInfoIdx(conf, fullPath)
 
-proc processImplicits(conf: ConfigRef; implicits: seq[string], nodeKind: TNodeKind,
+proc processImplicits(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind,
                       a: var TPassContextArray; m: PSym) =
   # XXX fixme this should actually be relative to the config file!
   let gCmdLineInfo = newLineInfo(FileIndex(0), 1, 1)
-  let relativeTo = toFullPath(conf, m.info)
+  let relativeTo = toFullPath(graph.config, m.info)
   for module in items(implicits):
     # implicit imports should not lead to a module importing itself
-    if m.position != resolveMod(conf, module, relativeTo).int32:
+    if m.position != resolveMod(graph.config, module, relativeTo).int32:
       var importStmt = newNodeI(nodeKind, gCmdLineInfo)
       var str = newStrNode(nkStrLit, module)
       str.info = gCmdLineInfo
       importStmt.addSon str
-      if not processTopLevelStmt(importStmt, a): break
+      if not processTopLevelStmt(graph, importStmt, a): break
 
 const
   imperativeCode = {low(TNodeKind)..high(TNodeKind)} - {nkTemplateDef, nkProcDef, nkMethodDef,
@@ -139,25 +122,25 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
     fileIdx = module.fileIdx
   if module.id < 0:
     # new module caching mechanism:
-    for i in 0..<gPassesLen:
-      if not isNil(gPasses[i].open) and not gPasses[i].isFrontend:
-        a[i] = gPasses[i].open(graph, module)
+    for i in 0 ..< graph.passes.len:
+      if not isNil(graph.passes[i].open) and not graph.passes[i].isFrontend:
+        a[i] = graph.passes[i].open(graph, module)
       else:
         a[i] = nil
 
     if not graph.stopCompile():
       let n = loadNode(graph, module)
       var m = n
-      for i in 0..<gPassesLen:
-        if not isNil(gPasses[i].process) and not gPasses[i].isFrontend:
-          m = gPasses[i].process(a[i], m)
+      for i in 0 ..< graph.passes.len:
+        if not isNil(graph.passes[i].process) and not graph.passes[i].isFrontend:
+          m = graph.passes[i].process(a[i], m)
           if isNil(m):
             break
 
     var m: PNode = nil
-    for i in 0..<gPassesLen:
-      if not isNil(gPasses[i].close) and not gPasses[i].isFrontend:
-        m = gPasses[i].close(graph, a[i], m)
+    for i in 0 ..< graph.passes.len:
+      if not isNil(graph.passes[i].close) and not graph.passes[i].isFrontend:
+        m = graph.passes[i].close(graph, a[i], m)
       a[i] = nil
   else:
     openPasses(graph, a, module)
@@ -177,8 +160,8 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
         # modules to include between compilation runs? we'd need to track that
         # in ROD files. I think we should enable this feature only
         # for the interactive mode.
-        processImplicits graph.config, graph.config.implicitImports, nkImportStmt, a, module
-        processImplicits graph.config, graph.config.implicitIncludes, nkIncludeStmt, a, module
+        processImplicits graph, graph.config.implicitImports, nkImportStmt, a, module
+        processImplicits graph, graph.config.implicitIncludes, nkIncludeStmt, a, module
 
       while true:
         if graph.stopCompile(): break
@@ -194,7 +177,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
             sl.add n
           if sfReorder in module.flags:
             sl = reorder(graph, sl, module)
-          discard processTopLevelStmt(sl, a)
+          discard processTopLevelStmt(graph, sl, a)
           break
         elif n.kind in imperativeCode:
           # read everything until the next proc declaration etc.
@@ -208,13 +191,13 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
               break
             sl.add n
           #echo "-----\n", sl
-          if not processTopLevelStmt(sl, a): break
+          if not processTopLevelStmt(graph, sl, a): break
           if rest != nil:
             #echo "-----\n", rest
-            if not processTopLevelStmt(rest, a): break
+            if not processTopLevelStmt(graph, rest, a): break
         else:
           #echo "----- single\n", n
-          if not processTopLevelStmt(n, a): break
+          if not processTopLevelStmt(graph, n, a): break
       closeParsers(p)
       if s.kind != llsStdIn: break
     closePasses(graph, a)

compiler/pathutils.nim

@@ -99,24 +99,26 @@ proc isDotDot(x: string; bounds: (int, int)): bool =
 proc isSlash(x: string; bounds: (int, int)): bool =
   bounds[1] == bounds[0] and x[bounds[0]] in {DirSep, AltSep}
 
+const canonDirSep = when isMainModule: '/' else: DirSep
+
 proc canon(x: string; result: var string; state: var int) =
   # state: 0th bit set if isAbsolute path. Other bits count
   # the number of path components.
   for b in dirs(x):
     if (state shr 1 == 0) and isSlash(x, b):
-      result.add DirSep
+      result.add canonDirSep
       state = state or 1
     elif result.len > (state and 1) and isDotDot(x, b):
       var d = result.len
       # f/..
-      while d > (state and 1) and result[d-1] != DirSep:
+      while (d-1) > (state and 1) and result[d-1] notin {DirSep, AltSep}:
         dec d
       if d > 0: setLen(result, d-1)
     elif isDot(x, b):
       discard "discard the dot"
     elif b[1] >= b[0]:
-      if result.len > 0 and result[^1] != DirSep:
-        result.add DirSep
+      if result.len > 0 and result[^1] notin {DirSep, AltSep}:
+        result.add canonDirSep
       result.add substr(x, b[0], b[1])
     inc state, 2
 
@@ -133,7 +135,7 @@ when FileSystemCaseSensitive:
 else:
   template `!=?`(a, b: char): bool = a != b
 
-proc relativeTo(full, base: string; sep = DirSep): string =
+proc relativeTo(full, base: string; sep = canonDirSep): string =
   if full.len == 0: return ""
   var f, b: PathIter
   var ff = (0, -1)
@@ -207,7 +209,7 @@ when true:
     canon(f.string, result.string, state)
 
   proc relativeTo*(fullPath: AbsoluteFile, baseFilename: AbsoluteDir;
-                   sep = DirSep): RelativeFile =
+                   sep = canonDirSep): RelativeFile =
     RelativeFile(relativeTo(fullPath.string, baseFilename.string, sep))
 
   proc toAbsolute*(file: string; base: AbsoluteDir): AbsoluteFile =
@@ -222,7 +224,7 @@ when true:
 
   proc writeFile*(x: AbsoluteFile; content: string) {.borrow.}
 
-when isMainModule and defined(posix):
+when isMainModule:
   doAssert canon"/foo/../bar" == "/bar"
   doAssert canon"foo/../bar" == "bar"
 
@@ -257,4 +259,4 @@ when isMainModule and defined(posix):
 
 when isMainModule and defined(windows):
   let nasty = string(AbsoluteDir(r"C:\Users\rumpf\projects\nim\tests\nimble\nimbleDir\linkedPkgs\pkgB-#head\../../simplePkgs/pkgB-#head/") / RelativeFile"pkgA/module.nim")
-  doAssert nasty == r"C:\Users\rumpf\projects\nim\tests\nimble\nimbleDir\simplePkgs\pkgB-#head\pkgA\module.nim"
+  doAssert nasty.replace('/', '\\') == r"C:\Users\rumpf\projects\nim\tests\nimble\nimbleDir\simplePkgs\pkgB-#head\pkgA\module.nim"

compiler/pragmas.nim

@@ -1104,10 +1104,19 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         else: sym.flags.incl sfUsed
       of wLiftLocals: discard
       else: invalidPragma(c, it)
-    elif sym.kind in {skVar,skLet,skParam,skField,skProc,skFunc,skConverter,skMethod,skType}:
+    elif sym != nil and sym.kind in {skVar, skLet, skParam, skField, skProc,
+                                     skFunc, skConverter, skMethod, skType}:
       n.sons[i] = semCustomPragma(c, it)
-    else:
+    elif sym != nil:
       illegalCustomPragma(c, it, sym)
+    else:
+      invalidPragma(c, it)
+
+proc mergePragmas(n, pragmas: PNode) =
+  if n[pragmasPos].kind == nkEmpty:
+    n[pragmasPos] = pragmas
+  else:
+    for p in pragmas: n.sons[pragmasPos].add p
 
 proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
                       validPragmas: TSpecialWords) =
@@ -1117,11 +1126,12 @@ proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
       if not o.isNil:
         pushInfoContext(c.config, n.info)
         var i = 0
-        while i < o.len():
+        while i < o.len:
           if singlePragma(c, sym, o, i, validPragmas):
             internalError(c.config, n.info, "implicitPragmas")
           inc i
         popInfoContext(c.config)
+        if sym.kind in routineKinds and sym.ast != nil: mergePragmas(sym.ast, o)
 
     if lfExportLib in sym.loc.flags and sfExportc notin sym.flags:
       localError(c.config, n.info, ".dynlib requires .exportc")

compiler/rodimpl.nim

@@ -853,12 +853,10 @@ proc loadNode*(g: ModuleGraph; module: PSym): PNode =
   result = newNodeI(nkStmtList, module.info)
   for row in db.rows(sql"select data from toplevelstmts where module = ? order by position asc",
                         abs module.id):
-
     var b = BlobReader(pos: 0)
     # ensure we can read without index checks:
     b.s = row[0] & '\0'
     result.add decodeNode(g, b, module.info)
-
   db.exec(sql"insert into controlblock(idgen) values (?)", gFrontEndId)
   replay(g, module, result)
 

compiler/scriptconfig.nim

@@ -117,6 +117,7 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
   cbconf setCommand:
     conf.command = a.getString 0
     let arg = a.getString 1
+    incl(conf.globalOptions, optWasNimscript)
     if arg.len > 0:
       conf.projectName = arg
       let path =
@@ -159,11 +160,8 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile;
 
   defineSymbol(conf.symbols, "nimscript")
   defineSymbol(conf.symbols, "nimconfig")
-  var registeredPasses {.global.} = false
-  if not registeredPasses:
-    registerPass(graph, semPass)
-    registerPass(graph, evalPass)
-    registeredPasses = true
+  registerPass(graph, semPass)
+  registerPass(graph, evalPass)
 
   conf.searchPaths.add(conf.libpath)
 

compiler/sem.nim

@@ -18,7 +18,7 @@ import
   evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity,
   semparallel, lowerings, pluginsupport, plugins/active, rod, lineinfos
 
-from modulegraphs import ModuleGraph
+from modulegraphs import ModuleGraph, PPassContext
 
 when defined(nimfix):
   import nimfix/prettybase

compiler/semfold.nim

@@ -379,7 +379,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode =
       result = newStrNodeT(s, n, g)
     else:
       result = newStrNodeT(getStrOrChar(a), n, g)
-  of mStrToStr: result = a
+  of mStrToStr: result = newStrNodeT(getStrOrChar(a), n, g)
   of mEnumToStr: result = newStrNodeT(ordinalValToString(a, g), n, g)
   of mArrToSeq:
     result = copyTree(a)

compiler/semmagic.nim

@@ -299,7 +299,13 @@ proc semOf(c: PContext, n: PNode): PNode =
         result.typ = getSysType(c.graph, n.info, tyBool)
         return result
       elif diff == high(int):
-        localError(c.config, n.info, "'$1' cannot be of this subtype" % typeToString(a))
+        if commonSuperclass(a, b) == nil:
+          localError(c.config, n.info, "'$1' cannot be of this subtype" % typeToString(a))
+        else:
+          message(c.config, n.info, hintConditionAlwaysFalse, renderTree(n))
+          result = newIntNode(nkIntLit, 0)
+          result.info = n.info
+          result.typ = getSysType(c.graph, n.info, tyBool)
   else:
     localError(c.config, n.info, "'of' takes 2 arguments")
   n.typ = getSysType(c.graph, n.info, tyBool)

compiler/semstmts.nim

@@ -207,6 +207,8 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags): PNode =
   typ = commonType(typ, n[0].typ)
 
   var last = sonsLen(n) - 1
+  var catchAllExcepts = 0
+
   for i in countup(1, last):
     let a = n.sons[i]
     checkMinSonsLen(a, 1, c.config)
@@ -227,8 +229,16 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags): PNode =
         # Overwrite symbol in AST with the symbol in the symbol table.
         a[0][2] = newSymNode(symbol, a[0][2].info)
 
+      elif a.len == 1:
+          # count number of ``except: body`` blocks
+          inc catchAllExcepts
+
       else:
         # support ``except KeyError, ValueError, ... : body``
+        if catchAllExcepts > 0:
+          # if ``except: body`` already encountered,
+          # cannot be followed by a ``except KeyError, ... : body`` block
+          inc catchAllExcepts
         var is_native, is_imported: bool
         for j in 0..a.len-2:
           let tmp = semExceptBranchType(a[j])
@@ -238,9 +248,18 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags): PNode =
         if is_native and is_imported:
           localError(c.config, a[0].info, "Mix of imported and native exception types is not allowed in one except branch")
 
-    elif a.kind != nkFinally:
+    elif a.kind == nkFinally:
+      if i != n.len-1:
+        localError(c.config, a.info, "Only one finally is allowed after all other branches")
+
+    else:
       illFormedAst(n, c.config)
 
+    if catchAllExcepts > 1:
+      # if number of ``except: body`` blocks is greater than 1
+      # or more specific exception follows a general except block, it is invalid
+      localError(c.config, a.info, "Only one general except clause is allowed after more specific exceptions")
+
     # last child of an nkExcept/nkFinally branch is a statement:
     a[^1] = semExprBranchScope(c, a[^1])
     if a.kind != nkFinally: typ = commonType(typ, a[^1])
@@ -804,11 +823,12 @@ proc semRaise(c: PContext, n: PNode): PNode =
   checkSonsLen(n, 1, c.config)
   if n[0].kind != nkEmpty:
     n[0] = semExprWithType(c, n[0])
-    let typ = n[0].typ
+    var typ = n[0].typ
     if not isImportedException(typ, c.config):
-      if typ.kind != tyRef or typ.lastSon.kind != tyObject:
+      typ = typ.skipTypes({tyAlias, tyGenericInst})
+      if typ.kind != tyRef:
         localError(c.config, n.info, errExprCannotBeRaised)
-      if typ.len > 0 and not isException(typ.lastSon):
+      if not isException(typ.lastSon):
         localError(c.config, n.info, "raised object of type $1 does not inherit from Exception",
                           [typeToString(typ)])
 
@@ -1917,7 +1937,8 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
         case n.sons[j].kind
         of nkPragma, nkCommentStmt, nkNilLit, nkEmpty, nkBlockExpr,
             nkBlockStmt, nkState: discard
-        else: localError(c.config, n.sons[j].info, "unreachable statement after 'return'")
+        else: localError(c.config, n.sons[j].info,
+          "unreachable statement after 'return' statement or '{.noReturn.}' proc")
     else: discard
 
   if result.len == 1 and

compiler/semtypes.nim

@@ -725,8 +725,8 @@ proc addInheritedFieldsAux(c: PContext, check: var IntSet, pos: var int,
       of nkOfBranch, nkElse:
         addInheritedFieldsAux(c, check, pos, lastSon(n.sons[i]))
       else: internalError(c.config, n.info, "addInheritedFieldsAux(record case branch)")
-  of nkRecList:
-    for i in countup(0, sonsLen(n) - 1):
+  of nkRecList, nkRecWhen, nkElifBranch, nkElse:
+    for i in 0 ..< sonsLen(n):
       addInheritedFieldsAux(c, check, pos, n.sons[i])
   of nkSym:
     incl(check, n.sym.name.id)
@@ -972,7 +972,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
       if lifted != nil: paramType.sons[i] = lifted
 
     let body = paramType.base
-    if body.kind == tyForward:
+    if body.kind in {tyForward, tyError}:
       # this may happen for proc type appearing in a type section
       # before one of its param types
       return

compiler/semtypinst.nim

@@ -67,11 +67,10 @@ proc cacheTypeInst*(inst: PType) =
   #      update the refcount
   let gt = inst.sons[0]
   let t = if gt.kind == tyGenericBody: gt.lastSon else: gt
-  if t.kind in {tyStatic, tyGenericParam} + tyTypeClasses:
+  if t.kind in {tyStatic, tyError, tyGenericParam} + tyTypeClasses:
     return
   gt.sym.typeInstCache.safeAdd(inst)
 
-
 type
   LayeredIdTable* = object
     topLayer*: TIdTable
@@ -336,6 +335,9 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
     # but we already raised an error!
     rawAddSon(result, header.sons[i])
 
+  if body.kind == tyError:
+    return
+
   let bbody = lastSon body
   var newbody = replaceTypeVarsT(cl, bbody)
   let bodyIsNew = newbody != bbody

compiler/sighashes.nim

@@ -196,18 +196,23 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) =
       else:
         c.hashSym(t.sym)
       if {sfAnon, sfGenSym} * t.sym.flags != {}:
-        # generated object names can be identical, so we need to
-        # disambiguate furthermore by hashing the field types and names:
-        # mild hack to prevent endless recursions (makes nimforum compile again):
-        let oldFlags = t.sym.flags
-        t.sym.flags = t.sym.flags - {sfAnon, sfGenSym}
-        let n = t.n
-        for i in 0 ..< n.len:
-          assert n[i].kind == nkSym
-          let s = n[i].sym
-          c.hashSym s
-          c.hashType s.typ, flags
-        t.sym.flags = oldFlags
+        # Generated object names can be identical, so we need to
+        # disambiguate furthermore by hashing the field types and names.
+        if t.n.len > 0:
+          let oldFlags = t.sym.flags
+          # Mild hack to prevent endless recursion.
+          t.sym.flags = t.sym.flags - {sfAnon, sfGenSym}
+          for n in t.n:
+            assert(n.kind == nkSym)
+            let s = n.sym
+            c.hashSym s
+            c.hashType s.typ, flags
+          t.sym.flags = oldFlags
+        else:
+          # The object has no fields: we _must_ add something here in order to
+          # make the hash different from the one we produce by hashing only the
+          # type name.
+          c &= ".empty"
     else:
       c &= t.id
     if t.len > 0 and t.sons[0] != nil:

compiler/sigmatch.nim

@@ -1292,7 +1292,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
   of tyDistinct:
     if a.kind == tyDistinct:
       if sameDistinctTypes(f, a): result = isEqual
-      elif f.base.kind == tyAnything: result = isGeneric
+      #elif f.base.kind == tyAnything: result = isGeneric  # issue 4435
       elif c.coerceDistincts: result = typeRel(c, f.base, a)
     elif a.kind == tyNil and f.base.kind in NilableTypes:
       result = f.allowsNil
@@ -1911,10 +1911,13 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType,
     else:
       var evaluated = c.semTryConstExpr(c, arg)
       if evaluated != nil:
-        arg.typ = newTypeS(tyStatic, c)
-        arg.typ.sons = @[evaluated.typ]
-        arg.typ.n = evaluated
-        a = arg.typ
+        # Don't build the type in-place because `evaluated` and `arg` may point
+        # to the same object and we'd end up creating recursive types (#9255)
+        let typ = newTypeS(tyStatic, c)
+        typ.sons = @[evaluated.typ]
+        typ.n = evaluated
+        arg.typ = typ
+        a = typ
       else:
         if m.callee.kind == tyGenericBody:
           if f.kind == tyStatic and typeRel(m, f.base, a) != isNone:
@@ -2236,8 +2239,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
 
       if n.sons[a].kind == nkHiddenStdConv:
         doAssert n.sons[a].sons[0].kind == nkEmpty and
-                 n.sons[a].sons[1].kind == nkArgList and
-                 n.sons[a].sons[1].len == 0
+                 n.sons[a].sons[1].kind in {nkBracket, nkArgList}
         # Steal the container and pass it along
         setSon(m.call, formal.position + 1, n.sons[a].sons[1])
       else:

compiler/syntaxes.nim

@@ -72,12 +72,16 @@ proc parsePipe(filename: AbsoluteFile, inputStream: PLLStream; cache: IdentCache
       i = 0
       inc linenumber
     if i+1 < line.len and line[i] == '#' and line[i+1] == '?':
-      inc(i, 2)
-      while i < line.len and line[i] in Whitespace: inc(i)
-      var q: TParser
-      parser.openParser(q, filename, llStreamOpen(substr(line, i)), cache, config)
-      result = parser.parseAll(q)
-      parser.closeParser(q)
+      when defined(nimpretty2):
+        # XXX this is a bit hacky, but oh well...
+        quit "can't nimpretty a source code filter"
+      else:
+        inc(i, 2)
+        while i < line.len and line[i] in Whitespace: inc(i)
+        var q: TParser
+        parser.openParser(q, filename, llStreamOpen(substr(line, i)), cache, config)
+        result = parser.parseAll(q)
+        parser.closeParser(q)
     llStreamClose(s)
 
 proc getFilter(ident: PIdent): TFilterKind =
@@ -156,14 +160,18 @@ proc openParsers*(p: var TParsers, fileIdx: FileIndex, inputstream: PLLStream;
 proc closeParsers*(p: var TParsers) =
   parser.closeParser(p.parser)
 
-proc parseFile*(fileIdx: FileIndex; cache: IdentCache; config: ConfigRef): PNode {.procvar.} =
-  var
-    p: TParsers
-    f: File
+proc setupParsers*(p: var TParsers; fileIdx: FileIndex; cache: IdentCache;
+                   config: ConfigRef): bool =
+  var f: File
   let filename = toFullPathConsiderDirty(config, fileIdx)
   if not open(f, filename.string):
     rawMessage(config, errGenerated, "cannot open file: " & filename.string)
-    return
+    return false
   openParsers(p, fileIdx, llStreamOpen(f), cache, config)
-  result = parseAll(p)
-  closeParsers(p)
+  result = true
+
+proc parseFile*(fileIdx: FileIndex; cache: IdentCache; config: ConfigRef): PNode {.procvar.} =
+  var p: TParsers
+  if setupParsers(p, fileIdx, cache, config):
+    result = parseAll(p)
+    closeParsers(p)

compiler/transf.nim

@@ -38,7 +38,7 @@ type
                               # if we encounter the 2nd yield statement
     next: PTransCon           # for stacking
 
-  TTransfContext = object of passes.TPassContext
+  TTransfContext = object of TPassContext
     module: PSym
     transCon: PTransCon      # top of a TransCon stack
     inlining: int            # > 0 if we are in inlining context (copy vars)
@@ -110,8 +110,8 @@ proc transformSons(c: PTransf, n: PNode): PTransNode =
   for i in countup(0, sonsLen(n)-1):
     result[i] = transform(c, n.sons[i])
 
-proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode =
-  result = newTransNode(nkFastAsgn, PNode(ri).info, 2)
+proc newAsgnStmt(c: PTransf, kind: TNodeKind, le: PNode, ri: PTransNode): PTransNode =
+  result = newTransNode(kind, PNode(ri).info, 2)
   result[0] = PTransNode(le)
   result[1] = ri
 
@@ -299,12 +299,6 @@ proc transformBreak(c: PTransf, n: PNode): PTransNode =
   else:
     result = n.PTransNode
 
-proc unpackTuple(c: PTransf, n: PNode, father: PTransNode) =
-  # XXX: BUG: what if `n` is an expression with side-effects?
-  for i in countup(0, sonsLen(c.transCon.forStmt) - 3):
-    add(father, newAsgnStmt(c, c.transCon.forStmt.sons[i],
-        transform(c, newTupleAccess(c.graph, n, i))))
-
 proc introduceNewLocalVars(c: PTransf, n: PNode): PTransNode =
   case n.kind
   of nkSym:
@@ -328,6 +322,18 @@ proc introduceNewLocalVars(c: PTransf, n: PNode): PTransNode =
       result[i] = introduceNewLocalVars(c, n.sons[i])
 
 proc transformYield(c: PTransf, n: PNode): PTransNode =
+  proc asgnTo(lhs: PNode, rhs: PTransNode): PTransNode =
+    # Choose the right assignment instruction according to the given ``lhs``
+    # node since it may not be a nkSym (a stack-allocated skForVar) but a
+    # nkDotExpr (a heap-allocated slot into the envP block)
+    case lhs.kind:
+    of nkSym:
+      internalAssert c.graph.config, lhs.sym.kind == skForVar
+      result = newAsgnStmt(c, nkFastAsgn, lhs, rhs)
+    of nkDotExpr:
+      result = newAsgnStmt(c, nkAsgn, lhs, rhs)
+    else:
+      internalAssert c.graph.config, false
   result = newTransNode(nkStmtList, n.info, 0)
   var e = n.sons[0]
   # c.transCon.forStmt.len == 3 means that there is one for loop variable
@@ -340,13 +346,20 @@ proc transformYield(c: PTransf, n: PNode): PTransNode =
       for i in countup(0, sonsLen(e) - 1):
         var v = e.sons[i]
         if v.kind == nkExprColonExpr: v = v.sons[1]
-        add(result, newAsgnStmt(c, c.transCon.forStmt.sons[i],
-                                transform(c, v)))
+        let lhs = c.transCon.forStmt.sons[i]
+        let rhs = transform(c, v)
+        add(result, asgnTo(lhs, rhs))
     else:
-      unpackTuple(c, e, result)
+      # Unpack the tuple into the loop variables
+      # XXX: BUG: what if `n` is an expression with side-effects?
+      for i in countup(0, sonsLen(c.transCon.forStmt) - 3):
+        let lhs = c.transCon.forStmt.sons[i]
+        let rhs = transform(c, newTupleAccess(c.graph, e, i))
+        add(result, asgnTo(lhs, rhs))
   else:
-    var x = transform(c, e)
-    add(result, newAsgnStmt(c, c.transCon.forStmt.sons[0], x))
+    let lhs = c.transCon.forStmt.sons[0]
+    let rhs = transform(c, e)
+    add(result, asgnTo(lhs, rhs))
 
   inc(c.transCon.yieldStmts)
   if c.transCon.yieldStmts <= 1:
@@ -584,7 +597,7 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
       # generate a temporary and produce an assignment statement:
       var temp = newTemp(c, formal.typ, formal.info)
       addVar(v, temp)
-      add(stmtList, newAsgnStmt(c, temp, arg.PTransNode))
+      add(stmtList, newAsgnStmt(c, nkFastAsgn, temp, arg.PTransNode))
       idNodeTablePut(newC.mapping, formal, temp)
     of paVarAsgn:
       assert(skipTypes(formal.typ, abstractInst).kind == tyVar)
@@ -595,7 +608,7 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
       addSonSkipIntLit(typ, formal.typ.sons[0])
       var temp = newTemp(c, typ, formal.info)
       addVar(v, temp)
-      add(stmtList, newAsgnStmt(c, temp, arg.PTransNode))
+      add(stmtList, newAsgnStmt(c, nkFastAsgn, temp, arg.PTransNode))
       idNodeTablePut(newC.mapping, formal, temp)
 
   var body = iter.getBody.copyTree

compiler/trees.nim

@@ -92,8 +92,7 @@ proc isCaseObj*(n: PNode): bool =
 
 proc isDeepConstExpr*(n: PNode): bool =
   case n.kind
-  of nkCharLit..nkInt64Lit, nkStrLit..nkTripleStrLit,
-      nkFloatLit..nkFloat64Lit, nkNilLit:
+  of nkCharLit..nkNilLit:
     result = true
   of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv:
     result = isDeepConstExpr(n.sons[1])

compiler/types.nim

@@ -130,6 +130,7 @@ proc elemType*(t: PType): PType =
   case t.kind
   of tyGenericInst, tyDistinct, tyAlias, tySink: result = elemType(lastSon(t))
   of tyArray: result = t.sons[1]
+  of tyError: result = t
   else: result = t.lastSon
   assert(result != nil)
 
@@ -1079,8 +1080,8 @@ proc inheritanceDiff*(a, b: PType): int =
   # | returns: +x iff `a` is the x'th direct subclass of `b`
   # | returns: `maxint` iff `a` and `b` are not compatible at all
   if a == b or a.kind == tyError or b.kind == tyError: return 0
-  assert a.kind == tyObject
-  assert b.kind == tyObject
+  assert a.kind in {tyObject} + skipPtrs
+  assert b.kind in {tyObject} + skipPtrs
   var x = a
   result = 0
   while x != nil:

compiler/vm.nim

@@ -24,7 +24,7 @@ import
 from semfold import leValueConv, ordinalValToString
 from evaltempl import evalTemplate
 
-from modulegraphs import ModuleGraph
+from modulegraphs import ModuleGraph, PPassContext
 
 when hasFFI:
   import evalffi
@@ -413,26 +413,12 @@ proc recSetFlagIsRef(arg: PNode) =
     arg.sons[i].recSetFlagIsRef
 
 proc setLenSeq(c: PCtx; node: PNode; newLen: int; info: TLineInfo) =
-  # FIXME: this doesn't attempt to solve incomplete
-  # support of tyPtr, tyRef in VM.
   let typ = node.typ.skipTypes(abstractInst+{tyRange}-{tyTypeDesc})
-  let typeEntry = typ.sons[0].skipTypes(abstractInst+{tyRange}-{tyTypeDesc})
-  let typeKind = case typeEntry.kind
-                 of tyUInt..tyUInt64: nkUIntLit
-                 of tyRange, tyEnum, tyBool, tyChar, tyInt..tyInt64: nkIntLit
-                 of tyFloat..tyFloat128: nkFloatLit
-                 of tyString: nkStrLit
-                 of tyObject: nkObjConstr
-                 of tySequence: nkNilLit
-                 of tyProc, tyTuple: nkTupleConstr
-                 else: nkEmpty
-
   let oldLen = node.len
   setLen(node.sons, newLen)
   if oldLen < newLen:
-    # TODO: This is still not correct for tyPtr, tyRef default value
     for i in oldLen ..< newLen:
-      node.sons[i] = newNodeI(typeKind, info)
+      node.sons[i] = getNullValue(typ.sons[0], info, c.config)
 
 const
   errIndexOutOfBounds = "index out of bounds"
@@ -458,7 +444,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     #if c.traceActive:
     when traceCode:
       echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra, " rb ", instr.regB, " rc ", instr.regC
-    #  message(c.config, c.debug[pc], warnUser, "Trace")
+      # message(c.config, c.debug[pc], warnUser, "Trace")
 
     case instr.opcode
     of opcEof: return regs[ra]
@@ -917,7 +903,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         stackTrace(c, tos, pc, errNilAccess)
     of opcGetImpl:
       decodeB(rkNode)
-      let a = regs[rb].node
+      var a = regs[rb].node
+      if a.kind == nkVarTy: a = a[0]
       if a.kind == nkSym:
         regs[ra].node = if a.sym.ast.isNil: newNode(nkNilLit)
                         else: copyTree(a.sym.ast)
@@ -1201,7 +1188,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcOf:
       decodeBC(rkInt)
       let typ = c.types[regs[rc].intVal.int]
-      regs[ra].intVal = ord(inheritanceDiff(regs[rb].node.typ, typ) >= 0)
+      regs[ra].intVal = ord(inheritanceDiff(regs[rb].node.typ, typ) <= 0)
     of opcIs:
       decodeBC(rkInt)
       let t1 = regs[rb].node.typ.skipTypes({tyTypeDesc})
@@ -1215,8 +1202,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       let newLen = regs[rb].intVal.int
       if regs[ra].node.isNil: stackTrace(c, tos, pc, errNilAccess)
       else: c.setLenSeq(regs[ra].node, newLen, c.debug[pc])
-    of opcReset:
-      internalError(c.config, c.debug[pc], "too implement")
     of opcNarrowS:
       decodeB(rkInt)
       let min = -(1.BiggestInt shl (rb-1))
@@ -1226,6 +1211,11 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcNarrowU:
       decodeB(rkInt)
       regs[ra].intVal = regs[ra].intVal and ((1'i64 shl rb)-1)
+    of opcSignExtend:
+      # like opcNarrowS, but no out of range possible
+      decodeB(rkInt)
+      let imm = 64 - rb
+      regs[ra].intVal = ashr(regs[ra].intVal shl imm, imm)
     of opcIsNil:
       decodeB(rkInt)
       let node = regs[rb].node

compiler/vmdef.nim

@@ -69,8 +69,9 @@ type
     opcContainsSet, opcRepr, opcSetLenStr, opcSetLenSeq,
     opcIsNil, opcOf, opcIs,
     opcSubStr, opcParseFloat, opcConv, opcCast,
-    opcQuit, opcReset,
+    opcQuit,
     opcNarrowS, opcNarrowU,
+    opcSignExtend,
 
     opcAddStrCh,
     opcAddStrStr,
@@ -191,7 +192,7 @@ type
   VmCallback* = proc (args: VmArgs) {.closure.}
 
   PCtx* = ref TCtx
-  TCtx* = object of passes.TPassContext # code gen context
+  TCtx* = object of TPassContext # code gen context
     code*: seq[TInstr]
     debug*: seq[TLineInfo]  # line info for every instruction; kept separate
                             # to not slow down interpretation

compiler/vmdeps.nim

@@ -237,8 +237,15 @@ proc mapTypeToAstX(cache: IdentCache; t: PType; info: TLineInfo;
   of tyRange:
     result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
     result.add atomicType("range", mRange)
-    result.add t.n.sons[0].copyTree
-    result.add t.n.sons[1].copyTree
+    if inst:
+      let rng = newNodeX(nkInfix)
+      rng.add newIdentNode(getIdent(cache, ".."), info)
+      rng.add t.n.sons[0].copyTree
+      rng.add t.n.sons[1].copyTree
+      result.add rng
+    else:
+      result.add t.n.sons[0].copyTree
+      result.add t.n.sons[1].copyTree
   of tyPointer: result = atomicType("pointer", mPointer)
   of tyString: result = atomicType("string", mString)
   of tyCString: result = atomicType("cstring", mCString)

compiler/vmgen.nim

@@ -361,16 +361,28 @@ proc genIf(c: PCtx, n: PNode; dest: var TDest) =
   for endPos in endings: c.patch(endPos)
   c.clearDest(n, dest)
 
+proc isTemp(c: PCtx; dest: TDest): bool =
+  result = dest >= 0 and c.prc.slots[dest].kind >= slotTempUnknown
+
 proc genAndOr(c: PCtx; n: PNode; opc: TOpcode; dest: var TDest) =
   #   asgn dest, a
   #   tjmp|fjmp L1
   #   asgn dest, b
   # L1:
-  if dest < 0: dest = getTemp(c, n.typ)
-  c.gen(n.sons[1], dest)
-  let L1 = c.xjmp(n, opc, dest)
-  c.gen(n.sons[2], dest)
+  let copyBack = dest < 0 or not isTemp(c, dest)
+  let tmp = if copyBack:
+              getTemp(c, n.typ)
+            else:
+              TRegister dest
+  c.gen(n.sons[1], tmp)
+  let L1 = c.xjmp(n, opc, tmp)
+  c.gen(n.sons[2], tmp)
   c.patch(L1)
+  if dest < 0:
+    dest = tmp
+  elif copyBack:
+    c.gABC(n, opcAsgnInt, dest, tmp)
+    freeTemp(c, tmp)
 
 proc canonValue*(n: PNode): PNode =
   result = n
@@ -938,11 +950,18 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     c.freeTemp(tmp)
     c.freeTemp(tmp2)
 
-  of mShlI: genBinaryABCnarrowU(c, n, dest, opcShlInt)
-  of mAshrI: genBinaryABCnarrow(c, n, dest, opcAshrInt)
-  of mBitandI: genBinaryABCnarrowU(c, n, dest, opcBitandInt)
-  of mBitorI: genBinaryABCnarrowU(c, n, dest, opcBitorInt)
-  of mBitxorI: genBinaryABCnarrowU(c, n, dest, opcBitxorInt)
+  of mShlI:
+    genBinaryABC(c, n, dest, opcShlInt)
+    # genNarrowU modified
+    let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
+    if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8):
+      c.gABC(n, opcNarrowU, dest, TRegister(t.size*8))
+    elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and t.size < 8):
+      c.gABC(n, opcSignExtend, dest, TRegister(t.size*8))
+  of mAshrI: genBinaryABC(c, n, dest, opcAshrInt)
+  of mBitandI: genBinaryABC(c, n, dest, opcBitandInt)
+  of mBitorI: genBinaryABC(c, n, dest, opcBitorInt)
+  of mBitxorI: genBinaryABC(c, n, dest, opcBitxorInt)
   of mAddU: genBinaryABCnarrowU(c, n, dest, opcAddu)
   of mSubU: genBinaryABCnarrowU(c, n, dest, opcSubu)
   of mMulU: genBinaryABCnarrowU(c, n, dest, opcMulu)
@@ -961,7 +980,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mLtPtr, mLtU, mLtU64: genBinaryABC(c, n, dest, opcLtu)
   of mEqProc, mEqRef, mEqUntracedRef:
     genBinaryABC(c, n, dest, opcEqRef)
-  of mXor: genBinaryABCnarrowU(c, n, dest, opcXor)
+  of mXor: genBinaryABC(c, n, dest, opcXor)
   of mNot: genUnaryABC(c, n, dest, opcNot)
   of mUnaryMinusI, mUnaryMinusI64:
     genUnaryABC(c, n, dest, opcUnaryMinusInt)
@@ -970,7 +989,10 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mUnaryPlusI, mUnaryPlusF64: gen(c, n.sons[1], dest)
   of mBitnotI:
     genUnaryABC(c, n, dest, opcBitnotInt)
-    genNarrowU(c, n, dest)
+    #genNarrowU modified, do not narrow signed types
+    let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
+    if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8):
+      c.gABC(n, opcNarrowU, dest, TRegister(t.size*8))
   of mToFloat, mToBiggestFloat, mToInt,
      mToBiggestInt, mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr,
      mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr:
@@ -1063,13 +1085,15 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mReset:
     unused(c, n, dest)
     var d = c.genx(n.sons[1])
-    c.gABC(n, opcReset, d)
+    c.gABx(n, opcLdNull, d, c.genType(n.sons[1].typ))
+    c.gABx(n, opcNodeToReg, d, d)
+    c.genAsgnPatch(n.sons[1], d)
   of mOf, mIs:
     if dest < 0: dest = c.getTemp(n.typ)
     var tmp = c.genx(n.sons[1])
     var idx = c.getTemp(getSysType(c.graph, n.info, tyInt))
     var typ = n.sons[2].typ
-    if m == mOf: typ = typ.skipTypes(abstractPtrs-{tyTypeDesc})
+    if m == mOf: typ = typ.skipTypes(abstractPtrs)
     c.gABx(n, opcLdImmInt, idx, c.genType(typ))
     c.gABC(n, if m == mOf: opcOf else: opcIs, dest, tmp, idx)
     c.freeTemp(tmp)
@@ -1088,13 +1112,14 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mEcho:
     unused(c, n, dest)
     let n = n[1].skipConv
-    let x = c.getTempRange(n.len, slotTempUnknown)
-    internalAssert c.config, n.kind == nkBracket
-    for i in 0..<n.len:
-      var r: TRegister = x+i
-      c.gen(n.sons[i], r)
-    c.gABC(n, opcEcho, x, n.len)
-    c.freeTempRange(x, n.len)
+    if n.kind == nkBracket:
+      # can happen for nim check, see bug #9609
+      let x = c.getTempRange(n.len, slotTempUnknown)
+      for i in 0..<n.len:
+        var r: TRegister = x+i
+        c.gen(n.sons[i], r)
+      c.gABC(n, opcEcho, x, n.len)
+      c.freeTempRange(x, n.len)
   of mAppendStrCh:
     unused(c, n, dest)
     genBinaryStmtVar(c, n, opcAddStrCh)
@@ -1383,9 +1408,6 @@ proc checkCanEval(c: PCtx; n: PNode) =
                   skIterator} and sfForward in s.flags:
     cannotEval(c, n)
 
-proc isTemp(c: PCtx; dest: TDest): bool =
-  result = dest >= 0 and c.prc.slots[dest].kind >= slotTempUnknown
-
 template needsAdditionalCopy(n): untyped =
   not c.isTemp(dest) and not fitsRegister(n.typ)
 
@@ -1648,8 +1670,9 @@ proc genVarSection(c: PCtx; n: PNode) =
     #assert(a.sons[0].kind == nkSym) can happen for transformed vars
     if a.kind == nkVarTuple:
       for i in 0 .. a.len-3:
-        if not a[i].sym.isGlobal: setSlot(c, a[i].sym)
-        checkCanEval(c, a[i])
+        if a[i].kind == nkSym:
+          if not a[i].sym.isGlobal: setSlot(c, a[i].sym)
+          checkCanEval(c, a[i])
       c.gen(lowerTupleUnpacking(c.graph, a, c.getOwner))
     elif a.sons[0].kind == nkSym:
       let s = a.sons[0].sym

config/nim.cfg

@@ -73,6 +73,19 @@ path="$lib/pure"
   opt:speed
 @end
 
+@if unix and mingw:
+  # Cross compile for Windows from Linux/OSX using MinGW
+  os = windows
+
+  i386.windows.gcc.path = "/usr/bin"
+  i386.windows.gcc.exe = "i686-w64-mingw32-gcc"
+  i386.windows.gcc.linkerexe = "i686-w64-mingw32-gcc"
+
+  amd64.windows.gcc.path = "/usr/bin"
+  amd64.windows.gcc.exe = "x86_64-w64-mingw32-gcc"
+  amd64.windows.gcc.linkerexe = "x86_64-w64-mingw32-gcc"
+@end
+
 @if unix:
   @if not bsd or haiku:
     # -fopenmp
@@ -89,9 +102,6 @@ path="$lib/pure"
     tlsEmulation:on
   @end
   @if haiku:
-    # Haiku currently have problems with TLS
-    # https://dev.haiku-os.org/ticket/14342
-    tlsEmulation:on
     gcc.options.linker = "-Wl,--as-needed -lnetwork"
     gcc.cpp.options.linker = "-Wl,--as-needed -lnetwork"
     clang.options.linker = "-Wl,--as-needed -lnetwork"

config/nimdoc.cfg

@@ -38,7 +38,7 @@ doc.section.toc = """
 # * $seeSrc: generated HTML from doc.item.seesrc (if some switches are used).
 
 doc.item = """
-<a id="$itemSymOrIdEnc"></a>
+<a id="$itemSymOrID"></a>
 <dt><pre>$header</pre></dt>
 <dd>
 $desc

doc/backends.rst

@@ -268,7 +268,7 @@ form the Nim code, then link them into a static binary along your main C
 program::
 
   $ nim c --noMain --noLinking --header:fib.h fib.nim
-  $ gcc -o m -Inimcache -Ipath/to/nim/lib nimcache/*.c maths.c
+  $ gcc -o m -I$HOME/.cache/nim/fib_d -Ipath/to/nim/lib $HOME/.cache/nim/fib_d/*.c maths.c
 
 The first command runs the Nim compiler with three special options to avoid
 generating a ``main()`` function in the generated files, avoid linking the

doc/contributing.rst

@@ -1,3 +1,20 @@
+============
+Contributing
+============
+
+.. contents::
+
+
+Contributing happens via "Pull requests" (PR) on github. Every PR needs to be
+reviewed before it can be merged and the Continuous Integration should be green.
+
+The PR has to be approved (and is often merged too) by one "code owner", either
+by the code owner who is responsible for the subsystem the PR belongs to or by
+two core developers or by Araq.
+
+See `codeowners <codeowners.html>`_ for more details.
+
+
 Writing tests
 =============
 
@@ -78,12 +95,27 @@ only want to see the output of failing tests, go for
 
 You can also run only a single category of tests. A category is a subdirectory
 in the ``tests`` directory. There are a couple of special categories; for a
-list of these, see ``tests/testament/categories.nim``, at the bottom.
+list of these, see ``testament/categories.nim``, at the bottom.
 
 ::
 
   ./koch tests c lib
 
+To run a single test:
+
+::
+
+  ./koch tests c <category>/<name>
+
+E.g. ``./koch test run stdlib/thttpclient_ssl``
+
+For reproducible tests (to reproduce an environment more similar to the one
+run by Continuous Integration on travis/appveyor), you may want to disable your
+local configuration (eg in ``~/.config/nim/nim.cfg``) which may affect some
+tests; this can also be achieved by using
+``export XDG_CONFIG_HOME=pathtoAlternateConfig`` before running ``./koch``
+commands.
+
 Comparing tests
 ===============
 
@@ -195,13 +227,100 @@ or
 
 the first is preferred.
 
+Best practices
+=============
+
+Note: these are general guidelines, not hard rules; there are always exceptions.
+Code reviews can just point to a specific section here to save time and
+propagate best practices.
+
+.. _noimplicitbool:
+Take advantage of no implicit bool conversion
+
+.. code-block:: nim
+
+  doAssert isValid() == true
+  doAssert isValid() # preferred
+
+.. _immediately_invoked_lambdas:
+Immediately invoked lambdas (https://en.wikipedia.org/wiki/Immediately-invoked_function_expression)
+
+.. code-block:: nim
+
+  let a = (proc (): auto = getFoo())()
+  let a = block:  # preferred
+    getFoo()
+
+.. _design_for_mcs:
+Design with method call syntax (UFCS in other languages) chaining in mind
+
+.. code-block:: nim
+
+  proc foo(cond: bool, lines: seq[string]) # bad
+  proc foo(lines: seq[string], cond: bool) # preferred
+  # can be called as: `getLines().foo(false)`
+
+.. _avoid_quit:
+Use exceptions (including assert / doAssert) instead of ``quit``
+rationale: https://forum.nim-lang.org/t/4089
+
+.. code-block:: nim
+
+  quit() # bad in almost all cases
+  doAssert() # preferred
+
+.. _tests_use_doAssert:
+Use ``doAssert`` (or ``require``, etc), not ``assert`` in all tests.
+
+.. code-block:: nim
+
+  runnableExamples: assert foo() # bad
+  runnableExamples: doAssert foo() # preferred
+
+.. _delegate_printing:
+Delegate printing to caller: return ``string`` instead of calling ``echo``
+rationale: it's more flexible (eg allows caller to call custom printing,
+including prepending location info, writing to log files, etc).
+
+.. code-block:: nim
+
+  proc foo() = echo "bar" # bad
+  proc foo(): string = "bar" # preferred (usually)
+
+.. _use_Option:
+[Ongoing debate] Consider using Option instead of return bool + var argument,
+unless stack allocation is needed (eg for efficiency).
+
+.. code-block:: nim
+
+  proc foo(a: var Bar): bool
+  proc foo(): Option[Bar]
+
+.. _use_doAssert_not_echo:
+Tests (including in testament) should always prefer assertions over ``echo``,
+except when that's not possible. It's more precise, easier for readers and
+maintaners to where expected values refer to. See for example
+https://github.com/nim-lang/Nim/pull/9335 and https://forum.nim-lang.org/t/4089
+
+.. code-block:: nim
+
+  echo foo() # adds a line in testament `discard` block.
+  doAssert foo() == [1, 2] # preferred, except when not possible to do so.
+
 The Git stuff
 =============
 
 General commit rules
 --------------------
 
-1. All changes introduced by the commit (diff lines) must be related to the
+1. Bugfixes that should be backported to the latest stable release should
+   contain the string ``[backport]`` in the commit message! There will be an
+   outmated process relying on these. However, bugfixes also have the inherent
+   risk of causing regressions which are worse for a "stable, bugfixes-only"
+   branch, so in doubt, leave out the ``[backport]``. Standard library bugfixes
+   are less critical than compiler bugfixes.
+
+2. All changes introduced by the commit (diff lines) must be related to the
    subject of the commit.
 
    If you change some other unrelated to the subject parts of the file, because
@@ -211,7 +330,7 @@ General commit rules
    *Tip:* Never commit everything as is using ``git commit -a``, but review
    carefully your changes with ``git add -p``.
 
-2. Changes should not introduce any trailing whitespace.
+3. Changes should not introduce any trailing whitespace.
 
    Always check your changes for whitespace errors using ``git diff --check``
    or add following ``pre-commit`` hook:
@@ -221,6 +340,52 @@ General commit rules
       #!/bin/sh
       git diff --check --cached || exit $?
 
-3. Describe your commit and use your common sense.
+4. Describe your commit and use your common sense.
+
+   Example Commit messages: ``Fixes #123; refs #124``
+
+   indicates that issue ``#123`` is completely fixed (github may automatically
+   close it when the PR is committed), wheres issue ``#124`` is referenced
+   (eg: partially fixed) and won't close the issue when committed.
+
+5. Commits should be always be rebased against devel (so a fast forward
+   merge can happen)
+
+   eg: use ``git pull --rebase origin devel``. This is to avoid messing up
+   git history, see `#8664 <https://github.com/nim-lang/Nim/issues/8664>`_ .
+   Exceptions should be very rare: when rebase gives too many conflicts, simply
+   squash all commits using the script shown in
+   https://github.com/nim-lang/Nim/pull/9356
+
+
+6. Do not mix pure formatting changes (eg whitespace changes, nimpretty) or
+   automated changes (eg nimfix) with other code changes: these should be in
+   separate commits (and the merge on github should not squash these into 1).
+
+
+Continuous Integration (CI)
+---------------------------
+
+1. Continuous Integration is by default run on every push in a PR; this clogs
+   the CI pipeline and affects other PR's; if you don't need it (eg for WIP or
+   documentation only changes), add ``[ci skip]`` to your commit message title.
+   This convention is supported by `Appveyor <https://www.appveyor.com/docs/how-to/filtering-commits/#skip-directive-in-commit-message>`_
+   and `Travis <https://docs.travis-ci.com/user/customizing-the-build/#skipping-a-build>`_
+
+
+2. Consider enabling CI (travis and appveyor) in your own Nim fork, and
+   waiting for CI to be green in that fork (fixing bugs as needed) before
+   opening your PR in original Nim repo, so as to reduce CI congestion. Same
+   applies for updates on a PR: you can test commits on a separate private
+   branch before updating the main PR.
+
+Code reviews
+------------
+
+1. Whenever possible, use github's new 'Suggested change' in code reviews, which
+   saves time explaining the change or applying it; see also
+   https://forum.nim-lang.org/t/4317
 
 .. include:: docstyle.rst
+
+

doc/docgen.rst

@@ -117,6 +117,7 @@ Output::
   {
     "orig": "docgen_sample.nim",
     "nimble": "",
+    "moduleDescription": "This module is a sample",
     "entries": [
       {
         "name": "helloWorld",

doc/docs.rst

@@ -6,6 +6,9 @@ The documentation consists of several documents:
 - | `Tutorial (part II) <tut2.html>`_
   | The Nim tutorial part two deals with the advanced language constructs.
 
+- | `Tutorial (part III) <tut3.html>`_
+  | The Nim tutorial part three about Nim's macro system.
+
 - | `Language Manual <manual.html>`_
   | The Nim manual is a draft that will evolve into a proper specification.
 

doc/filters.rst

@@ -4,7 +4,7 @@ Source Code Filters
 
 .. contents::
 
-A `Source Code Filter` transforms the input character stream to an in-memory
+A `Source Code Filter (SCF)`  transforms the input character stream to an in-memory
 output stream before parsing. A filter can be used to provide templating
 systems or preprocessors.
 
@@ -23,10 +23,23 @@ just like an ordinary procedure call with named or positional arguments. The
 available parameters depend on the invoked filter. Before version 0.12.0 of
 the language ``#!`` was used instead of ``#?``.
 
-**Hint:** With ``--hint[codeBegin]:on```or ``--verbosity:2``
-(or higher) Nim lists the processed code after each filter
-application.
+**Hint:** With ``--hint[codeBegin]:on`` or ``--verbosity:2``
+(or higher) while compiling or `nim check`, Nim lists the processed code after
+each filter application.
 
+Usage
+=====
+
+First, put your SCF code in a separate file with filters specified in the first line. 
+**Note:** You can name your SCF file with any file extension you want, but the conventional extension is ``.tmpl``.
+
+If we use `generateXML` code shown above and call the SCF file `xmlGen.tmpl`
+In your `main.nim`:
+
+.. code-block:: nim
+  include "xmlGen.tmpl"
+  
+  echo generateXML("John Smith","42")
 
 Pipe operator
 =============

doc/grammar.txt

@@ -9,7 +9,7 @@ operator =  OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9
          | 'is' | 'isnot' | 'in' | 'notin' | 'of'
          | 'div' | 'mod' | 'shl' | 'shr' | 'not' | 'static' | '..'
 prefixOperator = operator
-optInd = COMMENT?
+optInd = COMMENT? IND?
 optPar = (IND{>} | IND{=})?
 simpleExpr = arrowExpr (OP0 optInd arrowExpr)* pragma?
 arrowExpr = assignExpr (OP1 optInd assignExpr)*
@@ -85,17 +85,19 @@ paramListColon = paramList? (':' optInd typeDesc)?
 doBlock = 'do' paramListArrow pragmas? colcom stmt
 procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)?
 distinct = 'distinct' optInd typeDesc
+forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt
+forExpr = forStmt
 expr = (blockExpr
       | ifExpr
       | whenExpr
       | caseExpr
+      | forExpr
       | tryExpr)
       / simpleExpr
 typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple'
          | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum'
 primary = typeKeyw typeDescK
         /  prefixOperator* identOrLiteral primarySuffix*
-        / 'static' primary
         / 'bind' primary
 typeDesc = simpleExpr
 typeDefAux = simpleExpr
@@ -142,12 +144,11 @@ tryExpr = 'try' colcom stmt &(optInd 'except'|'finally')
            (optInd 'except' exprList colcom stmt)*
            (optInd 'finally' colcom stmt)?
 exceptBlock = 'except' colcom stmt
-forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt
 blockStmt = 'block' symbol? colcom stmt
 blockExpr = 'block' symbol? colcom stmt
 staticStmt = 'static' colcom stmt
 deferStmt = 'defer' colcom stmt
-asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLE_STR_LIT)
+asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLESTR_LIT)
 genericParam = symbol (comma symbol)* (colon expr)? ('=' optInd expr)?
 genericParamList = '[' optInd
   genericParam ^* (comma/semicolon) optPar ']'

doc/intern.rst

@@ -16,9 +16,9 @@ Directory structure
 
 The Nim project's directory structure is:
 
-============   ==============================================
+============   ===================================================
 Path           Purpose
-============   ==============================================
+============   ===================================================
 ``bin``        generated binary files
 ``build``      generated C code for the installation
 ``compiler``   the Nim compiler itself; note that this
@@ -32,7 +32,7 @@ Path           Purpose
 ``lib``        the Nim library
 ``web``        website of Nim; generated by ``nimweb``
                from the ``*.txt`` and ``*.tmpl`` files
-============   ==============================================
+============   ===================================================
 
 
 Bootstrapping the compiler
@@ -386,8 +386,9 @@ So now ``G.c`` MUST contain both ``P1`` and ``P2``, but we haven't even
 loaded ``P1`` from the symbol file, nor do we want to because we then quickly
 would restore large parts of the whole program.
 
+
 Solution
-~~~~~~~~ 
+~~~~~~~~
 
 The backend must have some logic so that if the currently processed module
 is from the compilation cache, the ``ast`` field is not accessed. Instead
@@ -517,6 +518,7 @@ Code generation for closures
 
 Code generation for closures is implemented by `lambda lifting`:idx:.
 
+
 Design
 ------
 
@@ -665,5 +667,3 @@ important the hidden formal param is ``void*`` and not something more
 specialized. However the more specialized env type needs to passed to the
 backend somehow. We deal with this by modifying ``s.ast[paramPos]`` to contain
 the formal hidden parameter, but not ``s.typ``!
-
-

doc/koch.rst

@@ -61,7 +61,7 @@ test command
 ------------
 
 The `test`:idx: command can also be invoked with the alias ``tests``. This
-command will compile and run ``tests/testament/tester.nim``, which is the main
+command will compile and run ``testament/tester.nim``, which is the main
 driver of Nim's test suite. You can pass options to the ``test`` command,
 they will be forwarded to the tester. See its source code for available
 options.

doc/lib.rst

@@ -18,9 +18,9 @@ low-level interface to a C library.
 
 Read this `document <apis.html>`_ for a quick overview of the API design.
 
-The `bottom <#nimble>`_ of this page includes a list of 3rd party packages
-created by the Nim community. These packages are a useful addition to the
-modules in the standard library.
+In addition to the modules in the standard library, third-party packages
+created by the Nim community can be used via `Nimble <#nimble>`_, Nim's
+package manager.
 
 
 Pure libraries
@@ -59,7 +59,7 @@ Core
   This module defines compile-time reflection procs for working with types.
 
 * `threadpool <threadpool.html>`_
-  Implements Nim's `spawn <manual.html#spawn>`_.
+  Implements Nim's `spawn <manual.html#parallel-amp-spawn>`_.
 
 * `cpuinfo <cpuinfo.html>`_
   This module implements procs to determine the number of CPUs / cores.
@@ -67,6 +67,8 @@ Core
 * `lenientops <lenientops.html>`_
   Provides binary operators for mixed integer/float expressions for convenience.
 
+* `bitops <bitops.html>`_
+  Provides a series of low level methods for bit manipulation.
 
 
 Collections and algorithms
@@ -74,26 +76,38 @@ Collections and algorithms
 
 * `algorithm <algorithm.html>`_
   Implements some common generic algorithms like sort or binary search.
+
 * `tables <tables.html>`_
   Nim hash table support. Contains tables, ordered tables and count tables.
+
 * `sets <sets.html>`_
   Nim hash and bit set support.
+
 * `lists <lists.html>`_
   Nim linked list support. Contains singly and doubly linked lists and
   circular lists ("rings").
+
 * `deques <deques.html>`_
   Implementation of a double-ended queue.
   The underlying implementation uses a ``seq``.
+
+* `heapqueue <heapqueue.html>`_
+  This module implements Heap queue (a.k.a. priority queue) algorithm.
+
 * `intsets <intsets.html>`_
   Efficient implementation of a set of ints as a sparse bit set.
+
 * `critbits <critbits.html>`_
   This module implements a *crit bit tree* which is an efficient
   container for a sorted set of strings, or for a sorted mapping of strings.
+
 * `sequtils <sequtils.html>`_
   This module implements operations for the built-in seq type
   which were inspired by functional programming languages.
+
 * `sharedtables <sharedtables.html>`_
   Nim shared hash table support. Contains shared tables.
+
 * `sharedlist <sharedlist.html>`_
   Nim shared linked list support. Contains shared singly linked list.
 
@@ -107,8 +121,8 @@ String handling
   substrings, replacing substrings.
 
 * `strformat <strformat.html>`_
-  Macro based standard string interpolation / formatting. Inpired by
-  Python's ```f``-strings.
+  Macro based standard string interpolation / formatting. Inspired by
+  Python's ``f``-strings.
 
 * `strmisc <strmisc.html>`_
   This module contains uncommon string handling operations that do not
@@ -129,6 +143,13 @@ String handling
 * `unicode <unicode.html>`_
   This module provides support to handle the Unicode UTF-8 encoding.
 
+* `unidecode <unidecode.html>`_
+  It provides a single proc that does Unicode to ASCII transliterations.
+  Based on Python's Unidecode module.
+
+* `punycode <punycode.html>`_
+  Implements a representation of Unicode with the limited ASCII character subset.
+
 * `encodings <encodings.html>`_
   Converts between different character encodings. On UNIX, this uses
   the ``iconv`` library, on Windows the Windows API.
@@ -189,10 +210,16 @@ Generic Operating System Services
   ``asyncdispatch``.
 
 * `distros <distros.html>`_
-  This module implements the basics for OS distribution ("distro") detection and the OS's native package manager.
-  Its primary purpose is to produce output for Nimble packages, but it also contains the widely used **Distribution** enum
+  This module implements the basics for OS distribution ("distro") detection
+  and the OS's native package manager.
+  Its primary purpose is to produce output for Nimble packages,
+  but it also contains the widely used **Distribution** enum
   that is useful for writing platform specific code.
 
+* `volatile <volatile.html>`_
+  This module contains code for generating volatile loads and stores,
+  which are useful in embedded and systems programming.
+
 
 Math libraries
 --------------
@@ -274,6 +301,7 @@ Internet Protocols and Support
   This module implements a selector API with backends specific to each OS.
   Currently epoll on Linux and select on other operating systems.
 
+
 Parsers
 -------
 
@@ -406,6 +434,7 @@ Miscellaneous
 * `segfaults <segfaults.html>`_
   Turns access violations or segfaults into a ``NilAccessError`` exception.
 
+
 Modules for JS backend
 ----------------------
 
@@ -434,7 +463,6 @@ Regular expressions
   expressions. The current implementation uses PCRE.
 
 
-
 Database support
 ----------------
 
@@ -465,6 +493,7 @@ Wrappers
 The generated HTML for some of these wrappers is so huge that it is
 not contained in the distribution. You can then find them on the website.
 
+
 Windows specific
 ----------------
 
@@ -520,4 +549,5 @@ Nimble is a package manager for the Nim programming language.
 For instructions on how to install Nimble packages see
 `its README <https://github.com/nim-lang/nimble#readme>`_.
 
-To see a list of Nimble's packages, check out `https://nimble.directory/ <https://nimble.directory/>`_ or the `packages repos <https://github.com/nim-lang/packages>`_ on GitHub.
+To see a list of Nimble's packages, check out `<https://nimble.directory/>`_
+or the `packages repo <https://github.com/nim-lang/packages>`_ on GitHub.

doc/manual.rst

@@ -475,7 +475,6 @@ The type suffixes are:
   ``'d``             float64
   ``'f32``           float32
   ``'f64``           float64
-  ``'f128``          float128
 =================    =========================
 
 Floating point literals may also be in binary, octal or hexadecimal
@@ -3037,7 +3036,7 @@ The `case expression` is again very similar to the case statement:
 
 As seen in the above example, the case expression can also introduce side
 effects. When multiple statements are given for a branch, Nim will use
-the last expression as the result value, much like in an `expr` template.
+the last expression as the result value.
 
 Table constructor
 -----------------
@@ -4611,8 +4610,8 @@ The concept types can be parametric just like the regular generic types:
     M.data[m * M.N + n] = v
 
   # Adapt the Matrix type to the concept's requirements
-  template Rows*(M: type Matrix): expr = M.M
-  template Cols*(M: type Matrix): expr = M.N
+  template Rows*(M: type Matrix): int = M.M
+  template Cols*(M: type Matrix): int = M.N
   template ValueType*(M: type Matrix): type = M.T
 
   -------------
@@ -5036,9 +5035,8 @@ an ``immediate`` pragma and then these templates do not take part in
 overloading resolution and the parameters' types are *ignored* by the
 compiler. Explicit immediate templates are now deprecated.
 
-**Note**: For historical reasons ``stmt`` is an alias for ``typed`` and
-``expr`` an alias for ``untyped``, but new code should use the newer,
-clearer names.
+**Note**: For historical reasons ``stmt`` was an alias for ``typed`` and
+``expr`` was an alias for ``untyped``, but they are removed.
 
 
 Passing a code block to a template
@@ -5129,9 +5127,6 @@ also ``varargs[untyped]`` so that not even the number of parameters is fixed:
 However, since a template cannot iterate over varargs, this feature is
 generally much more useful for macros.
 
-**Note**: For historical reasons ``varargs[expr]`` is not equivalent
-to ``varargs[untyped]``.
-
 
 Symbol binding in templates
 ---------------------------
@@ -5965,6 +5960,12 @@ curlies is the pattern to match against. The operators ``*``,  ``**``,
 notation, so to match verbatim against ``*`` the ordinary function call syntax
 needs to be used.
 
+Term rewriting macro are applied recursively, up to a limit. This means that
+if the result of a term rewriting macro is eligible for another rewriting,
+the compiler will try to perform it, and so on, until no more optimizations
+are applicable. To avoid putting the compiler into an infinite loop, there is
+a hard limit on how many times a single term rewriting macro can be applied.
+Once this limit has been passed, the term rewriting macro will be ignored.
 
 Unfortunately optimizations are hard to get right and even the tiny example
 is **wrong**:
@@ -8003,7 +8004,7 @@ To enable thread support the ``--threads:on`` command line switch needs to
 be used. The ``system`` module then contains several threading primitives.
 See the `threads <threads.html>`_ and `channels <channels.html>`_ modules
 for the low level thread API. There are also high level parallelism constructs
-available. See `spawn <#parallel-spawn>`_ for further details.
+available. See `spawn <#parallel-amp-spawn>`_ for further details.
 
 Nim's memory model for threads is quite different than that of other common
 programming languages (C, Pascal, Java): Each thread has its own (garbage

doc/nimc.rst

@@ -157,7 +157,7 @@ passed as a command line argument to the compiler.
 The ``nim`` executable processes configuration files in the following
 directories (in this order; later files overwrite previous settings):
 
-1) ``$nim/config/nim.cfg``, ``/etc/nim/nim.cfg`` (UNIX) or ``%NIM%/config/nim.cfg`` (Windows). This file can be skipped with the ``--skipCfg`` command line option.
+1) ``$nim/config/nim.cfg``, ``/etc/nim/nim.cfg`` (UNIX) or ``<Nim's installation director>\config\nim.cfg`` (Windows). This file can be skipped with the ``--skipCfg`` command line option.
 2) If environment variable ``XDG_CONFIG_HOME`` is defined, ``$XDG_CONFIG_HOME/nim/nim.cfg`` or ``~/.config/nim/nim.cfg`` (POSIX) or ``%APPDATA%/nim/nim.cfg`` (Windows). This file can be skipped with the ``--skipUserCfg`` command line option.
 3) ``$parentDir/nim.cfg`` where ``$parentDir`` stands for any parent  directory of the project file's path. These files can be skipped with the ``--skipParentCfg`` command line option.
 4) ``$projectDir/nim.cfg`` where ``$projectDir`` stands for the project  file's path. This file can be skipped with the ``--skipProjCfg`` command line option.
@@ -261,6 +261,21 @@ configuration file should contain something like::
   arm.linux.gcc.exe = "arm-linux-gcc"
   arm.linux.gcc.linkerexe = "arm-linux-gcc"
 
+Cross compilation for Windows
+=============================
+
+To cross compile for Windows from Linux or OSX using the MinGW-w64 toolchain::
+
+  nim c -d:mingw myproject.nim
+
+Use ``--cpu:i386`` or ``--cpu:amd64`` to switch the cpu arch.
+
+The MinGW-w64 toolchain can be installed as follows::
+
+  Ubuntu: apt install mingw-w64
+  CentOS: yum install mingw32-gcc | mingw64-gcc - requires EPEL
+  OSX: brew install mingw-w64
+
 Cross compilation for Nintendo Switch
 =====================================
 

doc/nims.rst

@@ -6,9 +6,25 @@ Strictly speaking, ``NimScript`` is the subset of Nim that can be evaluated
 by Nim's builtin virtual machine (VM). This VM is used for Nim's compiletime
 function evaluation features.
 
-You can use a ``<myproject>.nims`` file that simply contains Nim code
-controlling the compilation process. For a directory wide
-configuration, use ``config.nims`` instead of ``<myproject>.nims``.
+The ``nim`` executable processes the ``.nims`` configuration files in
+the following directories (in this order; later files overwrite
+previous settings):
+
+1) If environment variable ``XDG_CONFIG_HOME`` is defined,
+   ``$XDG_CONFIG_HOME/nim/config.nims`` or
+   ``~/.config/nim/config.nims`` (POSIX) or
+   ``%APPDATA%/nim/config.nims`` (Windows). This file can be skipped
+   with the ``--skipUserCfg`` command line option.
+2) ``$parentDir/config.nims`` where ``$parentDir`` stands for any
+   parent directory of the project file's path. These files can be
+   skipped with the ``--skipParentCfg`` command line option.
+3) ``$projectDir/config.nims`` where ``$projectDir`` stands for the
+   project's path. This file can be skipped with the ``--skipProjCfg``
+   command line option.
+4) A project can also have a project specific configuration file named
+   ``$project.nims`` that resides in the same directory as
+   ``$project.nim``. This file can be skipped with the same
+   ``--skipProjCfg`` command line option.
 
 The VM cannot deal with ``importc`` because the FFI is not
 available. So the stdlib modules using ``importc`` cannot be used with
@@ -74,6 +90,7 @@ In fact, as a convention the following tasks should be available:
 =========     ===================================================
 Task          Description
 =========     ===================================================
+``help``      List all the available NimScript tasks along with their docstrings.
 ``build``     Build the project with the required
               backend (``c``, ``cpp`` or ``js``).
 ``tests``     Runs the tests belonging to the project.

doc/nimsuggest.rst

@@ -48,7 +48,7 @@ by default.
 Specifying the location of the query
 ------------------------------------
 
-Nimsuggest than waits for queries to process. A query consists of a
+Nimsuggest then waits for queries to process. A query consists of a
 cryptic 3 letter "command" ``def`` or ``con`` or ``sug`` or ``use`` followed by
 a location. A query location consists of:
 
@@ -59,7 +59,7 @@ a location. A query location consists of:
 ``dirtyfile.nim``
     This is optional.
 
-    The ``file`` paramater is enough for static analysis, but IDEs
+    The ``file`` parameter is enough for static analysis, but IDEs
     tend to have *unsaved buffers* where the user may still be in
     the middle of typing a line. In such situations the IDE can
     save the current contents to a temporary file and then use the
@@ -143,7 +143,7 @@ tab characters (``\t``). The values of each column are:
    ``def`` for definition, ``sug`` for suggestion, etc).
 2. Type of the symbol. This can be ``skProc``, ``skLet``, and just
    about any of the enums defined in the module ``compiler/ast.nim``.
-3. Full qualitifed path of the symbol. If you are querying a symbol
+3. Fully qualified path of the symbol. If you are querying a symbol
    defined in the ``proj.nim`` file, this would have the form
    ``proj.symbolName``.
 4. Type/signature. For variables and enums this will contain the

doc/packaging.rst

@@ -0,0 +1,69 @@
+=============
+Packaging Nim
+=============
+
+
+Supported architectures
+-----------------------
+
+Nim runs on a wide variety of platforms. Support on amd64 and i386 is tested regularly, while less popular platforms are tested by the community.
+
+- amd64
+- arm64 (aka aarch64)
+- armel
+- armhf
+- i386
+- m68k
+- mips64el
+- mipsel
+- powerpc
+- ppc64
+- ppc64el (aka ppc64le)
+- riscv64
+
+The following platforms are seldomly tested:
+
+- alpha
+- hppa
+- ia64
+- mips
+- s390x
+- sparc64
+
+Packaging for Linux
+-------------------
+
+See https://github.com/nim-lang/Nim/labels/Installation for installation-related bugs.
+
+Build Nim from the released tarball at https://nim-lang.org/install_unix.html
+It is different from the GitHub sources as it contains Nimble, C sources & other tools.
+
+The Debian package ships bash and ksh completion and manpages that can be reused.
+
+Hints on the build process:
+
+::
+
+   # build from C sources and then using koch
+   ./build.sh --os $os_type --cpu $cpu_arch
+   ./bin/nim c koch
+   ./koch boot -d:release
+
+   # optionally generate docs into doc/html
+   ./koch docs
+
+   ./koch tools -d:release
+
+   # extract files to be really installed
+   ./install.sh <tempdir>
+
+   # also include the tools
+   for fn in nimble nimsuggest nimgrep; do cp ./bin/$fn <tempdir>/nim/bin/; done
+
+What to install:
+
+- The expected stdlib location is /usr/lib/nim
+- Global configuration files under /etc/nim
+- Optionally: manpages, documentation, shell completion
+- When installing documentation, .idx files are not required
+- The "compiler" directory contains compiler sources and should not be part of the compiler binary package

doc/tut2.rst

@@ -13,7 +13,6 @@ Introduction
 
   "Repetition renders the ridiculous reasonable." -- Norman Wildberger
 
-
 This document is a tutorial for the advanced constructs of the *Nim*
 programming language. **Note that this document is somewhat obsolete as the**
 `manual <manual.html>`_ **contains many more examples of the advanced language
@@ -652,369 +651,8 @@ avoid a common bug: to forget to close the file. Note how the
 ``let fn = filename`` statement ensures that ``filename`` is evaluated only
 once.
 
-Macros
-======
-
-Macros enable advanced compile-time code transformations, but they cannot
-change Nim's syntax. However, this is no real restriction because Nim's
-syntax is flexible enough anyway. Macros have to be implemented in pure Nim
-code if the `foreign function interface (FFI)
-<manual.html#foreign-function-interface>`_ is not enabled in the compiler, but
-other than that restriction (which at some point in the future will go away)
-you can write any kind of Nim code and the compiler will run it at compile
-time.
-
-There are two ways to write a macro, either *generating* Nim source code and
-letting the compiler parse it, or creating manually an abstract syntax tree
-(AST) which you feed to the compiler. In order to build the AST one needs to
-know how the Nim concrete syntax is converted to an abstract syntax tree
-(AST). The AST is documented in the `macros <macros.html>`_ module.
-
-Once your macro is finished, there are two ways to invoke it:
-(1) invoking a macro like a procedure call (expression macros)
-(2) invoking a macro with the special ``macrostmt``
-    syntax (statement macros)
-
-
-Expression Macros
------------------
-
-The following example implements a powerful ``debug`` command that accepts a
-variable number of arguments:
-
-.. code-block:: nim
-    :test: "nim c $1"
-  # to work with Nim syntax trees, we need an API that is defined in the
-  # ``macros`` module:
-  import macros
-
-  macro debug(n: varargs[untyped]): typed =
-    # `n` is a Nim AST that contains a list of expressions;
-    # this macro returns a list of statements (n is passed for proper line
-    # information):
-    result = newNimNode(nnkStmtList, n)
-    # iterate over any argument that is passed to this macro:
-    for x in n:
-      # add a call to the statement list that writes the expression;
-      # `toStrLit` converts an AST to its string representation:
-      result.add(newCall("write", newIdentNode("stdout"), toStrLit(x)))
-      # add a call to the statement list that writes ": "
-      result.add(newCall("write", newIdentNode("stdout"), newStrLitNode(": ")))
-      # add a call to the statement list that writes the expressions value:
-      result.add(newCall("writeLine", newIdentNode("stdout"), x))
-
-  var
-    a: array[0..10, int]
-    x = "some string"
-  a[0] = 42
-  a[1] = 45
-
-  debug(a[0], a[1], x)
-
-The macro call expands to:
-
-.. code-block:: nim
-  write(stdout, "a[0]")
-  write(stdout, ": ")
-  writeLine(stdout, a[0])
-
-  write(stdout, "a[1]")
-  write(stdout, ": ")
-  writeLine(stdout, a[1])
-
-  write(stdout, "x")
-  write(stdout, ": ")
-  writeLine(stdout, x)
-
-
-
-Statement Macros
-----------------
-
-Statement macros are defined just as expression macros. However, they are
-invoked by an expression following a colon.
-
-The following example outlines a macro that generates a lexical analyzer from
-regular expressions:
-
-.. code-block:: nim
-
-  macro case_token(n: varargs[untyped]): typed =
-    # creates a lexical analyzer from regular expressions
-    # ... (implementation is an exercise for the reader :-)
-    discard
-
-  case_token: # this colon tells the parser it is a macro statement
-  of r"[A-Za-z_]+[A-Za-z_0-9]*":
-    return tkIdentifier
-  of r"0-9+":
-    return tkInteger
-  of r"[\+\-\*\?]+":
-    return tkOperator
-  else:
-    return tkUnknown
-
-
-Building your first macro
--------------------------
-
-To give a footstart to writing macros we will show now how to turn your typical
-dynamic code into something that compiles statically. For the exercise we will
-use the following snippet of code as the starting point:
-
-.. code-block:: nim
-    :test: "nim c $1"
-
-  import strutils, tables
-
-  proc readCfgAtRuntime(cfgFilename: string): Table[string, string] =
-    let
-      inputString = readFile(cfgFilename)
-    var
-      source = ""
-
-    result = initTable[string, string]()
-    for line in inputString.splitLines:
-      # Ignore empty lines
-      if line.len < 1: continue
-      var chunks = split(line, ',')
-      if chunks.len != 2:
-        quit("Input needs comma split values, got: " & line)
-      result[chunks[0]] = chunks[1]
-
-    if result.len < 1: quit("Input file empty!")
-
-  let info = readCfgAtRuntime("data.cfg")
-
-  when isMainModule:
-    echo info["licenseOwner"]
-    echo info["licenseKey"]
-    echo info["version"]
-
-Presumably this snippet of code could be used in a commercial software, reading
-a configuration file to display information about the person who bought the
-software. This external file would be generated by an online web shopping cart
-to be included along the program containing the license information::
-
-  version,1.1
-  licenseOwner,Hyori Lee
-  licenseKey,M1Tl3PjBWO2CC48m
-
-The ``readCfgAtRuntime`` proc will open the given filename and return a
-``Table`` from the `tables module <tables.html>`_. The parsing of the file is
-done (without much care for handling invalid data or corner cases) using the
-`splitLines proc from the strutils module <strutils.html#splitLines>`_. There
-are many things which can fail; mind the purpose is explaining how to make
-this run at compile time, not how to properly implement a DRM scheme.
-
-The reimplementation of this code as a compile time proc will allow us to get
-rid of the ``data.cfg`` file we would need to distribute along the binary, plus
-if the information is really constant, it doesn't make from a logical point of
-view to have it *mutable* in a global variable, it would be better if it was a
-constant. Finally, and likely the most valuable feature, we can implement some
-verification at compile time. You could think of this as a *better unit
-testing*, since it is impossible to obtain a binary unless everything is
-correct, preventing you to ship to users a broken program which won't start
-because a small critical file is missing or its contents changed by mistake to
-something invalid.
-
-
-Generating source code
-++++++++++++++++++++++
-
-Our first attempt will start by modifying the program to generate a compile
-time string with the *generated source code*, which we then pass to the
-``parseStmt`` proc from the `macros module <macros.html>`_. Here is the
-modified source code implementing the macro:
-
-.. code-block:: nim
-   :number-lines:
-
-  import macros, strutils
-
-  macro readCfgAndBuildSource(cfgFilename: string): typed =
-    let
-      inputString = slurp(cfgFilename.strVal)
-    var
-      source = ""
-
-    for line in inputString.splitLines:
-      # Ignore empty lines
-      if line.len < 1: continue
-      var chunks = split(line, ',')
-      if chunks.len != 2:
-        error("Input needs comma split values, got: " & line)
-      source &= "const cfg" & chunks[0] & "= \"" & chunks[1] & "\"\n"
-
-    if source.len < 1: error("Input file empty!")
-    result = parseStmt(source)
-
-  readCfgAndBuildSource("data.cfg")
-
-  when isMainModule:
-    echo cfglicenseOwner
-    echo cfglicenseKey
-    echo cfgversion
-
-The good news is not much has changed! First, we need to change the handling
-of the input parameter (line 3). In the dynamic version the
-``readCfgAtRuntime`` proc receives a string parameter. However, in the macro
-version it is also declared as string, but this is the *outside* interface of
-the macro.  When the macro is run, it actually gets a ``PNimNode`` object
-instead of a string, and we have to call the `strVal proc
-<macros.html#strVal>`_ (line 5) from the `macros module <macros.html>`_ to
-obtain the string being passed in to the macro.
-
-Second, we cannot use the `readFile proc <system.html#readFile>`_ from the
-`system module <system.html>`_ due to FFI restriction at compile time. If we
-try to use this proc, or any other which depends on FFI, the compiler will
-error with the message ``cannot evaluate`` and a dump of the macro's source
-code, along with a stack trace where the compiler reached before bailing out.
-We can get around this limitation by using the `slurp proc
-<system.html#slurp>`_ from the `system module <system.html>`_, which was
-precisely made for compilation time (just like `gorge <system.html#gorge>`_
-which executes an external program and captures its output).
-
-The interesting thing is that our macro does not return a runtime `Table
-<tables.html#Table>`_ object. Instead, it builds up Nim source code into
-the ``source`` variable.  For each line of the configuration file a ``const``
-variable will be generated (line 15).  To avoid conflicts we prefix these
-variables with ``cfg``. In essence, what the compiler is doing is replacing
-the line calling the macro with the following snippet of code:
-
-.. code-block:: nim
-  const cfgversion = "1.1"
-  const cfglicenseOwner = "Hyori Lee"
-  const cfglicenseKey = "M1Tl3PjBWO2CC48m"
-
-You can verify this yourself adding the line ``echo source`` somewhere at the
-end of the macro and compiling the program. Another difference is that instead
-of calling the usual `quit proc <system.html#quit>`_ to abort (which we could
-still call) this version calls the `error proc <macros.html#error>`_ (line
-14). The ``error`` proc has the same behavior as ``quit`` but will dump also
-the source and file line information where the error happened, making it
-easier for the programmer to find where compilation failed. In this situation
-it would point to the line invoking the macro, but **not** the line of
-``data.cfg`` we are processing, that's something the macro itself would need
-to control.
-
-
-Generating AST by hand
-++++++++++++++++++++++
-
-To generate an AST we would need to intimately know the structures used by the
-Nim compiler exposed in the `macros module <macros.html>`_, which at first
-look seems a daunting task. But we can use as helper shortcut the `dumpTree
-macro <macros.html#dumpTree>`_, which is used as a statement macro instead of
-an expression macro.  Since we know that we want to generate a bunch of
-``const`` symbols we can create the following source file and compile it to
-see what the compiler *expects* from us:
-
-.. code-block:: nim
-    :test: "nim c $1"
-  import macros
-
-  dumpTree:
-    const cfgversion: string = "1.1"
-    const cfglicenseOwner = "Hyori Lee"
-    const cfglicenseKey = "M1Tl3PjBWO2CC48m"
-
-During compilation of the source code we should see the following lines in the
-output (again, since this is a macro, compilation is enough, you don't have to
-run any binary)::
-
-  StmtList
-    ConstSection
-      ConstDef
-        Ident !"cfgversion"
-        Ident !"string"
-        StrLit 1.1
-    ConstSection
-      ConstDef
-        Ident !"cfglicenseOwner"
-        Empty
-        StrLit Hyori Lee
-    ConstSection
-      ConstDef
-        Ident !"cfglicenseKey"
-        Empty
-        StrLit M1Tl3PjBWO2CC48m
-
-With this output we have a better idea of what kind of input the compiler
-expects. We need to generate a list of statements. For each constant the source
-code generates a ``ConstSection`` and a ``ConstDef``. If we were to move all
-the constants to a single ``const`` block we would see only a single
-``ConstSection`` with three children.
-
-Maybe you didn't notice, but in the ``dumpTree`` example the first constant
-explicitly specifies the type of the constant.  That's why in the tree output
-the two last constants have their second child ``Empty`` but the first has a
-string identifier. So basically a ``const`` definition is made up from an
-identifier, optionally a type (can be an *empty* node) and the value. Armed
-with this knowledge, let's look at the finished version of the AST building
-macro:
-
-.. code-block:: nim
-   :number-lines:
-
-  import macros, strutils
-
-  macro readCfgAndBuildAST(cfgFilename: string): typed =
-    let
-      inputString = slurp(cfgFilename.strVal)
-
-    result = newNimNode(nnkStmtList)
-    for line in inputString.splitLines:
-      # Ignore empty lines
-      if line.len < 1: continue
-      var chunks = split(line, ',')
-      if chunks.len != 2:
-        error("Input needs comma split values, got: " & line)
-      var
-        section = newNimNode(nnkConstSection)
-        constDef = newNimNode(nnkConstDef)
-      constDef.add(newIdentNode("cfg" & chunks[0]))
-      constDef.add(newEmptyNode())
-      constDef.add(newStrLitNode(chunks[1]))
-      section.add(constDef)
-      result.add(section)
-
-    if result.len < 1: error("Input file empty!")
-
-  readCfgAndBuildAST("data.cfg")
-
-  when isMainModule:
-    echo cfglicenseOwner
-    echo cfglicenseKey
-    echo cfgversion
-
-Since we are building on the previous example generating source code, we will
-only mention the differences to it. Instead of creating a temporary ``string``
-variable and writing into it source code as if it were written *by hand*, we
-use the ``result`` variable directly and create a statement list node
-(``nnkStmtList``) which will hold our children (line 7).
-
-For each input line we have to create a constant definition (``nnkConstDef``)
-and wrap it inside a constant section (``nnkConstSection``). Once these
-variables are created, we fill them hierarchichally (line 17) like the
-previous AST dump tree showed: the constant definition is a child of the
-section definition, and the constant definition has an identifier node, an
-empty node (we let the compiler figure out the type), and a string literal
-with the value.
-
-A last tip when writing a macro: if you are not sure the AST you are building
-looks ok, you may be tempted to use the ``dumpTree`` macro. But you can't use
-it *inside* the macro you are writting/debugging. Instead ``echo`` the string
-generated by `treeRepr <macros.html#treeRepr>`_. If at the end of the this
-example you add ``echo treeRepr(result)`` you should get the same output as
-using the ``dumpTree`` macro, but of course you can call that at any point of
-the macro where you might be having troubles.
-
-Example Templates and Macros
-============================
-
-Lifting Procs
-+++++++++++++
+Example: Lifting Procs
+----------------------
 
 .. code-block:: nim
     :test: "nim c $1"
@@ -1039,36 +677,6 @@ Lifting Procs
   liftScalarProc(sqrt)   # make sqrt() work for sequences
   echo sqrt(@[4.0, 16.0, 25.0, 36.0])   # => @[2.0, 4.0, 5.0, 6.0]
 
-Identifier Mangling
-+++++++++++++++++++
-
-.. code-block:: nim
-  proc echoHW() =
-    echo "Hello world"
-  proc echoHW0() =
-    echo "Hello world 0"
-  proc echoHW1() =
-    echo "Hello world 1"
-
-  template joinSymbols(a, b: untyped): untyped =
-    `a b`()
-
-  joinSymbols(echo, HW)
-
-  macro str2Call(s1, s2): typed =
-    result = newNimNode(nnkStmtList)
-    for i in 0..1:
-      # combines s1, s2 and an integer into an proc identifier
-      # that is called in a statement list
-      result.add(newCall(!($s1 & $s2 & $i)))
-
-  str2Call("echo", "HW")
-
-  # Output:
-  #   Hello world
-  #   Hello world 0
-  #   Hello world 1
-
 Compilation to JavaScript
 =========================
 
@@ -1083,3 +691,9 @@ JavaScript-compatible code you should remember the following:
 - ``cstring`` in JavaScript means JavaScript string. It is a good practice to
   use ``cstring`` only when it is semantically appropriate. E.g. don't use
   ``cstring`` as a binary data buffer.
+
+
+Part 3
+======
+
+Next part will be entirely about metaprogramming via macros: `Part III <tut3.html>`_

doc/tut3.rst

@@ -0,0 +1,354 @@
+=======================
+Nim Tutorial (Part III)
+=======================
+
+:Author: Arne Döring
+:Version: |nimversion|
+
+.. contents::
+
+
+Introduction
+============
+
+  "With Great Power Comes Great Responsibility." -- Spider Man's Uncle
+
+This document is a tutorial about Nim's macro system.
+A macro is a function that is executed at compile time and transforms
+a Nim syntax tree into a different tree.
+
+Examples of things that can be implemented in macros:
+
+ * An assert macro that prints both sides of a comparison operator, if
+the assertion fails. ``myAssert(a == b)`` is converted to
+``if a != b: quit($a " != " $b)``
+
+ * A debug macro that prints the value and the name of the symbol.
+``myDebugEcho(a)`` is converted to ``echo "a: ", a``
+
+ * Symbolic differentiation of an expression.
+``diff(a*pow(x,3) + b*pow(x,2) + c*x + d, x)``  is converted to
+``3*a*pow(x,2) + 2*a*x + c``
+
+
+Macro Arguments
+---------------
+
+The types of macro arguments have two faces. One face is used for
+the overload resolution, and the other face is used within the macro
+body. For example, if ``macro foo(arg: int)`` is called in an
+expression ``foo(x)``, ``x`` has to be of a type compatible to int, but
+*within* the macro's body ``arg`` has the type ``NimNode``, not ``int``!
+Why it is done this way will become obvious later, when we have seen
+concrete examples.
+
+There are two ways to pass arguments to a macro, an argument can be
+either ``typed`` or ``untyped``.
+
+
+Untyped Arguments
+-----------------
+
+Untyped macro arguments are passed to the macro before they are
+semantically checked. This means the syntax tree that is passed down
+to the macro does not need to make sense for Nim yet, the only
+limitation is that it needs to be parseable. Usually the macro does
+not check the argument either but uses it in the transformation's
+result somehow. The result of a macro expansion is always checked
+by the compiler, so apart from weird error messages nothing bad
+can happen.
+
+The downside for an ``untyped`` argument is that these do not play
+well with Nim's overloading resolution.
+
+The upside for untyped arguments is that the syntax tree is
+quite predictable and less complex compared to its ``typed``
+counterpart.
+
+
+Typed Arguments
+---------------
+
+For typed arguments, the semantic checker runs on the argument and
+does transformations on it, before it is passed to the macro. Here
+identifier nodes are resolved as symbols, implicit type
+conversions are visible in the tree as calls, templates are
+expanded and probably most importantly, nodes have type information.
+Typed arguments can have the type ``typed`` in the arguments list.
+But all other types, such as ``int``, ``float`` or ``MyObjectType``
+are typed arguments as well, and they are passed to the macro as a
+syntax tree.
+
+
+Static Arguments
+----------------
+
+Static arguments are a way to pass values as values and not as syntax
+tree nodes to a macro. For example for ``macro foo(arg: static[int])``
+in the expression ``foo(x)``, ``x`` needs to be an integer constant,
+but in the macro body ``arg`` is just like a normal parameter of type
+``int``.
+
+.. code-block:: nim
+
+  import macros
+
+  macro myMacro(arg: static[int]): untyped =
+    echo arg # just an int (7), not ``NimNode``
+
+  myMacro(1 + 2 * 3)
+
+
+Code blocks as arguments
+------------------------
+
+It is possible to pass the last argument of a call expression in a
+separate code block with indentation. For example the following code
+example is a valid (but not a recommended) way to call ``echo``:
+
+.. code-block:: nim
+
+  echo "Hello ":
+    let a = "Wor"
+    let b = "ld!"
+    a & b
+
+For macros this way of calling is very useful; syntax trees of arbitrary
+complexity can be passed to macros with this notation.
+
+
+The Syntax Tree
+---------------
+
+In order to build a Nim syntax tree one needs to know how Nim source
+code is represented as a syntax tree, and how such a tree needs to
+look like so that the Nim compiler will understand it. The nodes of the
+Nim syntax tree are documented in the `macros <macros.html>`_ module.
+But a more interactive way to explore the Nim
+syntax tree is with ``macros.treeRepr``, it converts a syntax tree
+into a multi line string for printing on the console. It can be used
+to explore how the argument expressions are represented in tree form
+and for debug printing of generated syntax tree. ``dumpTree`` is a
+predefined macro that just prints its argument in tree representation,
+but does nothing else. Here is an example of such a tree representation:
+
+.. code-block:: nim
+
+  dumpTree:
+    var mt: MyType = MyType(a:123.456, b:"abcdef")
+
+  # output:
+  #   StmtList
+  #     VarSection
+  #       IdentDefs
+  #         Ident "mt"
+  #         Ident "MyType"
+  #         ObjConstr
+  #           Ident "MyType"
+  #           ExprColonExpr
+  #             Ident "a"
+  #             FloatLit 123.456
+  #           ExprColonExpr
+  #             Ident "b"
+  #             StrLit "abcdef"
+
+
+Custom sematic checking
+-----------------------
+
+The first thing that a macro should do with its arguments is to check
+if the argument is in the correct form. Not every type of wrong input
+needs to be caught here, but anything that could cause a crash during
+macro evaluation should be caught and create a nice error message.
+``macros.expectKind`` and ``macros.expectLen`` are a good start. If
+the checks need to be more complex, arbitrary error messages can
+be created with the ``macros.error`` proc.
+
+.. code-block:: nim
+
+  macro myAssert(arg: untyped): untyped =
+    arg.expectKind nnkInfix
+
+
+Generating Code
+---------------
+
+There are two ways to generate the code. Either by creating the syntax
+tree with expressions that contain a lot of calls to ``newTree`` and
+``newLit``, or with ``quote do:`` expressions. The first option offers
+the best low level control for the syntax tree generation, but the
+second option is much less verbose. If you choose to create the syntax
+tree with calls to ``newTree`` and ``newLit`` the macro
+``marcos.dumpAstGen`` can help you with the verbosity. ``quote do:``
+allows you to write the code that you want to generate literally,
+backticks are used to insert code from ``NimNode`` symbols into the
+generated expression. This means that you can't use backticks within
+``quote do:`` for anything else than injecting symbols.  Make sure to
+inject only symbols of type ``NimNode`` into the generated syntax
+tree. You can use ``newLit`` to convert arbitrary values into
+expressions trees of type ``NimNode`` so that it is safe to inject
+them into the tree.
+
+
+.. code-block:: nim
+    :test: "nim c $1"
+
+  import macros
+
+  type
+    MyType = object
+      a: float
+      b: string
+
+  macro myMacro(arg: untyped): untyped =
+    var mt: MyType = MyType(a:123.456, b:"abcdef")
+
+    # ...
+
+    let mtLit = newLit(mt)
+
+    result = quote do:
+      echo `arg`
+      echo `mtLit`
+
+  myMacro("Hallo")
+
+The call to ``myMacro`` will generate the following code:
+
+.. code-block:: nim
+  echo "Hallo"
+  echo MyType(a: 123.456'f64, b: "abcdef")
+
+
+Building your first macro
+-------------------------
+
+To give a footstart to writing macros we will show now how to
+implement the ``myDebug`` macro mentioned earlier. The first thing to
+do is to build a simple example of the macro usage, and then just
+print the argument. This way it is possible to get an idea of a
+correct argument should be look like.
+
+.. code-block:: nim
+    :test: "nim c $1"
+
+  import macros
+
+  macro myAssert(arg: untyped): untyped =
+    echo arg.treeRepr
+
+  let a = 1
+  let b = 2
+
+  myAssert(a != b)
+
+.. code-block::
+
+  Infix
+    Ident "!="
+    Ident "a"
+    Ident "b"
+
+
+From the output it is possible to see that the information that the
+argument is an infix operator (node kind is "Infix"), as well as that the two
+operands are at index 1 and 2. With this information the actual
+macro can be written.
+
+.. code-block:: nim
+    :test: "nim c $1"
+
+  import macros
+
+  macro myAssert(arg: untyped): untyped =
+    # all node kind identifiers are prefixed with "nnk"
+    arg.expectKind nnkInfix
+    arg.expectLen 3
+    # operator as string literal
+    let op  = newLit(" " & arg[0].repr & " ")
+    let lhs = arg[1]
+    let rhs = arg[2]
+
+    result = quote do:
+      if not `arg`:
+        raise newException(AssertionError,$`lhs` & `op` & $`rhs`)
+
+  let a = 1
+  let b = 2
+
+  myAssert(a != b)
+  myAssert(a == b)
+
+
+This is the code that will be generated. To debug what the macro
+actually generated, the statement ``echo result.repr`` can be used, in
+the last line of the macro. It is also the statement that has been
+used to get this output.
+
+.. code-block:: nim
+  if not (a != b):
+    raise newException(AssertionError, $a & " != " & $b)
+
+With Power Comes Responsibility
+-------------------------------
+
+Macros are very powerful. A good advice is to use them as little as
+possible, but as much as necessary. Macros can change the semantics of
+expressions, making the code incomprehensible for anybody who does not
+know exactly what the macro does with it. So whenever a macro is not
+necessary and the same logic can be implemented using templates or
+generics, it is probably better not to use a macro. And when a macro
+is used for something, the macro should better have a well written
+documentation. For all the people who claim to write only perfectly
+self-explanatory code: when it comes to macros, the implementation is
+not enough for documentation.
+
+Limitations
+-----------
+
+Since macros are evaluated in the compiler in the NimVM, macros share
+all the limitations of the NimVM. They have to be implemented in pure Nim
+code. Macros can start external processes on the shell, but they
+cannot call C functions except from those that are built in the
+compiler.
+
+
+More Examples
+=============
+
+This tutorial can only cover the basics of the macro system. There are
+macros out there that could be an inspiration for you of what is
+possible with it.
+
+
+Strformat
+---------
+
+In the Nim standard library, the ``strformat`` library provides a
+macro that parses a string literal at compile time. Parsing a string
+in a macro like here is generally not recommended. The parsed AST
+cannot have type information, and parsing implemented on the VM is
+generally not very fast. Working on AST nodes is almost always the
+recommended way. But still ``strformat`` is a good example for a
+practical use case for a macro that is slightly more complex that the
+``assert`` macro.
+
+`Strformat <https://github.com/nim-lang/Nim/blob/5845716df8c96157a047c2bd6bcdd795a7a2b9b1/lib/pure/strformat.nim#L280>`_
+
+Ast Pattern Matching
+--------------------
+
+Ast Pattern Matching is a macro library to aid in writing complex
+macros. This can be seen as a good example of how to repurpose the
+Nim syntax tree with new semantics.
+
+`Ast Pattern Matching <https://github.com/krux02/ast-pattern-matching>`_
+
+OpenGL Sandbox
+--------------
+
+This project has a working Nim to GLSL compiler written entirely in
+macros. It scans recursively though all used function symbols to
+compile them so that cross library functions can be executed on the GPU.
+
+`OpenGL Sandbox <https://github.com/krux02/opengl-sandbox>`_

koch.nim

@@ -74,7 +74,7 @@ template withDir(dir, body) =
     setCurrentDir(dir)
     body
   finally:
-    setCurrentdir(old)
+    setCurrentDir(old)
 
 proc tryExec(cmd: string): bool =
   echo(cmd)
@@ -110,6 +110,11 @@ proc bundleNimbleSrc(latest: bool) =
       exec("git checkout -f stable")
       exec("git pull")
 
+proc bundleC2nim() =
+  if not dirExists("dist/c2nim/.git"):
+    exec("git clone https://github.com/nim-lang/c2nim.git dist/c2nim")
+  nimCompile("dist/c2nim/c2nim", options = "--noNimblePath --path:.")
+
 proc bundleNimbleExe(latest: bool) =
   bundleNimbleSrc(latest)
   # now compile Nimble and copy it to $nim/bin for the installer.ini
@@ -160,6 +165,7 @@ proc bundleWinTools() =
   buildVccTool()
   nimexec("c -o:bin/nimgrab.exe -d:ssl tools/nimgrab.nim")
   nimexec("c -o:bin/nimgrep.exe tools/nimgrep.nim")
+  bundleC2nim()
   when false:
     # not yet a tool worth including
     nimexec(r"c --cc:vcc --app:gui -o:bin\downloader.exe -d:ssl --noNimblePath " &
@@ -380,11 +386,11 @@ template `|`(a, b): string = (if a.len > 0: a else: b)
 proc tests(args: string) =
   # we compile the tester with taintMode:on to have a basic
   # taint mode test :-)
-  nimexec "cc --taintMode:on --opt:speed tests/testament/tester"
+  nimexec "cc --taintMode:on --opt:speed testament/tester"
   # Since tests take a long time (on my machine), and we want to defy Murhpys
   # law - lets make sure the compiler really is freshly compiled!
   nimexec "c --lib:lib -d:release --opt:speed compiler/nim.nim"
-  let tester = quoteShell(getCurrentDir() / "tests/testament/tester".exe)
+  let tester = quoteShell(getCurrentDir() / "testament/tester".exe)
   let success = tryExec tester & " " & (args|"all")
   if not existsEnv("TRAVIS") and not existsEnv("APPVEYOR"):
     exec tester & " html"
@@ -474,8 +480,7 @@ proc testUnixInstall(cmdLineRest: string) =
       execCleanPath("./koch --latest tools")
       # check the tests work:
       putEnv("NIM_EXE_NOT_IN_PATH", "NOT_IN_PATH")
-      execCleanPath("./koch tests", destDir / "bin")
-      #execCleanPath("./koch tests cat newconfig", destDir / "bin")
+      execCleanPath("./koch tests cat megatest", destDir / "bin")
     else:
       echo "Version check: failure"
   finally:
@@ -549,6 +554,7 @@ when isMainModule:
         else: buildTools(existsDir(".git") or latest)
       of "pushcsource", "pushcsources": pushCsources()
       of "valgrind": valgrind(op.cmdLineRest)
+      of "c2nim": bundleC2nim()
       else: showHelp()
       break
     of cmdEnd: break

lib/core/macros.nim

@@ -649,7 +649,7 @@ proc newLit*(f: float64): NimNode {.compileTime.} =
   result = newNimNode(nnkFloat64Lit)
   result.floatVal = f
 
-when compiles(float128):
+when declared(float128):
   proc newLit*(f: float128): NimNode {.compileTime.} =
     ## produces a new float literal node.
     result = newNimNode(nnkFloat128Lit)
@@ -976,6 +976,8 @@ proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]):
   ##    )
   ##
   result = newNimNode(nnkIfStmt)
+  if len(branches) < 1:
+    error("If statement must have at least one branch")
   for i in branches:
     result.add(newTree(nnkElifBranch, i.cond, i.body))
 
@@ -1034,10 +1036,18 @@ proc name*(someProc: NimNode): NimNode {.compileTime.} =
   someProc.expectRoutine
   result = someProc[0]
   if result.kind == nnkPostfix:
-    result = result[1]
+    if result[1].kind == nnkAccQuoted:
+      result = result[1][0]
+    else:
+      result = result[1]
+  elif result.kind == nnkAccQuoted:
+    result = result[0]
+
 proc `name=`*(someProc: NimNode; val: NimNode) {.compileTime.} =
   someProc.expectRoutine
-  someProc[0] = val
+  if someProc[0].kind == nnkPostfix:
+    someProc[0][1] = val
+  else: someProc[0] = val
 
 proc params*(someProc: NimNode): NimNode {.compileTime.} =
   someProc.expectRoutine

lib/core/typeinfo.nim

@@ -108,6 +108,7 @@ proc getDiscriminant(aa: pointer, n: ptr TNimNode): int =
   of 1: d = ze(cast[ptr int8](a +% n.offset)[])
   of 2: d = ze(cast[ptr int16](a +% n.offset)[])
   of 4: d = int(cast[ptr int32](a +% n.offset)[])
+  of 8: d = int(cast[ptr int64](a +% n.offset)[])
   else: assert(false)
   return d
 

lib/deprecated/pure/asyncio.nim

@@ -180,7 +180,7 @@ proc asyncSocket*(domain: Domain = AF_INET, typ: SockType = SOCK_STREAM,
                   protocol: Protocol = IPPROTO_TCP,
                   buffered = true): AsyncSocket =
   ## Initialises an AsyncSocket object. If a socket cannot be initialised
-  ## EOS is raised.
+  ## OSError is raised.
   result = newAsyncSocket()
   result.socket = socket(domain, typ, protocol, buffered)
   result.proto = protocol
@@ -476,7 +476,7 @@ proc recvLine*(s: AsyncSocket, line: var TaintedString): bool {.deprecated.} =
   ## if this function can only retrieve some data; it will save this data and
   ## add it to the result when a full line is retrieved.
   ##
-  ## Unlike ``sockets.recvLine`` this function will raise an EOS or ESSL
+  ## Unlike ``sockets.recvLine`` this function will raise an OSError or SslError
   ## exception if an error occurs.
   ##
   ## **Deprecated since version 0.9.2**: This function has been deprecated in
@@ -512,7 +512,7 @@ proc readLine*(s: AsyncSocket, line: var TaintedString): bool =
   ## retrieved or the socket has been disconnected in which case ``line`` will
   ## be set to "".
   ##
-  ## This function will raise an EOS exception when a socket error occurs.
+  ## This function will raise an OSError exception when a socket error occurs.
   setLen(line.string, 0)
   var dataReceived = "".TaintedString
   var ret = s.socket.readLineAsync(dataReceived)

lib/deprecated/pure/sockets.nim

@@ -23,7 +23,7 @@
 ##
 ## SSL is supported through the OpenSSL library. This support can be activated
 ## by compiling with the ``-d:ssl`` switch. When an SSL socket is used it will
-## raise ESSL exceptions when SSL errors occur.
+## raise SslError exceptions when SSL errors occur.
 ##
 ## Asynchronous sockets are supported, however a better alternative is to use
 ## the `asyncio <asyncio.html>`_ module.
@@ -262,7 +262,7 @@ proc socket*(domain: Domain = AF_INET, typ: SockType = SOCK_STREAM,
              protocol: Protocol = IPPROTO_TCP, buffered = true): Socket =
   ## Creates a new socket; returns `InvalidSocket` if an error occurs.
 
-  # TODO: Perhaps this should just raise EOS when an error occurs.
+  # TODO: Perhaps this should just raise OSError when an error occurs.
   when defined(Windows):
     result = newTSocket(winlean.socket(cint(domain), cint(typ), cint(protocol)), buffered)
   else:
@@ -422,7 +422,7 @@ proc parseIp4*(s: string): BiggestInt =
   ##
   ## This is equivalent to `inet_ntoa`:idx:.
   ##
-  ## Raises EInvalidValue in case of an error.
+  ## Raises ValueError in case of an error.
   var a, b, c, d: int
   var i = 0
   var j = parseInt(s, a, i)
@@ -543,7 +543,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string) {.
   ## If ``server`` is non-blocking then this function returns immediately, and
   ## if there are no connections queued the returned socket will be
   ## ``InvalidSocket``.
-  ## This function will raise EOS if an error occurs.
+  ## This function will raise OSError if an error occurs.
   ##
   ## The resulting client will inherit any properties of the server socket. For
   ## example: whether the socket is buffered or not.
@@ -664,7 +664,7 @@ proc close*(socket: Socket) =
     discard winlean.closesocket(socket.fd)
   else:
     discard posix.close(socket.fd)
-  # TODO: These values should not be discarded. An EOS should be raised.
+  # TODO: These values should not be discarded. An OSError should be raised.
   # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times
   when defined(ssl):
     if socket.isSSL:
@@ -920,7 +920,7 @@ when defined(ssl):
     ## Returns ``False`` whenever the socket is not yet ready for a handshake,
     ## ``True`` whenever handshake completed successfully.
     ##
-    ## A ESSL error is raised on any other errors.
+    ## A SslError error is raised on any other errors.
     result = true
     if socket.isSSL:
       var ret = SSLConnect(socket.sslHandle)
@@ -946,7 +946,7 @@ when defined(ssl):
     ## Determines whether a handshake has occurred between a client (``socket``)
     ## and the server that ``socket`` is connected to.
     ##
-    ## Throws ESSL if ``socket`` is not an SSL socket.
+    ## Throws SslError if ``socket`` is not an SSL socket.
     if socket.isSSL:
       return not socket.sslNoHandshake
     else:
@@ -1215,7 +1215,7 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1): int =
   ##
   ## When 0 is returned the socket's connection has been closed.
   ##
-  ## This function will throw an EOS exception when an error occurs. A value
+  ## This function will throw an OSError exception when an error occurs. A value
   ## lower than 0 is never returned.
   ##
   ## A timeout may be specified in milliseconds, if enough data is not received
@@ -1273,7 +1273,7 @@ proc recvLine*(socket: Socket, line: var TaintedString, timeout = -1): bool {.
   ## will be set to it.
   ##
   ## ``True`` is returned if data is available. ``False`` suggests an
-  ## error, EOS exceptions are not raised and ``False`` is simply returned
+  ## error, OSError exceptions are not raised and ``False`` is simply returned
   ## instead.
   ##
   ## If the socket is disconnected, ``line`` will be set to ``""`` and ``True``
@@ -1321,7 +1321,7 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1) {.
   ##
   ## If the socket is disconnected, ``line`` will be set to ``""``.
   ##
-  ## An EOS exception will be raised in the case of a socket error.
+  ## An OSError exception will be raised in the case of a socket error.
   ##
   ## A timeout can be specified in milliseconds, if data is not received within
   ## the specified time an ETimeout exception will be raised.
@@ -1394,7 +1394,7 @@ proc readLineAsync*(socket: Socket,
   ##   * If some data has been retrieved; ``ReadPartialLine`` is returned.
   ##   * If the socket has been disconnected; ``ReadDisconnected`` is returned.
   ##   * If no data could be retrieved; ``ReadNone`` is returned.
-  ##   * If call to ``recv`` failed; **an EOS exception is raised.**
+  ##   * If call to ``recv`` failed; **an OSError exception is raised.**
   setLen(line.string, 0)
 
   template errorOrNone =
@@ -1421,7 +1421,7 @@ proc readLineAsync*(socket: Socket,
 
 proc recv*(socket: Socket): TaintedString {.tags: [ReadIOEffect], deprecated.} =
   ## receives all the available data from the socket.
-  ## Socket errors will result in an ``EOS`` error.
+  ## Socket errors will result in an ``OSError`` error.
   ## If socket is not a connectionless socket and socket is not connected
   ## ``""`` will be returned.
   ##
@@ -1470,7 +1470,7 @@ proc recvAsync*(socket: Socket, s: var TaintedString): bool {.
   tags: [ReadIOEffect], deprecated.} =
   ## receives all the data from a non-blocking socket. If socket is non-blocking
   ## and there are no messages available, `False` will be returned.
-  ## Other socket errors will result in an ``EOS`` error.
+  ## Other socket errors will result in an ``OSError`` error.
   ## If socket is not a connectionless socket and socket is not connected
   ## ``s`` will be set to ``""``.
   ##
@@ -1547,7 +1547,7 @@ proc recvFromAsync*(socket: Socket, data: var string, length: int,
                     address: var string, port: var Port,
                     flags = 0'i32): bool {.tags: [ReadIOEffect].} =
   ## Variant of ``recvFrom`` for non-blocking sockets. Unlike ``recvFrom``,
-  ## this function will raise an EOS error whenever a socket error occurs.
+  ## this function will raise an OSError error whenever a socket error occurs.
   ##
   ## If there is no data to be read from the socket ``False`` will be returned.
   result = true
@@ -1622,7 +1622,7 @@ proc sendAsync*(socket: Socket, data: string): int {.tags: [WriteIOEffect].} =
   ## returns the amount of bytes of ``data`` that was successfully sent. This
   ## number may not always be the length of ``data`` but typically is.
   ##
-  ## An EOS (or ESSL if socket is an SSL socket) exception is raised if an error
+  ## An OSError (or SslError if socket is an SSL socket) exception is raised if an error
   ## occurs.
   result = send(socket, cstring(data), data.len)
   when defined(ssl):
@@ -1656,7 +1656,7 @@ proc sendAsync*(socket: Socket, data: string): int {.tags: [WriteIOEffect].} =
 
 
 proc trySend*(socket: Socket, data: string): bool {.tags: [WriteIOEffect].} =
-  ## safe alternative to ``send``. Does not raise an EOS when an error occurs,
+  ## safe alternative to ``send``. Does not raise an OSError when an error occurs,
   ## and instead returns ``false`` on failure.
   result = send(socket, cstring(data), data.len) == data.len
 

lib/impure/db_odbc.nim

@@ -133,7 +133,7 @@ proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {.
               511.TSqlSmallInt, retSz.addr.PSQLSMALLINT)
   except:
     discard
-  return (res.int, $sqlState, $nativeErr, $errMsg)
+  return (res.int, $(addr sqlState), $(addr nativeErr), $(addr errMsg))
 
 proc dbError*(db: var DbConn) {.
           tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} =

lib/impure/db_sqlite.nim

@@ -31,7 +31,7 @@
 ##
 ## .. code-block:: Nim
 ##     import db_sqlite
-##     let db = open("mytest.db", nil, nil, nil)  # user, password, database name can be nil
+##     let db = open("mytest.db", "", "", "")  # user, password, database name can be empty
 ##     db.close()
 ##
 ## Creating a table

lib/impure/nre.nim

@@ -8,7 +8,7 @@
 
 
 from pcre import nil
-import nre.private.util
+import nre/private/util
 import tables
 from strutils import `%`
 from math import ceil
@@ -469,7 +469,8 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt
   # 1x capture count as slack space for PCRE
   let vecsize = (pattern.captureCount() + 1) * 3
   # div 2 because each element is 2 cints long
-  myResult.pcreMatchBounds = newSeq[HSlice[cint, cint]](ceil(vecsize / 2).int)
+  # plus 1 because we need the ceiling, not the floor
+  myResult.pcreMatchBounds = newSeq[HSlice[cint, cint]]((vecsize + 1) div 2)
   myResult.pcreMatchBounds.setLen(vecsize div 3)
 
   let strlen = if endpos == int.high: str.len else: endpos+1

lib/impure/re.nim

@@ -408,7 +408,7 @@ proc startsWith*(s: string, prefix: Regex): bool {.inline.} =
   result = matchLen(s, prefix) >= 0
 
 proc endsWith*(s: string, suffix: Regex): bool {.inline.} =
-  ## returns true if `s` ends with the pattern `prefix`
+  ## returns true if `s` ends with the pattern `suffix`
   for i in 0 .. s.len-1:
     if matchLen(s, suffix, i) == s.len - i: return true
 
@@ -428,7 +428,7 @@ proc replace*(s: string, sub: Regex, by = ""): string =
   ##   "; "
   result = ""
   var prev = 0
-  while true:
+  while prev < s.len:
     var match = findBounds(s, sub, prev)
     if match.first < 0: break
     add(result, substr(s, prev, match.first-1))
@@ -453,7 +453,7 @@ proc replacef*(s: string, sub: Regex, by: string): string =
   result = ""
   var caps: array[MaxSubpatterns, string]
   var prev = 0
-  while true:
+  while prev < s.len:
     var match = findBounds(s, sub, caps, prev)
     if match.first < 0: break
     add(result, substr(s, prev, match.first-1))

lib/nimbase.h

@@ -389,8 +389,6 @@ static N_INLINE(NI32, float32ToInt32)(float x) {
      NIM_CHAR data[(length) + 1];          \
   } name = {{length, (NI) ((NU)length | NIM_STRLIT_FLAG)}, str}
 
-typedef struct TStringDesc* string;
-
 /* declared size of a sequence/variable length array: */
 #if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
 #  define SEQ_DECL_SIZE /* empty is correct! */

lib/packages/fsmonitor.nim

@@ -105,7 +105,7 @@ proc add*(monitor: FSMonitor, target: string,
 proc del*(monitor: FSMonitor, wd: cint) =
   ## Removes watched directory or file as specified by ``wd`` from ``monitor``.
   ##
-  ## If ``wd`` is not a part of ``monitor`` an EOS error is raised.
+  ## If ``wd`` is not a part of ``monitor`` an OSError error is raised.
   if inotifyRmWatch(monitor.fd, wd) < 0:
     raiseOSError(osLastError())
 

lib/pure/asyncdispatch.nim

@@ -145,7 +145,9 @@ export asyncfutures, asyncstreams
 ##
 ## Futures should **never** be discarded. This is because they may contain
 ## errors. If you do not care for the result of a Future then you should
-## use the ``asyncCheck`` procedure instead of the ``discard`` keyword.
+## use the ``asyncCheck`` procedure instead of the ``discard`` keyword. Note
+## however that this does not wait for completion, and you should use
+## ``waitFor`` for that purpose.
 ##
 ## Examples
 ## --------

lib/pure/asyncfutures.nim

@@ -339,7 +339,8 @@ proc asyncCheck*[T](future: Future[T]) =
   ## Sets a callback on ``future`` which raises an exception if the future
   ## finished with an error.
   ##
-  ## This should be used instead of ``discard`` to discard void futures.
+  ## This should be used instead of ``discard`` to discard void futures,
+  ## or use ``waitFor`` if you need to wait for the future's completion.
   assert(not future.isNil, "Future is nil")
   future.callback =
     proc () =

lib/pure/asyncmacro.nim

@@ -245,6 +245,12 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
 
   var outerProcBody = newNimNode(nnkStmtList, prc.body)
 
+  # Extract the documentation comment from the original procedure declaration.
+  # Note that we're not removing it from the body in order not to make this
+  # transformation even more complex.
+  if prc.body.len > 1 and prc.body[0].kind == nnkCommentStmt:
+    outerProcBody.add(prc.body[0])
+
   # -> var retFuture = newFuture[T]()
   var retFutureSym = genSym(nskVar, "retFuture")
   var subRetType =

lib/pure/asyncnet.nim

@@ -218,7 +218,7 @@ when defineSsl:
       var data = await recv(socket.fd.AsyncFD, BufferSize, flags)
       let length = len(data)
       if length > 0:
-        let ret = bioWrite(socket.bioIn, addr data[0], data.len.cint)
+        let ret = bioWrite(socket.bioIn, addr data[0], length.cint)
         if ret < 0:
           raiseSSLError()
       elif length == 0:
@@ -599,7 +599,7 @@ proc listen*(socket: AsyncSocket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].}
   ## ``Backlog`` specifies the maximum length of the
   ## queue of pending connections.
   ##
-  ## Raises an EOS error upon failure.
+  ## Raises an OSError error upon failure.
   if listen(socket.fd, backlog) < 0'i32: raiseOSError(osLastError())
 
 proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {.

lib/pure/asyncstreams.nim

@@ -81,6 +81,9 @@ proc read*[T](future: FutureStream[T]): Future[(bool, T)] =
   let savedCb = future.cb
   future.callback =
     proc (fs: FutureStream[T]) =
+      # Exit early if `resFut` is already complete. (See #8994).
+      if resFut.finished: return
+
       # We don't want this callback called again.
       future.cb = nil
 
@@ -93,8 +96,7 @@ proc read*[T](future: FutureStream[T]): Future[(bool, T)] =
         res[0] = true
         res[1] = fs.queue.popFirst()
 
-      if not resFut.finished:
-        resFut.complete(res)
+      resFut.complete(res)
 
       # If the saved callback isn't nil then let's call it.
       if not savedCb.isNil: savedCb()

lib/pure/bitops.nim

@@ -32,6 +32,18 @@ const useICC_builtins = defined(icc) and useBuiltins
 const useVCC_builtins = defined(vcc) and useBuiltins
 const arch64 = sizeof(int) == 8
 
+template forwardImpl(impl, arg) {.dirty.} =
+  when sizeof(x) <= 4:
+    when x is SomeSignedInt:
+      impl(cast[uint32](x.int32))
+    else:
+      impl(x.uint32)
+  else:
+    when x is SomeSignedInt:
+      impl(cast[uint64](x.int64))
+    else:
+      impl(x.uint64)
+
 # #### Pure Nim version ####
 
 proc firstSetBit_nim(x: uint32): int {.inline, nosideeffect.} =
@@ -185,8 +197,7 @@ proc countSetBits*(x: SomeInteger): int {.inline, nosideeffect.} =
   # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT.
   # like GCC and MSVC
   when nimvm:
-    when sizeof(x) <= 4: result = countSetBits_nim(x.uint32)
-    else:                result = countSetBits_nim(x.uint64)
+    result = forwardImpl(countSetBits_nim, x)
   else:
     when useGCC_builtins:
       when sizeof(x) <= 4: result = builtin_popcount(x.cuint).int
@@ -216,8 +227,7 @@ proc parityBits*(x: SomeInteger): int {.inline, nosideeffect.} =
   # Can be used a base if creating ASM version.
   # https://stackoverflow.com/questions/21617970/how-to-check-if-value-has-even-parity-of-bits-or-odd
   when nimvm:
-    when sizeof(x) <= 4: result = parity_impl(x.uint32)
-    else:                result = parity_impl(x.uint64)
+    result = forwardImpl(parity_impl, x)
   else:
     when useGCC_builtins:
       when sizeof(x) <= 4: result = builtin_parity(x.uint32).int
@@ -235,8 +245,7 @@ proc firstSetBit*(x: SomeInteger): int {.inline, nosideeffect.} =
     when noUndefined:
       if x == 0:
         return 0
-    when sizeof(x) <= 4: result = firstSetBit_nim(x.uint32)
-    else:                result = firstSetBit_nim(x.uint64)
+    result = forwardImpl(firstSetBit_nim, x)
   else:
     when noUndefined and not useGCC_builtins:
       if x == 0:
@@ -270,8 +279,7 @@ proc fastLog2*(x: SomeInteger): int {.inline, nosideeffect.} =
     if x == 0:
       return -1
   when nimvm:
-    when sizeof(x) <= 4: result = fastlog2_nim(x.uint32)
-    else:                result = fastlog2_nim(x.uint64)
+    result = forwardImpl(fastlog2_nim, x)
   else:
     when useGCC_builtins:
       when sizeof(x) <= 4: result = 31 - builtin_clz(x.uint32).int
@@ -302,8 +310,7 @@ proc countLeadingZeroBits*(x: SomeInteger): int {.inline, nosideeffect.} =
     if x == 0:
       return 0
   when nimvm:
-      when sizeof(x) <= 4: result = sizeof(x)*8 - 1 - fastlog2_nim(x.uint32)
-      else:                result = sizeof(x)*8 - 1 - fastlog2_nim(x.uint64)
+    result = sizeof(x)*8 - 1 - forwardImpl(fastlog2_nim, x)
   else:
     when useGCC_builtins:
       when sizeof(x) <= 4: result = builtin_clz(x.uint32).int - (32 - sizeof(x)*8)

lib/pure/collections/sets.nim

@@ -158,10 +158,14 @@ template rawGetKnownHCImpl() {.dirty.} =
     h = nextTry(h, high(s.data))
   result = -1 - h                   # < 0 => MISSING; insert idx = -1 - result
 
-template rawGetImpl() {.dirty.} =
-  hc = hash(key)
+template genHash(key: typed): Hash =
+  var hc = hash(key)
   if hc == 0:       # This almost never taken branch should be very predictable.
     hc = 314159265  # Value doesn't matter; Any non-zero favorite is fine.
+  hc
+
+template rawGetImpl() {.dirty.} =
+  hc = genHash(key)
   rawGetKnownHCImpl()
 
 template rawInsertImpl() {.dirty.} =
@@ -794,39 +798,24 @@ proc incl*[A](s: var HashSet[A], other: OrderedSet[A]) =
 
 proc exclImpl[A](s: var OrderedSet[A], key: A) : bool {. inline .} =
   assert s.isValid, "The set needs to be initialized."
-  var hc: Hash
-  var i = rawGet(s, key, hc)
-  var msk = high(s.data)
+  var n: OrderedKeyValuePairSeq[A]
+  newSeq(n, len(s.data))
+  var h = s.first
+  s.first = -1
+  s.last = -1
+  swap(s.data, n)
+  let hc = genHash(key)
   result = true
-
-  if i >= 0:
-    result = false
-    # Fix ordering
-    if s.first == i:
-      s.first = s.data[i].next
-    else:
-      var itr = s.first
-      while true:
-        if (s.data[itr].next == i):
-          s.data[itr].next = s.data[i].next
-          if s.last == i:
-            s.last = itr
-          break
-        itr = s.data[itr].next
-
-    dec(s.counter)
-    while true:         # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1
-      var j = i         # The correctness of this depends on (h+1) in nextTry,
-      var r = j         # though may be adaptable to other simple sequences.
-      s.data[i].hcode = 0              # mark current EMPTY
-      s.data[i].key = default(type(s.data[i].key))
-      s.data[i].next = 0
-      doWhile((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)):
-        i = (i + 1) and msk            # increment mod table size
-        if isEmpty(s.data[i].hcode):   # end of collision cluster; So all done
-          return
-        r = s.data[i].hcode and msk    # "home" location of key@i
-      shallowCopy(s.data[j], s.data[i]) # data[i] will be marked EMPTY next loop
+  while h >= 0:
+    var nxt = n[h].next
+    if isFilled(n[h].hcode):
+      if n[h].hcode == hc and n[h].key == key:
+        dec s.counter
+        result = false
+      else:
+        var j = -1 - rawGetKnownHC(s, n[h].key, n[h].hcode)
+        rawInsert(s, s.data, n[h].key, n[h].hcode, j)
+    h = nxt
 
 proc missingOrExcl*[A](s: var OrderedSet[A], key: A): bool =
   ## Excludes `key` in the set `s` and tells if `key` was removed from `s`. Efficiency: O(n).
@@ -1097,6 +1086,12 @@ when isMainModule and not defined(release):
       for item in s: items.add item
       assert items == @[2, 6, 7]
 
+    block: #9005
+      var s = initOrderedSet[(int, int)]()
+      for i in 0 .. 30: incl(s, (i, 0))
+      for i in 0 .. 30: excl(s, (i, 0))
+      doAssert s.len == 0
+
     #block orderedSetIterator:
     #  var a = initOrderedSet[int]()
     #  for value in [9, 2, 1, 5, 1, 8, 4, 2]:

lib/pure/colors.nim

@@ -377,7 +377,7 @@ proc colorNameCmp(x: tuple[name: string, col: Color], y: string): int =
 
 proc parseColor*(name: string): Color =
   ## parses `name` to a color value. If no valid color could be
-  ## parsed ``EInvalidValue`` is raised. Case insensitive.
+  ## parsed ``ValueError`` is raised. Case insensitive.
   if name[0] == '#':
     result = Color(parseHexInt(name))
   else:

lib/pure/concurrency/threadpool.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-## Implements Nim's 'spawn'.
+## Implements Nim's `spawn <manual.html#parallel-amp-spawn>`_.
 
 when not compileOption("threads"):
   {.error: "Threadpool requires --threads:on option.".}
@@ -320,6 +320,10 @@ gSomeReady.initSemaphore()
 proc slave(w: ptr Worker) {.thread.} =
   isSlave = true
   while true:
+    if w.shutdown:
+      w.shutdown = false
+      atomicDec currentPoolSize
+      break
     when declared(atomicStoreN):
       atomicStoreN(addr(w.ready), true, ATOMIC_SEQ_CST)
     else:
@@ -340,9 +344,6 @@ proc slave(w: ptr Worker) {.thread.} =
       dec numSlavesRunning
 
     if w.q.len != 0: w.cleanFlowVars
-    if w.shutdown:
-      w.shutdown = false
-      atomicDec currentPoolSize
 
 proc distinguishedSlave(w: ptr Worker) {.thread.} =
   while true: