bump NimVersion to 2.0.0

Revert "fixes #22246; generate `__builtin_unreachable` hints for case defaults (#22350)"

This reverts commit b56df5c07f.

.github/stale.yml

@@ -1,69 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 365
-
-# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
-# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
-daysUntilClose: 30
-
-# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
-exemptLabels:
-  - ARC
-  - bounty
-  - Codegen
-  - Crash
-  - Generics
-  - High Priority
-  - Macros
-  - Next release
-  - Showstopper
-  - Static[T]
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-
-# Set to true to ignore issues with an assignee (defaults to false)
-exemptAssignees: false
-
-# Label to use when marking as stale
-staleLabel: stale
-
-# Comment to post when marking as stale. Set to `false` to disable
-markComment: >
-  This pull request has been automatically marked as stale because it has not had
-  recent activity.
-  If you think it is still a valid PR, please rebase it on the latest devel;
-  otherwise it will be closed. Thank you for your contributions.
-
-# Comment to post when removing the stale label.
-# unmarkComment: >
-#   Your comment here.
-
-# Comment to post when closing a stale Issue or Pull Request.
-# closeComment: >
-#   Your comment here.
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 20
-
-# Limit to only `issues` or `pulls`
-only: pulls
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-#   daysUntilStale: 30
-#   markComment: >
-#     This pull request has been automatically marked as stale because it has not had
-#     recent activity. It will be closed if no further activity occurs. Thank you
-#     for your contributions.
-
-# issues:
-#   exemptLabels:
-#     - confirmed

.github/workflows/bisects.yml

@@ -0,0 +1,23 @@
+# See https://github.com/juancarlospaco/nimrun-action/issues/3#issuecomment-1607344901
+name: issue comments bisects
+on:
+  issue_comment:
+    types: created
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    # nimrun-action requires Nim installed.
+    - uses: jiro4989/setup-nim-action@v1
+      with:
+        nim-version: 'devel'
+
+    - name: Install Dependencies
+      run: sudo apt-get install --no-install-recommends -yq valgrind
+
+    - uses: juancarlospaco/nimrun-action@nim
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci_bench.yml

@@ -0,0 +1,115 @@
+name: Benchmarks CI
+on:
+  pull_request:
+  push:
+    branches:
+      - 'devel'
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04]
+        cpu: [amd64]
+    name: '${{ matrix.os }}'
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60 # refs bug #18178
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - name: 'Install node.js 16.x'
+        uses: actions/setup-node@v3
+        with:
+          node-version: '16.x'
+
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -yq \
+              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
+              valgrind libc6-dbg libblas-dev xorg-dev
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build csourcesAny'
+        shell: bash
+        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build Nim'
+        shell: bash
+        run: ./koch boot -d:release -d:nimStrictMode --lib:lib
+
+      - name: 'Build Nimble'
+        shell: bash
+        run: ./koch nimble
+
+      - name: 'Action'
+        shell: bash
+        run: nim c -r -d:release ci/action.nim
+
+      - name: 'Checkout minimize'
+        uses: actions/checkout@v3
+        with:
+          repository: 'nim-lang/ci_bench'
+          path: minimize
+
+      - name: 'Run minimize benchmarks'
+        shell: bash
+        run: ./minimize/minimize ci-bench
+
+      - name: 'Restore minimize cached database'
+        uses: actions/cache/restore@v3
+        with:
+          path: minimize.csv
+          key: minimize-db-key
+
+      - name: 'Update minimize db'
+        shell: bash
+        run: ./minimize/minimize update-db
+
+      - name: 'Save minimize cached database'
+        if: |
+          github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
+          matrix.target == 'linux'
+        uses: actions/cache/save@v3
+        with:
+          path: minimize.csv
+          key: minimize-db-key
+
+      - name: 'Generate minimize report'
+        shell: bash
+        run: ./minimize/minimize generate-report
+
+      - name: 'Archive minimize report'
+        uses: actions/upload-artifact@v3
+        with:
+          name: minimize-report
+          path: |
+            minimize/minimize.html
+            minimize/minimize.csv
+
+      # Requires additional permissions, see:
+      # https://github.com/nim-lang/Nim/actions/runs/4778177321/jobs/8494423792?pr=21566
+      # - name: 'Publish HTML report'
+      #   uses: rossjrw/pr-preview-action@v1
+      #   with:
+      #     source-dir: minimize
+      #     umbrella-dir: minimize
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract md summary
+        run: |
+          cat minimize/summary.md  >> $GITHUB_STEP_SUMMARY

.github/workflows/stale.yml

@@ -0,0 +1,25 @@
+# https://github.com/actions/stale#usage
+name: Stale pull requests
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Midnight.
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v8
+        with:
+          days-before-pr-stale:    365
+          days-before-pr-close:    30
+          days-before-issue-stale: -1
+          days-before-issue-close: -1
+          exempt-pr-labels:    "ARC,bounty,Codegen,Crash,Generics,High Priority,Macros,Next release,Showstopper,Static[T]"
+          exempt-issue-labels: "Showstopper,Severe,bounty,Compiler Crash,Medium Priority"
+          stale-pr-message: >
+            This pull request is stale because it has been open for 1 year with no activity.
+            Contribute more commits on the pull request and rebase it on the latest devel,
+            or it will be closed in 30 days. Thank you for your contributions.
+          close-pr-message: >
+            This pull request has been marked as stale and closed due to inactivity after 395 days.

.gitignore

@@ -67,6 +67,7 @@ testament.db
 
 /csources
 /csources_v1
+/csources_v2
 
 /dist/
 # /lib/fusion # fusion is now unbundled; `git status` should reveal if it's there so users can act on it
@@ -109,3 +110,5 @@ htmldocs
 nimdoc.out.css
 # except here:
 !/nimdoc/testproject/expected/*
+pkgs/
+/compiler/compiler/

.gitlab-ci.yml

@@ -1,62 +0,0 @@
-# xxx unused, out of date
-
-image: ubuntu:18.04
-
-stages:
-  - pre-build
-  - build
-  - deploy
-  - test
-
-.linux_set_path: &linux_set_path_def
-  before_script:
-    - export PATH=$(pwd)/bin${PATH:+:$PATH}
-  tags:
-    - linux
-
-.windows_set_path: &win_set_path_def
-  before_script:
-    - set PATH=%CD%\bin;%PATH%
-  tags:
-    - windows
-
-
-build-windows:
-  stage: build
-  script:
-    - ci\build.bat
-  artifacts:
-    paths:
-      - bin\nim.exe
-      - bin\nimd.exe
-      - compiler\nim.exe
-      - koch.exe
-    expire_in: 1 week
-  tags:
-    - windows
-
-deploy-windows:
-  stage: deploy
-  script:
-    - koch.exe winrelease
-  artifacts:
-    paths:
-      - build/*.exe
-      - build/*.zip
-    expire_in: 1 week
-  tags:
-    - windows
-    - fast
-
-
-
-test-windows:
-  stage: test
-  <<: *win_set_path_def
-  script:
-    - call ci\deps.bat
-    - nim c testament\tester
-    - testament\tester.exe all
-  tags:
-    - windows
-    - fast

azure-pipelines.yml

@@ -23,11 +23,11 @@ jobs:
         vmImage: 'ubuntu-20.04'
         CPU: amd64
       # regularly breaks, refs bug #17325
-      Linux_i386:
-        # on 'ubuntu-16.04' (not supported anymore anyways) it errored with:
-        # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed
-        vmImage: 'ubuntu-18.04'
-        CPU: i386
+#       Linux_i386:
+#         # on 'ubuntu-16.04' (not supported anymore anyways) it errored with:
+#         # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed
+#         vmImage: 'ubuntu-18.04'
+#         CPU: i386
       OSX_amd64:
         vmImage: 'macOS-11'
         CPU: amd64

bin/nim-gdb

@@ -14,7 +14,7 @@ else
 fi
 
 # Find out where the pretty printer Python module is
-GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"
+GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/debug/nim-gdb.py"
 
 # Run GDB with the additional arguments that load the pretty printers
 # Set the environment variable `NIM_GDB` to overwrite the call to a

bin/nim-gdb.bat

@@ -3,7 +3,7 @@ for %%i in (nim.exe) do (set NIM_BIN=%%~dp$PATH:i)
 
 for %%i in ("%NIM_BIN%\..\") do (set NIM_ROOT=%%~fi)
 
-set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\nim-gdb.py
+set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\debug\nim-gdb.py
 set @NIM_GDB=gdb.exe
 
 @echo source %@GDB_PYTHON_MODULE_PATH%> wingdbcommand.txt

changelog.md

@@ -1,319 +1,30 @@
-# v2.0.0 - yyyy-mm-dd
+# v2.2.0 - yyyy-mm-dd
 
 
 ## Changes affecting backward compatibility
-- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response, it followed Apache HttpClient(Java), http(go) and .Net HttpWebResponse(C#) behavior. Previously it raised `ValueError`.
 
-- `addr` is now available for all addressable locations,
-  `unsafeAddr` is now deprecated and an alias for `addr`.
-
-- Certain definitions from the default `system` module have been moved to
-  the following new modules:
-
-  - `std/syncio`
-  - `std/assertions`
-  - `std/formatfloat`
-  - `std/objectdollar`
-  - `std/widestrs`
-  - `std/typedthreads`
-  - `std/sysatomics`
-
-  In the future, these definitions will be removed from the `system` module,
-  and their respective modules will have to be imported to use them.
-  Currently, to make these imports required, the `-d:nimPreviewSlimSystem` option
-  may be used.
-
-- Enabling `-d:nimPreviewSlimSystem` also removes the following deprecated
-  symbols in the `system` module:
-  - Aliases with `Error` suffix to exception types that have a `Defect` suffix
-    (see [exceptions](https://nim-lang.org/docs/exceptions.html)):
-    `ArithmeticError`, `DivByZeroError`, `OverflowError`,
-    `AccessViolationError`, `AssertionError`, `OutOfMemError`, `IndexError`,
-    `FieldError`, `RangeError`, `StackOverflowError`, `ReraiseError`,
-    `ObjectAssignmentError`, `ObjectConversionError`, `FloatingPointError`,
-    `FloatOverflowError`, `FloatUnderflowError`, `FloatInexactError`,
-    `DeadThreadError`, `NilAccessError`
-  - `addQuitProc`, replaced by `exitprocs.addExitProc`
-  - Legacy unsigned conversion operations: `ze`, `ze64`, `toU8`, `toU16`, `toU32`
-  - `TaintedString`, formerly a distinct alias to `string`
-  - `PInt32`, `PInt64`, `PFloat32`, `PFloat64`, aliases to
-    `ptr int32`, `ptr int64`, `ptr float32`, `ptr float64`
-
-- Enabling `-d:nimPreviewSlimSystem` removes the import of `channels_builtin` in
-  in the `system` module.
-
-- Enabling `-d:nimPreviewCstringConversion`, `ptr char`, `ptr array[N, char]` and `ptr UncheckedArray[N, char]` don't support conversion to cstring anymore.
-
-- The `gc:v2` option is removed.
-
-- The `mainmodule` and `m` options are removed.
-
-- The `threads:on` option is now the default.
-
-- Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via
-  `experimental:flexibleOptionalParams`.
-
-- Automatic dereferencing (experimental feature) is removed.
-
-- The `Math.trunc` polyfill for targeting Internet Explorer was
-  previously included in most JavaScript output files.
-  Now, it is only included with `-d:nimJsMathTruncPolyfill`.
-  If you are targeting Internet Explorer, you may choose to enable this option
-  or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma).
-  Nim uses `Math.trunc` for the division and modulo operators for integers.
-
-- `shallowCopy` and `shallow` are removed for ARC/ORC. Use `move` when possible or combine assignment and
-`sink` for optimization purposes.
-
-- The `nimPreviewDotLikeOps` define is going to be removed or deprecated.
-
-- The `{.this.}` pragma, deprecated since 0.19, has been removed.
-- `nil` literals can no longer be directly assigned to variables or fields of `distinct` pointer types. They must be converted instead.
-  ```nim
-  type Foo = distinct ptr int
-
-  # Before:
-  var x: Foo = nil
-  # After:
-  var x: Foo = Foo(nil)
-  ```
-- Removed two type pragma syntaxes deprecated since 0.20, namely
-  `type Foo = object {.final.}`, and `type Foo {.final.} [T] = object`.
-
-- `foo a = b` now means `foo(a = b)` rather than `foo(a) = b`. This is consistent
-  with the existing behavior of `foo a, b = c` meaning `foo(a, b = c)`.
-  This decision was made with the assumption that the old syntax was used rarely;
-  if your code used the old syntax, please be aware of this change.
-
-- [Overloadable enums](https://nim-lang.github.io/Nim/manual.html#overloadable-enum-value-names) and Unicode Operators
-  are no longer experimental.
-
-- Removed the `nimIncrSeqV3` define.
-
-- `macros.getImpl` for `const` symbols now returns the full definition node
-  (as `nnkConstDef`) rather than the AST of the constant value.
-
-- Lock levels are deprecated, now a noop.
-
-- ORC is now the default memory management strategy. Use
-  `--mm:refc` for a transition period.
-
-- `strictEffects` are no longer experimental.
-  Use `legacy:laxEffects` to keep backward compatibility.
-
-- The `gorge`/`staticExec` calls will now return a descriptive message in the output
-  if the execution fails for whatever reason. To get back legacy behaviour use `-d:nimLegacyGorgeErrors`.
-
-- Pointer to `cstring` conversion now triggers a `[PtrToCstringConv]` warning.
-  This warning will become an error in future versions! Use a `cast` operation
-  like `cast[cstring](x)` instead.
-
-- `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`.
-
-- Object fields now support default values, see https://nim-lang.github.io/Nim/manual.html#types-default-values-for-object-fields for details.
-
-- Redefining templates with the same signature was previously
-  allowed to support certain macro code. To do this explicitly, the
-  `{.redefine.}` pragma has been added. Note that this is only for templates.
-  Implicit redefinition of templates is now deprecated and will give an error in the future.
-
-- Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead.
-
-- Several Standard libraries are moved to nimble packages, use `nimble` to install them:
-  - `std/punycode` => `punycode`
-  - `std/asyncftpclient` => `asyncftpclient`
-  - `std/smtp` => `smtp`
-  - `std/db_common` => `db_connector/db_common`
-  - `std/db_sqlite` => `db_connector/db_sqlite`
-  - `std/db_mysql` => `db_connector/db_mysql`
-  - `std/db_postgres` => `db_connector/db_postgres`
-  - `std/db_odbc` => `db_connector/db_odbc`
-
-- Previously, calls like `foo(a, b): ...` or `foo(a, b) do: ...` where the final argument of
-  `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`.
-  This behavior is now deprecated. Use `foo(a, b) do (): ...` or `foo(a, b, proc () = ...)` instead.
 
 ## Standard library additions and changes
 
 [//]: # "Changes:"
-- OpenSSL 3 is now supported.
-- `macros.parseExpr` and `macros.parseStmt` now accept an optional
-  filename argument for more informative errors.
-- Module `colors` expanded with missing colors from the CSS color standard.
-  `colPaleVioletRed` and `colMediumPurple` have also been changed to match the CSS color standard.
-- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)).
-- The `md5` module now works at compile time and in JavaScript.
-- Changed `mimedb` to use an `OrderedTable` instead of `OrderedTableRef` to support `const` tables.
-- `strutils.find` now uses and defaults to `last = -1` for whole string searches,
-  making limiting it to just the first char (`last = 0`) valid.
-- `random.rand` now works with `Ordinal`s.
-- Undeprecated `os.isvalidfilename`.
-- `std/oids` now uses `int64` to store time internally (before it was int32).
-- `std/uri.Uri` dollar `$` improved, precalculates the `string` result length from the `Uri`.
-- `std/uri.Uri.isIpv6` is now exported.
-- `std/logging.ConsoleLogger` and `FileLogger` now have a `flushThreshold` attribute to set what log message levels are automatically flushed. For Nim v1 use `-d:nimFlushAllLogs` to automatically flush all message levels. Flushing all logs is the default behavior for Nim v2.
-
-
-- `std/net.IpAddress` dollar `$` improved, uses a fixed capacity for the `string` result based from the `IpAddressFamily`.
-- `std/jsfetch.newFetchOptions` now has default values for all parameters
-- `std/jsformdata` now accepts `Blob` data type.
 
 
 [//]: # "Additions:"
-- Added ISO 8601 week date utilities in `times`:
-  - Added `IsoWeekRange`, a range type for weeks in a week-based year.
-  - Added `IsoYear`, a distinct type for a week-based year in contrast to a regular year.
-  - Added a `initDateTime` overload to create a datetime from an ISO week date.
-  - Added `getIsoWeekAndYear` to get an ISO week number and week-based year from a datetime.
-  - Added `getIsoWeeksInYear` to return the number of weeks in a week-based year.
-- Added new modules which were part of `std/os`:
-  - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling.
-  - Added `std/paths`, `std/dirs`, `std/files`, `std/symlinks` and `std/appdirs`.
-  - Added `std/cmdline` for reading command line parameters.
-- Added `sep` parameter in `std/uri` to specify the query separator.
-- Added bindings to [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift)
-  and [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask)
-  in `jscore` for JavaScript targets.
-- Added `UppercaseLetters`, `LowercaseLetters`, `PunctuationChars`, `PrintableChars` sets to `std/strutils`.
-- Added `complex.sgn` for obtaining the phase of complex numbers.
-- Added `insertAdjacentText`, `insertAdjacentElement`, `insertAdjacentHTML`,
-  `after`, `before`, `closest`, `append`, `hasAttributeNS`, `removeAttributeNS`,
-  `hasPointerCapture`, `releasePointerCapture`, `requestPointerLock`,
-  `replaceChildren`, `replaceWith`, `scrollIntoViewIfNeeded`, `setHTML`,
-  `toggleAttribute`, and `matches` to `std/dom`.
-- Added [`jsre.hasIndices`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/hasIndices)
-- Added `capacity` for `string` and `seq` to return the current capacity, see https://github.com/nim-lang/RFCs/issues/460
-- Added `openArray[char]` overloads for `std/parseutils` allowing more code reuse.
-- Added `openArray[char]` overloads for `std/unicode` allowing more code reuse.
-- Added `safe` parameter to `base64.encodeMime`.
 
 [//]: # "Deprecations:"
-- Deprecated `selfExe` for Nimscript.
-- Deprecated `std/sums`.
-- Deprecated `std/base64.encode` for collections of arbitrary integer element type.
-  Now only `byte` and `char` are supported.
+
 
 [//]: # "Removals:"
-- Removed deprecated module `parseopt2`.
-- Removed deprecated module `sharedstrings`.
-- Removed deprecated module `dom_extensions`.
-- Removed deprecated module `LockFreeHash`.
-- Removed deprecated module `events`.
-- Removed deprecated `oids.oidToString`.
-- Removed define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`.
-- Removed deprecated `jsre.test` and `jsre.toString`.
-- Removed deprecated `math.c_frexp`.
-- Removed deprecated `` httpcore.`==` ``.
-- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types.
-- Removed deprecated `osproc.poDemon`, symbol with typo.
-- Removed deprecated `tables.rightSize`.
-
-
-- Removed deprecated `posix.CLONE_STOPPED`.
 
 
 ## Language changes
 
-- [Tag tracking](https://nim-lang.github.io/Nim/manual.html#effect-system-tag-tracking) supports the definition of forbidden tags by the `.forbids` pragma
-  which can be used to disable certain effects in proc types.
-- [Case statement macros](https://nim-lang.github.io/Nim/manual.html#macros-case-statement-macros) are no longer experimental,
-  meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them.
-- Full command syntax and block arguments i.e. `foo a, b: c` are now allowed
-  for the right-hand side of type definitions in type sections. Previously
-  they would error with "invalid indentation".
-
-- Compile-time define changes:
-  - `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`.
-    In the command line, this is defined as `-d:a.b.c`. Older versions can
-    use accents as in ``defined(`a.b.c`)`` to access such defines.
-  - [Define pragmas for constants](https://nim-lang.github.io/Nim/manual.html#implementation-specific-pragmas-compileminustime-define-pragmas)
-    now support a string argument for qualified define names.
-
-    ```nim
-    # -d:package.FooBar=42
-    const FooBar {.intdefine: "package.FooBar".}: int = 5
-    echo FooBar # 42
-    ```
-
-    This was added to help disambiguate similar define names for different packages.
-    In older versions, this could only be achieved with something like the following:
-
-    ```nim
-    const FooBar = block:
-      const `package.FooBar` {.intdefine.}: int = 5
-      `package.FooBar`
-    ```
-  - A generic `define` pragma for constants has been added that interprets
-    the value of the define based on the type of the constant value.
-    See the [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#generic-define-pragma)
-    for a list of supported types.
-
-- [Macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas) changes:
-  - Templates now accept macro pragmas.
-  - Macro pragmas for var/let/const sections have been redesigned in a way that works
-    similarly to routine macro pragmas. The new behavior is documented in the
-    [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#extended-macro-pragmas).
-  - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`,
-    allowing multiple type definitions to be injected in place of the original type definition.
-
-    ```nim
-    import macros
-    macro multiply(amount: static int, s: untyped): untyped =
-      let name = $s[0].basename
-      result = newNimNode(nnkTypeSection)
-      for i in 1 .. amount:
-        result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2]))
-    type
-      Foo = object
-      Bar {.multiply: 3.} = object
-        x, y, z: int
-      Baz = object
-    # becomes
-    type
-      Foo = object
-      Bar1 = object
-        x, y, z: int
-      Bar2 = object
-        x, y, z: int
-      Bar3 = object
-        x, y, z: int
-      Baz = object
-    ```
-
-- A new form of type inference called [top-down inference](https://nim-lang.github.io/Nim/manual_experimental.html#topminusdown-type-inference)
-  has been implemented for a variety of basic cases. For example, code like the following now compiles:
-
-  ```nim
-  let foo: seq[(float, byte, cstring)] = @[(1, 2, "abc")]
-  ```
-
-- `cstring` is now accepted as a selector in `case` statements, removing the
-  need to convert to `string`. On the JS backend, this is translated directly
-  to a `switch` statement.
-
-- Nim now supports `out` parameters and ["strict definitions"](https://nim-lang.github.io/Nim/manual_experimental.html#strict-definitions-and-nimout-parameters).
-- Nim now offers a [strict mode](https://nim-lang.github.io/Nim/manual_experimental.html#strict-case-objects) for `case objects`.
 
 
 ## Compiler changes
 
-- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the
-  reality better. (Nim moved away from all techniques based on "tracing".)
 
-- Defines the `gcRefc` symbol which allows writing specific code for the refc GC.
-
-- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`,
-  without requiring `-d:nimVersion140` which is now a noop.
-
-- `--styleCheck`, `--hintAsError` and `--warningAsError` now only apply to the current package.
-
-- The switch `--nimMainPrefix:prefix` has been added to add a prefix to the names of `NimMain` and
-  related functions produced on the backend. This prevents conflicts with other Nim
-  static libraries.
-
-- When compiling for Release the flag `-fno-math-errno` is used for GCC.
 
 
 ## Tool changes
 
-- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs`).

changelogs/changelog.md

@@ -1,33 +0,0 @@
-# v1.xx.x - yyyy-mm-dd
-
-## Changes affecting backward compatibility
-
-## Standard library additions and changes
-
-- `std/sharedlist` and `std/sharedtables` are now deprecated, see RFC [#433](https://github.com/nim-lang/RFCs/issues/433).
-
-### New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall
-
-This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17.
-
-On linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall.
-
-When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be:
-```sh
-$ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details
-$ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8
-$ sh build.sh  # per unix install instructions
-$ bin/nim c koch  # per unix install instructions
-$ ./koch boot -d:release  # per unix install instructions
-$ ./koch tools -d:nimNoGetRandom  # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall
-```
-
-This is necessary to pass when building nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for stdlib (sysrand in particular).
-
-## Language changes
-
-
-## Compiler changes
-
-
-## Tool changes

changelogs/changelog_2_0_0.md

@@ -0,0 +1,330 @@
+# v2.0.0 - 2023-08-01
+
+Version 2.0 is a big milestone with too many changes to list them all here.
+
+For a full list see [details](changelog_2_0_0_details.html).
+
+
+## New features
+
+### Better tuple unpacking
+
+Tuple unpacking for variables is now treated as syntax sugar that directly
+expands into multiple assignments. Along with this, tuple unpacking for
+variables can now be nested.
+
+```nim
+proc returnsNestedTuple(): (int, (int, int), int, int) = (4, (5, 7), 2, 3)
+
+# Now nesting is supported!
+let (x, (_, y), _, z) = returnsNestedTuple()
+
+```
+
+### Improved type inference
+
+A new form of type inference called [top-down inference](https://nim-lang.github.io/Nim/manual_experimental.html#topminusdown-type-inference) has been implemented for a variety of basic cases.
+
+For example, code like the following now compiles:
+
+```nim
+let foo: seq[(float, byte, cstring)] = @[(1, 2, "abc")]
+```
+
+### Forbidden Tags
+
+[Tag tracking](https://nim-lang.github.io/Nim/manual.html#effect-system-tag-tracking) now supports the definition
+of forbidden tags by the `.forbids` pragma which can be used to disable certain effects in proc types.
+
+For example:
+
+```nim
+
+type IO = object ## input/output effect
+proc readLine(): string {.tags: [IO].} = discard
+proc echoLine(): void = discard
+
+proc no_IO_please() {.forbids: [IO].} =
+  # this is OK because it didn't define any tag:
+  echoLine()
+  # the compiler prevents this:
+  let y = readLine()
+
+```
+
+### New standard library modules
+
+The famous `os` module got an overhaul. Several of its features are available
+under a new interface that introduces a `Path` abstraction. A `Path` is
+a `distinct string`, which improves the type safety when dealing with paths, files
+and directories.
+
+Use:
+
+- `std/oserrors` for OS error reporting.
+- `std/envvars` for environment variables handling.
+- `std/paths` for path handling.
+- `std/dirs` for directory creation/deletion/traversal.
+- `std/files` for file existence checking, file deletions and moves.
+- `std/symlinks` for symlink handling.
+- `std/appdirs` for accessing configuration/home/temp directories.
+- `std/cmdline` for reading command line parameters.
+
+### Consistent underscore handling
+
+The underscore identifier (`_`) is now generally not added to scope when
+used as the name of a definition. While this was already the case for
+variables, it is now also the case for routine parameters, generic
+parameters, routine declarations, type declarations, etc. This means that the following code now does not compile:
+
+```nim
+proc foo(_: int): int = _ + 1
+echo foo(1)
+
+proc foo[_](t: typedesc[_]): seq[_] = @[default(_)]
+echo foo[int]()
+
+proc _() = echo "_"
+_()
+
+type _ = int
+let x: _ = 3
+```
+
+Whereas the following code now compiles:
+
+```nim
+proc foo(_, _: int): int = 123
+echo foo(1, 2)
+
+proc foo[_, _](): int = 123
+echo foo[int, bool]()
+
+proc foo[T, U](_: typedesc[T], _: typedesc[U]): (T, U) = (default(T), default(U))
+echo foo(int, bool)
+
+proc _() = echo "one"
+proc _() = echo "two"
+
+type _ = int
+type _ = float
+```
+
+### JavaScript codegen improvement
+
+The JavaScript backend now uses [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt)
+for 64-bit integer types (`int64` and `uint64`) by default. As this affects
+JS code generation, code using these types to interface with the JS backend
+may need to be updated. Note that `int` and `uint` are not affected.
+
+For compatibility with [platforms that do not support BigInt](https://caniuse.com/bigint)
+and in the case of potential bugs with the new implementation, the
+old behavior is currently still supported with the command line option
+`--jsbigint64:off`.
+
+
+## Docgen improvements
+
+`Markdown` is now the default markup language of doc comments (instead
+of the legacy `RstMarkdown` mode). In this release we begin to separate
+RST and Markdown features to better follow specification of each
+language, with the focus on Markdown development.
+See also [the docs](https://nim-lang.github.io/Nim/markdown_rst.html).
+
+* Added a `{.doctype: Markdown | RST | RstMarkdown.}` pragma allowing to
+  select the markup language mode in the doc comments of the current `.nim`
+  file for processing by `nim doc`:
+
+    1. `Markdown` (default) is basically CommonMark (standard Markdown) +
+        some Pandoc Markdown features + some RST features that are missing
+        in our current implementation of CommonMark and Pandoc Markdown.
+    2. `RST` closely follows the RST spec with few additional Nim features.
+    3. `RstMarkdown` is a maximum mix of RST and Markdown features, which
+        is kept for the sake of compatibility and ease of migration.
+
+* Added separate `md2html` and `rst2html` commands for processing
+  standalone `.md` and `.rst` files respectively (and also `md2tex`/`rst2tex`).
+
+* Added Pandoc Markdown bracket syntax `[...]` for making anchor-less links.
+* Docgen now supports concise syntax for referencing Nim symbols:
+  instead of specifying HTML anchors directly one can use original
+  Nim symbol declarations (adding the aforementioned link brackets
+  `[...]` around them).
+  * To use this feature across modules, a new `importdoc` directive was added.
+    Using this feature for referencing also helps to ensure that links
+    (inside one module or the whole project) are not broken.
+* Added support for RST & Markdown quote blocks (blocks starting with `>`).
+* Added a popular Markdown definition lists extension.
+* Added Markdown indented code blocks (blocks indented by >= 4 spaces).
+* Added syntax for additional parameters to Markdown code blocks:
+
+       ```nim test="nim c $1"
+       ...
+       ```
+
+
+## C++ interop enhancements
+
+Nim 2.0 takes C++ interop to the next level. With the new [virtual](https://nim-lang.github.io/Nim/manual_experimental.html#virtual-pragma) pragma and the extended [constructor](https://nim-lang.github.io/Nim/manual_experimental.html#constructor-pragma) pragma.
+Now one can define constructors and virtual procs that maps to C++ constructors and virtual methods, allowing one to further customize
+the interoperability. There is also extended support for the [codeGenDecl](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-codegendecl-pragma) pragma, so that it works on types.
+
+It's a common pattern in C++ to use inheritance to extend a library. Some even use multiple inheritance as a mechanism to make interfaces.
+
+Consider the following example:
+
+```cpp
+
+struct Base {
+  int someValue;
+  Base(int inValue)  {
+    someValue = inValue;
+  };
+};
+
+class IPrinter {
+public:
+  virtual void print() = 0;
+};
+```
+
+```nim
+
+type
+  Base* {.importcpp, inheritable.} = object
+    someValue*: int32
+  IPrinter* {.importcpp.} = object
+
+const objTemplate = """
+  struct $1 : public $3, public IPrinter {
+    $2
+  };
+""";
+
+type NimChild {.codegenDecl: objTemplate .} = object of Base
+
+proc makeNimChild(val: int32): NimChild {.constructor: "NimClass('1 #1) : Base(#1)".} =
+  echo "It calls the base constructor passing " & $this.someValue
+  this.someValue = val * 2 # Notice how we can access `this` inside the constructor. It's of the type `ptr NimChild`.
+
+proc print*(self: NimChild) {.virtual.} =
+  echo "Some value is " & $self.someValue
+
+let child = makeNimChild(10)
+child.print()
+```
+
+It outputs:
+
+```
+It calls the base constructor passing 10
+Some value is 20
+```
+
+
+## ARC/ORC refinements
+
+With the 2.0 release, the ARC/ORC model got refined once again and is now finally complete:
+
+1. Programmers now have control over the "item was moved from" state as `=wasMoved` is overridable.
+2. There is a new `=dup` hook which is more efficient than the old combination of `=wasMoved(tmp); =copy(tmp, x)` operations.
+3. Destructors now take a parameter of the attached object type `T` directly and don't have to take a `var T` parameter.
+
+With these important optimizations we improved the runtime of the compiler and important benchmarks by 0%! Wait ... what?
+Yes, unfortunately it turns out that for a modern optimizer like in GCC or LLVM there is no difference.
+
+But! This refined model is more efficient once separate compilation enters the picture. In other words, as we think of
+providing a stable ABI it is important not to lose any efficiency in the calling conventions.
+
+
+## Tool changes
+
+- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs` before). Use `nimble develop --global` to create an old style link file in the special links directory documented at https://github.com/nim-lang/nimble#nimble-develop.
+- nimgrep now offers the option `--inContext` (and `--notInContext`), which
+  allows to filter only matches with the context block containing a given pattern.
+- nimgrep: names of options containing "include/exclude" are deprecated,
+  e.g. instead of `--includeFile` and `--excludeFile` we have
+  `--filename` and `--notFilename` respectively.
+  Also, the semantics are now consistent for such positive/negative filters.
+- Nim now ships with an alternative package manager called Atlas. More on this in upcoming versions.
+
+
+## Porting guide
+
+### Block and Break
+
+Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead. In other words, turn:
+
+```nim
+
+block:
+  a()
+  if cond:
+    break
+  b()
+
+```
+
+Into:
+
+```nim
+
+block maybePerformB:
+  a()
+  if cond:
+    break maybePerformB
+  b()
+
+```
+
+### Strict funcs
+
+The definition of `"strictFuncs"` was changed.
+The old definition was roughly: "A store to a ref/ptr deref is forbidden unless it's coming from a `var T` parameter".
+The new definition is: "A store to a ref/ptr deref is forbidden."
+
+This new definition is much easier to understand, the price is some expressitivity. The following code used to be
+accepted:
+
+```nim
+
+{.experimental: "strictFuncs".}
+
+type Node = ref object
+  s: string
+
+func create(s: string): Node =
+  result = Node()
+  result.s = s # store to result[]
+
+```
+
+Now it has to be rewritten to:
+
+```nim
+
+{.experimental: "strictFuncs".}
+
+type Node = ref object
+  s: string
+
+func create(s: string): Node =
+  result = Node(s: s)
+
+```
+
+### Standard library
+
+Several standard library modules have been moved to nimble packages, use `nimble` or `atlas` to install them:
+
+- `std/punycode` => `punycode`
+- `std/asyncftpclient` => `asyncftpclient`
+- `std/smtp` => `smtp`
+- `std/db_common` => `db_connector/db_common`
+- `std/db_sqlite` => `db_connector/db_sqlite`
+- `std/db_mysql` => `db_connector/db_mysql`
+- `std/db_postgres` => `db_connector/db_postgres`
+- `std/db_odbc` => `db_connector/db_odbc`
+- `std/md5` => `checksums/md5`
+- `std/sha1` => `checksums/sha1`
+- `std/sums` => `sums`

changelogs/changelog_2_0_0_details.md

@@ -0,0 +1,560 @@
+# v2.0.0 - 2023-08-01
+
+
+## Changes affecting backward compatibility
+
+- ORC is now the default memory management strategy. Use
+  `--mm:refc` for a transition period.
+
+- The `threads:on` option is now the default.
+
+- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response. It follows Apache's `HttpClient` (Java), `http` (go) and .NET `HttpWebResponse` (C#) behaviors. Previously it raised a `ValueError`.
+
+- `addr` is now available for all addressable locations,
+  `unsafeAddr` is now deprecated and an alias for `addr`.
+
+- Certain definitions from the default `system` module have been moved to
+  the following new modules:
+
+  - `std/syncio`
+  - `std/assertions`
+  - `std/formatfloat`
+  - `std/objectdollar`
+  - `std/widestrs`
+  - `std/typedthreads`
+  - `std/sysatomics`
+
+  In the future, these definitions will be removed from the `system` module,
+  and their respective modules will have to be imported to use them.
+  Currently, to make these imports required, the `-d:nimPreviewSlimSystem` option
+  may be used.
+
+- Enabling `-d:nimPreviewSlimSystem` also removes the following deprecated
+  symbols in the `system` module:
+  - Aliases with an `Error` suffix to exception types that have a `Defect` suffix
+    (see [exceptions](https://nim-lang.github.io/Nim/exceptions.html)):
+    `ArithmeticError`, `DivByZeroError`, `OverflowError`,
+    `AccessViolationError`, `AssertionError`, `OutOfMemError`, `IndexError`,
+    `FieldError`, `RangeError`, `StackOverflowError`, `ReraiseError`,
+    `ObjectAssignmentError`, `ObjectConversionError`, `FloatingPointError`,
+    `FloatOverflowError`, `FloatUnderflowError`, `FloatInexactError`,
+    `DeadThreadError`, `NilAccessError`
+  - `addQuitProc`, replaced by `exitprocs.addExitProc`
+  - Legacy unsigned conversion operations: `ze`, `ze64`, `toU8`, `toU16`, `toU32`
+  - `TaintedString`, formerly a distinct alias to `string`
+  - `PInt32`, `PInt64`, `PFloat32`, `PFloat64`, aliases to
+    `ptr int32`, `ptr int64`, `ptr float32`, `ptr float64`
+
+- Enabling `-d:nimPreviewSlimSystem` removes the import of `channels_builtin` in
+  in the `system` module, which is replaced by [threading/channels](https://github.com/nim-lang/threading/blob/master/threading/channels.nim). Use the command `nimble install threading` and import `threading/channels`.
+
+- Enabling `-d:nimPreviewCstringConversion` causes `ptr char`, `ptr array[N, char]` and `ptr UncheckedArray[N, char]` to not support conversion to `cstring` anymore.
+
+- Enabling `-d:nimPreviewProcConversion` causes `proc` to not support conversion to
+  `pointer` anymore. `cast` may be used instead.
+
+- The `gc:v2` option is removed.
+
+- The `mainmodule` and `m` options are removed.
+
+- Optional parameters in combination with `: body` syntax ([RFC #405](https://github.com/nim-lang/RFCs/issues/405))
+  are now opt-in via `experimental:flexibleOptionalParams`.
+
+- Automatic dereferencing (experimental feature) is removed.
+
+- The `Math.trunc` polyfill for targeting Internet Explorer was
+  previously included in most JavaScript output files.
+  Now, it is only included with `-d:nimJsMathTruncPolyfill`.
+  If you are targeting Internet Explorer, you may choose to enable this option
+  or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma).
+  Nim uses `Math.trunc` for the division and modulo operators for integers.
+
+- `shallowCopy` and `shallow` are removed for ARC/ORC. Use `move` when possible or combine assignment and
+`sink` for optimization purposes.
+
+- The experimental `nimPreviewDotLikeOps` switch is going to be removed or deprecated because it didn't fullfill its promises.
+
+- The `{.this.}` pragma, deprecated since 0.19, has been removed.
+- `nil` literals can no longer be directly assigned to variables or fields of `distinct` pointer types. They must be converted instead.
+  ```nim
+  type Foo = distinct ptr int
+
+  # Before:
+  var x: Foo = nil
+  # After:
+  var x: Foo = Foo(nil)
+  ```
+- Removed two type pragma syntaxes deprecated since 0.20, namely
+  `type Foo = object {.final.}`, and `type Foo {.final.} [T] = object`. Instead,
+  use `type Foo[T] {.final.} = object`.
+
+- `foo a = b` now means `foo(a = b)` rather than `foo(a) = b`. This is consistent
+  with the existing behavior of `foo a, b = c` meaning `foo(a, b = c)`.
+  This decision was made with the assumption that the old syntax was used rarely;
+  if your code used the old syntax, please be aware of this change.
+
+- [Overloadable enums](https://nim-lang.github.io/Nim/manual.html#overloadable-enum-value-names) and Unicode Operators
+  are no longer experimental.
+
+- `macros.getImpl` for `const` symbols now returns the full definition node
+  (as `nnkConstDef`) rather than the AST of the constant value.
+
+- Lock levels are deprecated, now a noop.
+
+- `strictEffects` are no longer experimental.
+  Use `legacy:laxEffects` to keep backward compatibility.
+
+- The `gorge`/`staticExec` calls will now return a descriptive message in the output
+  if the execution fails for whatever reason. To get back legacy behaviour, use `-d:nimLegacyGorgeErrors`.
+
+- Pointer to `cstring` conversions now trigger a `[PtrToCstringConv]` warning.
+  This warning will become an error in future versions! Use a `cast` operation
+  like `cast[cstring](x)` instead.
+
+- `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`.
+
+- Redefining templates with the same signature was previously
+  allowed to support certain macro code. To do this explicitly, the
+  `{.redefine.}` pragma has been added. Note that this is only for templates.
+  Implicit redefinition of templates is now deprecated and will give an error in the future.
+
+- Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead.
+
+- Several Standard libraries have been moved to nimble packages, use `nimble` to install them:
+  - `std/punycode` => `punycode`
+  - `std/asyncftpclient` => `asyncftpclient`
+  - `std/smtp` => `smtp`
+  - `std/db_common` => `db_connector/db_common`
+  - `std/db_sqlite` => `db_connector/db_sqlite`
+  - `std/db_mysql` => `db_connector/db_mysql`
+  - `std/db_postgres` => `db_connector/db_postgres`
+  - `std/db_odbc` => `db_connector/db_odbc`
+  - `std/md5` => `checksums/md5`
+  - `std/sha1` => `checksums/sha1`
+  - `std/sums` => `std/sums`
+
+- Previously, calls like `foo(a, b): ...` or `foo(a, b) do: ...` where the final argument of
+  `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`.
+  This behavior is now deprecated. Use `foo(a, b) do (): ...` or `foo(a, b, proc () = ...)` instead.
+
+- When `--warning[BareExcept]:on` is enabled, if an `except` specifies no exception or any exception not inheriting from `Defect` or `CatchableError`, a `warnBareExcept` warning will be triggered. For example, the following code will emit a warning:
+  ```nim
+  try:
+    discard
+  except: # Warning: The bare except clause is deprecated; use `except CatchableError:` instead [BareExcept]
+    discard
+  ```
+
+- The experimental `strictFuncs` feature now disallows a store to the heap via a `ref` or `ptr` indirection.
+
+- The underscore identifier (`_`) is now generally not added to scope when
+  used as the name of a definition. While this was already the case for
+  variables, it is now also the case for routine parameters, generic
+  parameters, routine declarations, type declarations, etc. This means that the following code now does not compile:
+
+  ```nim
+  proc foo(_: int): int = _ + 1
+  echo foo(1)
+
+  proc foo[_](t: typedesc[_]): seq[_] = @[default(_)]
+  echo foo[int]()
+
+  proc _() = echo "_"
+  _()
+
+  type _ = int
+  let x: _ = 3
+  ```
+
+  Whereas the following code now compiles:
+
+  ```nim
+  proc foo(_, _: int): int = 123
+  echo foo(1, 2)
+
+  proc foo[_, _](): int = 123
+  echo foo[int, bool]()
+
+  proc foo[T, U](_: typedesc[T], _: typedesc[U]): (T, U) = (default(T), default(U))
+  echo foo(int, bool)
+
+  proc _() = echo "one"
+  proc _() = echo "two"
+
+  type _ = int
+  type _ = float
+  ```
+
+- Added the `--legacy:verboseTypeMismatch` switch to get legacy type mismatch error messages.
+
+- The JavaScript backend now uses [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt)
+  for 64-bit integer types (`int64` and `uint64`) by default. As this affects
+  JS code generation, code using these types to interface with the JS backend
+  may need to be updated. Note that `int` and `uint` are not affected.
+
+  For compatibility with [platforms that do not support BigInt](https://caniuse.com/bigint)
+  and in the case of potential bugs with the new implementation, the
+  old behavior is currently still supported with the command line option
+  `--jsbigint64:off`.
+
+- The `proc` and `iterator` type classes now respectively only match
+  procs and iterators. Previously both type classes matched any of
+  procs or iterators.
+
+  ```nim
+  proc prc(): int =
+    123
+
+  iterator iter(): int =
+    yield 123
+
+  proc takesProc[T: proc](x: T) = discard
+  proc takesIter[T: iterator](x: T) = discard
+
+  # always compiled:
+  takesProc(prc)
+  takesIter(iter)
+  # no longer compiles:
+  takesProc(iter)
+  takesIter(prc)
+  ```
+
+- The `proc` and `iterator` type classes now accept a calling convention pragma
+  (i.e. `proc {.closure.}`) that must be shared by matching proc or iterator
+  types. Previously, pragmas were parsed but discarded if no parameter list
+  was given.
+
+  This is represented in the AST by an `nnkProcTy`/`nnkIteratorTy` node with
+  an `nnkEmpty` node in the place of the `nnkFormalParams` node, and the pragma
+  node in the same place as in a concrete `proc` or `iterator` type node. This
+  state of the AST may be unexpected to existing code, both due to the
+  replacement of the `nnkFormalParams` node as well as having child nodes
+  unlike other type class AST.
+
+- Signed integer literals in `set` literals now default to a range type of
+  `0..255` instead of `0..65535` (the maximum size of sets).
+
+- `case` statements with `else` branches put before `elif`/`of` branches in macros
+  are rejected with "invalid order of case branches".
+
+- Destructors now default to `.raises: []` (i.e. destructors must not raise
+  unlisted exceptions) and explicitly raising destructors are implementation
+  defined behavior.
+
+- The very old, undocumented `deprecated` pragma statement syntax for
+  deprecated aliases is now a no-op. The regular deprecated pragma syntax is
+  generally sufficient instead.
+
+  ```nim
+  # now does nothing:
+  {.deprecated: [OldName: NewName].}
+
+  # instead use:
+  type OldName* {.deprecated: "use NewName instead".} = NewName
+  const oldName* {.deprecated: "use newName instead".} = newName
+  ```
+
+  `defined(nimalias)` can be used to check for versions when this syntax was
+  available; however since code that used this syntax is usually very old,
+  these deprecated aliases are likely not used anymore and it may make sense
+  to simply remove these statements.
+
+- `getProgramResult` and `setProgramResult` in `std/exitprocs` are no longer
+  declared when they are not available on the backend. Previously it would call
+  `doAssert false` at runtime despite the condition being checkable at compile-time.
+
+- Custom destructors now supports non-var parameters, e.g. ``proc `=destroy`[T: object](x: T)`` is valid. ``proc `=destroy`[T: object](x: var T)`` is deprecated.
+
+- Relative imports will not resolve to searched paths anymore, e.g. `import ./tables` now reports an error properly.
+
+## Standard library additions and changes
+
+[//]: # "Changes:"
+- OpenSSL 3 is now supported.
+- `macros.parseExpr` and `macros.parseStmt` now accept an optional
+  `filename` argument for more informative errors.
+- The `colors` module is expanded with missing colors from the CSS color standard.
+  `colPaleVioletRed` and `colMediumPurple` have also been changed to match the CSS color standard.
+- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)).
+- The `md5` module now works at compile time and in JavaScript.
+- Changed `mimedb` to use an `OrderedTable` instead of `OrderedTableRef`, to support `const` tables.
+- `strutils.find` now uses and defaults to `last = -1` for whole string searches,
+  making limiting it to just the first char (`last = 0`) valid.
+- `strutils.split` and `strutils.rsplit` now return the source string as a single element for an empty separator.
+- `random.rand` now works with `Ordinal`s.
+- Undeprecated `os.isvalidfilename`.
+- `std/oids` now uses `int64` to store time internally (before, it was int32).
+- `std/uri.Uri` dollar (`$`) improved, precalculates the `string` result length from the `Uri`.
+- `std/uri.Uri.isIpv6` is now exported.
+- `std/logging.ConsoleLogger` and `FileLogger` now have a `flushThreshold` attribute to set what log message levels are automatically flushed. For Nim v1 use `-d:nimFlushAllLogs` to automatically flush all message levels. Flushing all logs is the default behavior for Nim v2.
+
+
+- `std/jsfetch.newFetchOptions` now has default values for all parameters.
+- `std/jsformdata` now accepts the `Blob` data type.
+
+- `std/sharedlist` and `std/sharedtables` are now deprecated, see [RFC #433](https://github.com/nim-lang/RFCs/issues/433).
+
+- There is a new compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove the dependency on the Linux `getrandom` syscall.
+
+  This compile flag only affects Linux builds and is necessary if either compiling on a Linux kernel version < 3.17, or if code built will be executing on kernel < 3.17.
+
+  On Linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since Linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall.
+
+  When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be:
+  ```sh
+  $ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details
+  $ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8
+  $ sh build.sh  # per unix install instructions
+  $ bin/nim c koch  # per unix install instructions
+  $ ./koch boot -d:release  # per unix install instructions
+  $ ./koch tools -d:nimNoGetRandom  # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall
+  ```
+
+  This is necessary to pass when building Nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for the stdlib (`sysrand` in particular).
+
+[//]: # "Additions:"
+- Added ISO 8601 week date utilities in `times`:
+  - Added `IsoWeekRange`, a range type for weeks in a week-based year.
+  - Added `IsoYear`, a distinct type for a week-based year in contrast to a regular year.
+  - Added an `initDateTime` overload to create a `DateTime` from an ISO week date.
+  - Added `getIsoWeekAndYear` to get an ISO week number and week-based year from a datetime.
+  - Added `getIsoWeeksInYear` to return the number of weeks in a week-based year.
+- Added new modules which were previously part of `std/os`:
+  - Added `std/oserrors` for OS error reporting.
+  - Added `std/envvars` for environment variables handling.
+  - Added `std/cmdline` for reading command line parameters.
+  - Added `std/paths`, `std/dirs`, `std/files`, `std/symlinks` and `std/appdirs`.
+- Added `sep` parameter in `std/uri` to specify the query separator.
+- Added `UppercaseLetters`, `LowercaseLetters`, `PunctuationChars`, `PrintableChars` sets to `std/strutils`.
+- Added `complex.sgn` for obtaining the phase of complex numbers.
+- Added `insertAdjacentText`, `insertAdjacentElement`, `insertAdjacentHTML`,
+  `after`, `before`, `closest`, `append`, `hasAttributeNS`, `removeAttributeNS`,
+  `hasPointerCapture`, `releasePointerCapture`, `requestPointerLock`,
+  `replaceChildren`, `replaceWith`, `scrollIntoViewIfNeeded`, `setHTML`,
+  `toggleAttribute`, and `matches` to `std/dom`.
+- Added [`jsre.hasIndices`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/hasIndices).
+- Added `capacity` for `string` and `seq` to return the current capacity, see [RFC #460](https://github.com/nim-lang/RFCs/issues/460).
+- Added `openArray[char]` overloads for `std/parseutils` and `std/unicode`, allowing for more code reuse.
+- Added a `safe` parameter to `base64.encodeMime`.
+- Added `parseutils.parseSize` - inverse to `strutils.formatSize` - to parse human readable sizes.
+- Added `minmax` to `sequtils`, as a more efficient `(min(_), max(_))` over sequences.
+- `std/jscore` for the JavaScript target:
+  + Added bindings to [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift)
+  and [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask).
+  + Added `toDateString`, `toISOString`, `toJSON`, `toTimeString`, `toUTCString` converters for `DateTime`.
+- Added `BackwardsIndex` overload for `CacheSeq`.
+- Added support for nested `with` blocks in `std/with`.
+- Added `ensureMove` to the system module. It ensures that the passed argument is moved, otherwise an error is given at the compile time.
+
+
+[//]: # "Deprecations:"
+- Deprecated `selfExe` for Nimscript.
+- Deprecated `std/base64.encode` for collections of arbitrary integer element type.
+  Now only `byte` and `char` are supported.
+
+[//]: # "Removals:"
+- Removed deprecated module `parseopt2`.
+- Removed deprecated module `sharedstrings`.
+- Removed deprecated module `dom_extensions`.
+- Removed deprecated module `LockFreeHash`.
+- Removed deprecated module `events`.
+- Removed deprecated `oids.oidToString`.
+- Removed define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`.
+- Removed deprecated `jsre.test` and `jsre.toString`.
+- Removed deprecated `math.c_frexp`.
+- Removed deprecated `` httpcore.`==` ``.
+- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that take wrong argument types.
+- Removed deprecated `osproc.poDemon`, symbol with typo.
+- Removed deprecated `tables.rightSize`.
+
+
+- Removed deprecated `posix.CLONE_STOPPED`.
+
+
+## Language changes
+
+- [Tag tracking](https://nim-lang.github.io/Nim/manual.html#effect-system-tag-tracking) now supports the definition of forbidden tags by the `.forbids` pragma
+  which can be used to disable certain effects in proc types.
+- [Case statement macros](https://nim-lang.github.io/Nim/manual.html#macros-case-statement-macros) are no longer experimental,
+  meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them.
+- Full command syntax and block arguments i.e. `foo a, b: c` are now allowed
+  for the right-hand side of type definitions in type sections. Previously
+  they would error with "invalid indentation".
+
+- Compile-time define changes:
+  - `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`.
+    In the command line, this is defined as `-d:a.b.c`. Older versions can
+    use backticks as in ``defined(`a.b.c`)`` to access such defines.
+  - [Define pragmas for constants](https://nim-lang.github.io/Nim/manual.html#implementation-specific-pragmas-compileminustime-define-pragmas)
+    now support a string argument for qualified define names.
+
+    ```nim
+    # -d:package.FooBar=42
+    const FooBar {.intdefine: "package.FooBar".}: int = 5
+    echo FooBar # 42
+    ```
+
+    This was added to help disambiguate similar define names for different packages.
+    In older versions, this could only be achieved with something like the following:
+
+    ```nim
+    const FooBar = block:
+      const `package.FooBar` {.intdefine.}: int = 5
+      `package.FooBar`
+    ```
+  - A generic `define` pragma for constants has been added that interprets
+    the value of the define based on the type of the constant value.
+    See the [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#generic-nimdefine-pragma)
+    for a list of supported types.
+
+- [Macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas) changes:
+  - Templates now accept macro pragmas.
+  - Macro pragmas for var/let/const sections have been redesigned in a way that works
+    similarly to routine macro pragmas. The new behavior is documented in the
+    [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#extended-macro-pragmas).
+  - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`,
+    allowing multiple type definitions to be injected in place of the original type definition.
+
+    ```nim
+    import macros
+
+    macro multiply(amount: static int, s: untyped): untyped =
+      let name = $s[0].basename
+      result = newNimNode(nnkTypeSection)
+      for i in 1 .. amount:
+        result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2]))
+
+    type
+      Foo = object
+      Bar {.multiply: 3.} = object
+        x, y, z: int
+      Baz = object
+    # becomes
+    type
+      Foo = object
+      Bar1 = object
+        x, y, z: int
+      Bar2 = object
+        x, y, z: int
+      Bar3 = object
+        x, y, z: int
+      Baz = object
+    ```
+
+- A new form of type inference called [top-down inference](https://nim-lang.github.io/Nim/manual_experimental.html#topminusdown-type-inference)
+  has been implemented for a variety of basic cases. For example, code like the following now compiles:
+
+  ```nim
+  let foo: seq[(float, byte, cstring)] = @[(1, 2, "abc")]
+  ```
+
+- `cstring` is now accepted as a selector in `case` statements, removing the
+  need to convert to `string`. On the JS backend, this is translated directly
+  to a `switch` statement.
+
+- Nim now supports `out` parameters and ["strict definitions"](https://nim-lang.github.io/Nim/manual_experimental.html#strict-definitions-and-nimout-parameters).
+- Nim now offers a [strict mode](https://nim-lang.github.io/Nim/manual_experimental.html#strict-case-objects) for `case objects`.
+
+- IBM Z architecture and macOS m1 arm64 architecture are supported.
+
+- `=wasMoved` can now be overridden by users.
+
+- There is a new pragma called [quirky](https://nim-lang.github.io/Nim/manual_experimental.html#quirky-routines) that can be used to affect the code
+  generation of goto based exception handling. It can improve the produced code size but its effects can be subtle so use it with care.
+
+- Tuple unpacking for variables is now treated as syntax sugar that directly
+  expands into multiple assignments. Along with this, tuple unpacking for
+  variables can now be nested.
+
+  ```nim
+  proc returnsNestedTuple(): (int, (int, int), int, int) = (4, (5, 7), 2, 3)
+
+  let (x, (_, y), _, z) = returnsNestedTuple()
+  # roughly becomes
+  let
+    tmpTup1 = returnsNestedTuple()
+    x = tmpTup1[0]
+    tmpTup2 = tmpTup1[1]
+    y = tmpTup2[1]
+    z = tmpTup1[3]
+  ```
+
+  As a result `nnkVarTuple` nodes in variable sections will no longer be
+  reflected in `typed` AST.
+
+- C++ interoperability:
+  - New [`virtual`](https://nim-lang.github.io/Nim/manual_experimental.html#virtual-pragma) pragma added.
+  - Improvements to [`constructor`](https://nim-lang.github.io/Nim/manual_experimental.html#constructor-pragma) pragma.
+
+## Compiler changes
+
+- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the
+  reality better. (Nim moved away from all techniques based on "tracing".)
+
+- Defines the `gcRefc` symbol which allows writing specific code for the refc GC.
+
+- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`,
+  without requiring `-d:nimVersion140` which is now a noop.
+
+- `--styleCheck`, `--hintAsError` and `--warningAsError` now only apply to the current package.
+
+- The switch `--nimMainPrefix:prefix` has been added to add a prefix to the names of `NimMain` and
+  related functions produced on the backend. This prevents conflicts with other Nim
+  static libraries.
+
+- When compiling for release, the flag `-fno-math-errno` is used for GCC.
+- Removed deprecated `LineTooLong` hint.
+- Line numbers and file names of source files work correctly inside templates for JavaScript targets.
+
+- Removed support for LCC (Local C), Pelles C, Digital Mars and Watcom compilers.
+
+
+## Docgen
+
+- `Markdown` is now the default markup language of doc comments (instead
+  of the legacy `RstMarkdown` mode). In this release we begin to separate
+  RST and Markdown features to better follow specification of each
+  language, with the focus on Markdown development.
+  See also [the docs](https://nim-lang.github.io/Nim/markdown_rst.html).
+
+  * Added a `{.doctype: Markdown | RST | RstMarkdown.}` pragma allowing to
+    select the markup language mode in the doc comments of the current `.nim`
+    file for processing by `nim doc`:
+
+      1. `Markdown` (default) is basically CommonMark (standard Markdown) +
+         some Pandoc Markdown features + some RST features that are missing
+         in our current implementation of CommonMark and Pandoc Markdown.
+      2. `RST` closely follows the RST spec with few additional Nim features.
+      3. `RstMarkdown` is a maximum mix of RST and Markdown features, which
+          is kept for the sake of compatibility and ease of migration.
+
+  * Added separate `md2html` and `rst2html` commands for processing
+    standalone `.md` and `.rst` files respectively (and also `md2tex`/`rst2tex`).
+
+- Added Pandoc Markdown bracket syntax `[...]` for making anchor-less links.
+- Docgen now supports concise syntax for referencing Nim symbols:
+  instead of specifying HTML anchors directly one can use original
+  Nim symbol declarations (adding the aforementioned link brackets
+  `[...]` around them).
+    * To use this feature across modules, a new `importdoc` directive was added.
+      Using this feature for referencing also helps to ensure that links
+      (inside one module or the whole project) are not broken.
+- Added support for RST & Markdown quote blocks (blocks starting with `>`).
+- Added a popular Markdown definition lists extension.
+- Added Markdown indented code blocks (blocks indented by >= 4 spaces).
+- Added syntax for additional parameters to Markdown code blocks:
+
+       ```nim test="nim c $1"
+       ...
+       ```
+
+## Tool changes
+
+- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs` before). Use `nimble develop --global` to create an old style link file in the special links directory documented at https://github.com/nim-lang/nimble#nimble-develop.
+- nimgrep added the option `--inContext` (and `--notInContext`), which
+  allows to filter only matches with the context block containing a given pattern.
+- nimgrep: names of options containing "include/exclude" are deprecated,
+  e.g. instead of `--includeFile` and `--excludeFile` we have
+  `--filename` and `--notFilename` respectively.
+  Also the semantics are now consistent for such positive/negative filters.
+- koch now supports the `--skipIntegrityCheck` option. The command `koch --skipIntegrityCheck boot -d:release` always builds the compiler twice.

ci/funs.sh

@@ -58,7 +58,11 @@ _nimNumCpu(){
   # FreeBSD | macOS: $(sysctl -n hw.ncpu)
   # OpenBSD: $(sysctl -n hw.ncpuonline)
   # windows: $NUMBER_OF_PROCESSORS ?
-  echo $(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
+  if env | grep -q '^NIMCORES='; then
+    echo $NIMCORES
+  else
+    echo $(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
+  fi
 }
 
 _nimBuildCsourcesIfNeeded(){

compiler/ast.nim

@@ -231,7 +231,7 @@ type
   TNodeKinds* = set[TNodeKind]
 
 type
-  TSymFlag* = enum    # 49 flags!
+  TSymFlag* = enum    # 51 flags!
     sfUsed,           # read access of sym (for warnings) or simply used
     sfExported,       # symbol is exported from module
     sfFromGeneric,    # symbol is instantiation of a generic; this is needed
@@ -283,7 +283,7 @@ type
     sfNamedParamCall, # symbol needs named parameter call syntax in target
                       # language; for interfacing with Objective C
     sfDiscardable,    # returned value may be discarded implicitly
-    sfOverriden,      # proc is overridden
+    sfOverridden,      # proc is overridden
     sfCallsite        # A flag for template symbols to tell the
                       # compiler it should use line information from
                       # the calling side of the macro, not from the
@@ -303,6 +303,8 @@ type
     sfUsedInFinallyOrExcept  # symbol is used inside an 'except' or 'finally'
     sfSingleUsedTemp  # For temporaries that we know will only be used once
     sfNoalias         # 'noalias' annotation, means C's 'restrict'
+                      # for templates and macros, means cannot be called
+                      # as a lone symbol (cannot use alias syntax)
     sfEffectsDelayed  # an 'effectsDelayed' parameter
     sfGeneratedType   # A anonymous generic type that is generated by the compiler for
                       # objects that do not have generic parameters in case one of the
@@ -310,6 +312,9 @@ type
                       #
                       # This is disallowed but can cause the typechecking to go into
                       # an infinite loop, this flag is used as a sentinel to stop it.
+    sfVirtual         # proc is a C++ virtual function
+    sfByCopy          # param is marked as pass bycopy
+    sfCodegenDecl     # type, proc, global or proc param is marked as codegenDecl
 
   TSymFlags* = set[TSymFlag]
 
@@ -335,8 +340,8 @@ const
 
   sfCompileToCpp* = sfInfixCall       # compile the module as C++ code
   sfCompileToObjc* = sfNamedParamCall # compile the module as Objective-C code
-  sfExperimental* = sfOverriden       # module uses the .experimental switch
-  sfGoto* = sfOverriden               # var is used for 'goto' code generation
+  sfExperimental* = sfOverridden       # module uses the .experimental switch
+  sfGoto* = sfOverridden               # var is used for 'goto' code generation
   sfWrittenTo* = sfBorrow             # param is assigned to
                                       # currently unimplemented
   sfBase* = sfDiscriminant
@@ -510,10 +515,10 @@ type
     nfLastRead  # this node is a last read
     nfFirstWrite # this node is a first write
     nfHasComment # node has a comment
-    nfUseDefaultField # node has a default value (object constructor)
+    nfSkipFieldChecking # node skips field visable checking
 
   TNodeFlags* = set[TNodeFlag]
-  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 46)
+  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 47)
     tfVarargs,        # procedure has C styled varargs
                       # tyArray type represeting a varargs list
     tfNoSideEffect,   # procedure type does not allow side effects
@@ -583,6 +588,7 @@ type
     tfIsConstructor
     tfEffectSystemWorkaround
     tfIsOutParam
+    tfSendable
 
   TTypeFlags* = set[TTypeFlag]
 
@@ -618,13 +624,12 @@ type
                           # file (it is loaded on demand, which may
                           # mean: never)
     skPackage,            # symbol is a package (used for canonicalization)
-    skAlias               # an alias (needs to be resolved immediately)
   TSymKinds* = set[TSymKind]
 
 const
   routineKinds* = {skProc, skFunc, skMethod, skIterator,
                    skConverter, skMacro, skTemplate}
-  ExportableSymKinds* = {skVar, skLet, skConst, skType, skEnumField, skStub, skAlias} + routineKinds
+  ExportableSymKinds* = {skVar, skLet, skConst, skType, skEnumField, skStub} + routineKinds
 
   tfUnion* = tfNoSideEffect
   tfGcSafe* = tfThread
@@ -633,7 +638,7 @@ const
   skError* = skUnknown
 
 var
-  eqTypeFlags* = {tfIterator, tfNotNil, tfVarIsPtr, tfGcSafe, tfNoSideEffect, tfIsOutParam}
+  eqTypeFlags* = {tfIterator, tfNotNil, tfVarIsPtr, tfGcSafe, tfNoSideEffect, tfIsOutParam, tfSendable}
     ## type flags that are essential for type equality.
     ## This is now a variable because for emulation of version:1.0 we
     ## might exclude {tfGcSafe, tfNoSideEffect}.
@@ -685,7 +690,7 @@ type
     mIsPartOf, mAstToStr, mParallel,
     mSwap, mIsNil, mArrToSeq, mOpenArrayToSeq,
     mNewString, mNewStringOfCap, mParseBiggestFloat,
-    mMove, mWasMoved, mDestroy, mTrace,
+    mMove, mEnsureMove, mWasMoved, mDup, mDestroy, mTrace,
     mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mAccessTypeField, mReset,
     mArray, mOpenArray, mRange, mSet, mSeq, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
@@ -706,7 +711,7 @@ type
     mNctPut, mNctLen, mNctGet, mNctHasNext, mNctNext,
 
     mNIntVal, mNFloatVal, mNSymbol, mNIdent, mNGetType, mNStrVal, mNSetIntVal,
-    mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo,
+    mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetStrVal, mNLineInfo,
     mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mNSigHash, mNSizeOf,
     mNBindSym, mNCallSite,
     mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym,
@@ -796,6 +801,8 @@ type
       ident*: PIdent
     else:
       sons*: TNodeSeq
+    when defined(nimsuggest):
+      endInfo*: TLineInfo
 
   TStrTable* = object         # a table[PIdent] of PSym
     counter*: int
@@ -816,9 +823,6 @@ type
     locOther                  # location is something other
   TLocFlag* = enum
     lfIndirect,               # backend introduced a pointer
-    lfFullExternalName, # only used when 'conf.cmd == cmdNimfix': Indicates
-      # that the symbol has been imported via 'importc: "fullname"' and
-      # no format string.
     lfNoDeepCopy,             # no need for a deep copy
     lfNoDecl,                 # do not declare it in C
     lfDynamicLib,             # link symbol to dynamic library
@@ -853,7 +857,7 @@ type
                               # keep in sync with PackedLib
     kind*: TLibKind
     generated*: bool          # needed for the backends:
-    isOverriden*: bool
+    isOverridden*: bool
     name*: Rope
     path*: PNode              # can be a string literal!
 
@@ -892,6 +896,8 @@ type
     typ*: PType
     name*: PIdent
     info*: TLineInfo
+    when defined(nimsuggest):
+      endInfo*: TLineInfo
     owner*: PSym
     flags*: TSymFlags
     ast*: PNode               # syntax tree of proc, iterator, etc.:
@@ -913,7 +919,9 @@ type
                               # for modules, an unique index corresponding
                               # to the module's fileIdx
                               # for variables a slot index for the evaluator
-    offset*: int              # offset of record field
+    offset*: int32            # offset of record field
+    disamb*: int32            # disambiguation number; the basic idea is that
+                              # `<procname>__<module>_<disamb>`
     loc*: TLoc
     annex*: PLib              # additional fields (seldom used, so we use a
                               # reference to another object to save space)
@@ -921,7 +929,7 @@ type
       cname*: string          # resolved C declaration name in importc decl, e.g.:
                               # proc fun() {.importc: "$1aux".} => cname = funaux
     constraint*: PNode        # additional constraints like 'lit|result'; also
-                              # misused for the codegenDecl pragma in the hope
+                              # misused for the codegenDecl and virtual pragmas in the hope
                               # it won't cause problems
                               # for skModule the string literal to output for
                               # deprecated modules.
@@ -931,8 +939,10 @@ type
   TTypeSeq* = seq[PType]
 
   TTypeAttachedOp* = enum ## as usual, order is important here
+    attachedWasMoved,
     attachedDestructor,
     attachedAsgn,
+    attachedDup,
     attachedSink,
     attachedTrace,
     attachedDeepCopy
@@ -1076,7 +1086,7 @@ const
                                       nfIsRef, nfIsPtr, nfPreventCg, nfLL,
                                       nfFromTemplate, nfDefaultRefsParam,
                                       nfExecuteOnReload, nfLastRead,
-                                      nfFirstWrite, nfUseDefaultField}
+                                      nfFirstWrite, nfSkipFieldChecking}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2
@@ -1116,11 +1126,10 @@ const
 
 proc getPIdent*(a: PNode): PIdent {.inline.} =
   ## Returns underlying `PIdent` for `{nkSym, nkIdent}`, or `nil`.
-  # xxx consider whether also returning the 1st ident for {nkOpenSymChoice, nkClosedSymChoice}
-  # which may simplify code.
   case a.kind
   of nkSym: a.sym.name
   of nkIdent: a.ident
+  of nkOpenSymChoice, nkClosedSymChoice: a.sons[0].sym.name
   else: nil
 
 const
@@ -1136,13 +1145,17 @@ type
     symId*: int32
     typeId*: int32
     sealed*: bool
+    disambTable*: CountTable[PIdent]
 
 const
   PackageModuleId* = -3'i32
 
 proc idGeneratorFromModule*(m: PSym): IdGenerator =
   assert m.kind == skModule
-  result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0)
+  result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0, disambTable: initCountTable[PIdent]())
+
+proc idGeneratorForPackage*(nextIdWillBe: int32): IdGenerator =
+  result = IdGenerator(module: PackageModuleId, symId: nextIdWillBe - 1'i32, typeId: 0, disambTable: initCountTable[PIdent]())
 
 proc nextSymId*(x: IdGenerator): ItemId {.inline.} =
   assert(not x.sealed)
@@ -1334,11 +1347,15 @@ when false:
       echo k
       echo v
 
-proc newSym*(symKind: TSymKind, name: PIdent, id: ItemId, owner: PSym,
+proc newSym*(symKind: TSymKind, name: PIdent, idgen: IdGenerator; owner: PSym,
              info: TLineInfo; options: TOptions = {}): PSym =
   # generates a symbol and initializes the hash field too
+  assert not name.isNil
+  let id = nextSymId idgen
   result = PSym(name: name, kind: symKind, flags: {}, info: info, itemId: id,
-                options: options, owner: owner, offset: defaultOffset)
+                options: options, owner: owner, offset: defaultOffset,
+                disamb: getOrDefault(idgen.disambTable, name).int32)
+  idgen.disambTable.inc name
   when false:
     if id.module == 48 and id.item == 39:
       writeStackTrace()
@@ -1498,7 +1515,7 @@ proc newProcNode*(kind: TNodeKind, info: TLineInfo, body: PNode,
 
 const
   AttachedOpToStr*: array[TTypeAttachedOp, string] = [
-    "=destroy", "=copy", "=sink", "=trace", "=deepcopy"]
+    "=wasMoved", "=destroy", "=copy", "=dup", "=sink", "=trace", "=deepcopy"]
 
 proc `$`*(s: PSym): string =
   if s != nil:
@@ -1552,8 +1569,8 @@ proc copyType*(t: PType, id: ItemId, owner: PSym): PType =
 proc exactReplica*(t: PType): PType =
   result = copyType(t, t.itemId, t.owner)
 
-proc copySym*(s: PSym; id: ItemId): PSym =
-  result = newSym(s.kind, s.name, id, s.owner, s.info, s.options)
+proc copySym*(s: PSym; idgen: IdGenerator): PSym =
+  result = newSym(s.kind, s.name, idgen, s.owner, s.info, s.options)
   #result.ast = nil            # BUGFIX; was: s.ast which made problems
   result.typ = s.typ
   result.flags = s.flags
@@ -1568,9 +1585,9 @@ proc copySym*(s: PSym; id: ItemId): PSym =
     result.bitsize = s.bitsize
     result.alignment = s.alignment
 
-proc createModuleAlias*(s: PSym, id: ItemId, newIdent: PIdent, info: TLineInfo;
+proc createModuleAlias*(s: PSym, idgen: IdGenerator, newIdent: PIdent, info: TLineInfo;
                         options: TOptions): PSym =
-  result = newSym(s.kind, newIdent, id, s.owner, info, options)
+  result = newSym(s.kind, newIdent, idgen, s.owner, info, options)
   # keep ID!
   result.ast = s.ast
   #result.id = s.id # XXX figure out what to do with the ID.
@@ -1690,6 +1707,8 @@ proc copyNode*(src: PNode): PNode =
   of nkIdent: result.ident = src.ident
   of nkStrLit..nkTripleStrLit: result.strVal = src.strVal
   else: discard
+  when defined(nimsuggest):
+    result.endInfo = src.endInfo
 
 template transitionNodeKindCommon(k: TNodeKind) =
   let obj {.inject.} = n[]
@@ -1742,6 +1761,8 @@ template copyNodeImpl(dst, src, processSonsStmt) =
   if src == nil: return
   dst = newNode(src.kind)
   dst.info = src.info
+  when defined(nimsuggest):
+    result.endInfo = src.endInfo
   dst.typ = src.typ
   dst.flags = src.flags * PersistentNodeFlags
   dst.comment = src.comment
@@ -1906,20 +1927,6 @@ proc isCompileTimeProc*(s: PSym): bool {.inline.} =
   result = s.kind == skMacro or
            s.kind in {skProc, skFunc} and sfCompileTime in s.flags
 
-proc isRunnableExamples*(n: PNode): bool =
-  # Templates and generics don't perform symbol lookups.
-  result = n.kind == nkSym and n.sym.magic == mRunnableExamples or
-    n.kind == nkIdent and n.ident.s == "runnableExamples"
-
-proc requiredParams*(s: PSym): int =
-  # Returns the number of required params (without default values)
-  # XXX: Perhaps we can store this in the `offset` field of the
-  # symbol instead?
-  for i in 1..<s.typ.len:
-    if s.typ.n[i].sym.ast != nil:
-      return i - 1
-  return s.typ.len - 1
-
 proc hasPattern*(s: PSym): bool {.inline.} =
   result = isRoutine(s) and s.ast[patternPos].kind != nkEmpty
 
@@ -1993,7 +2000,7 @@ proc toObjectFromRefPtrGeneric*(typ: PType): PType =
     of tyRef, tyPtr, tyGenericInst, tyGenericInvocation, tyAlias: result = result[0]
       # automatic dereferencing is deep, refs #18298.
     else: break
-  assert result.sym != nil
+  # result does not have to be object type
 
 proc isImportedException*(t: PType; conf: ConfigRef): bool =
   assert t != nil

compiler/astalgo.nim

@@ -20,9 +20,6 @@ import strutils except addf
 when defined(nimPreviewSlimSystem):
   import std/assertions
 
-when not defined(nimHasCursor):
-  {.pragma: cursor.}
-
 proc hashNode*(p: RootRef): Hash
 proc treeToYaml*(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope
   # Convert a tree into its YAML representation; this is used by the
@@ -333,7 +330,7 @@ proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int,
     sonsRope = rope("null")
   elif containsOrIncl(marker, n.id):
     sonsRope = "\"$1 @$2\"" % [rope($n.kind), rope(
-        strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))]
+        strutils.toHex(cast[int](n), sizeof(n) * 2))]
   else:
     if n.len > 0:
       sonsRope = rope("[")

compiler/backendpragmas.nim

@@ -0,0 +1,25 @@
+import pragmas, options, ast, trees
+
+proc pushBackendOption(optionsStack: var seq[TOptions], options: var TOptions) =
+  optionsStack.add options
+
+proc popBackendOption(optionsStack: var seq[TOptions], options: var TOptions) =
+  options = optionsStack[^1]
+  optionsStack.setLen(optionsStack.len-1)
+
+proc processPushBackendOption*(optionsStack: var seq[TOptions], options: var TOptions,
+                           n: PNode, start: int) =
+  pushBackendOption(optionsStack, options)
+  for i in start..<n.len:
+    let it = n[i]
+    if it.kind in nkPragmaCallKinds and it.len == 2 and it[1].kind == nkIntLit:
+      let sw = whichPragma(it[0])
+      let opts = pragmaToOptions(sw)
+      if opts != {}:
+        if it[1].intVal != 0:
+          options.incl opts
+        else:
+          options.excl opts
+
+template processPopBackendOption*(optionsStack: var seq[TOptions], options: var TOptions) =
+  popBackendOption(optionsStack, options)

compiler/btrees.nim

@@ -68,7 +68,7 @@ proc copyHalf[Key, Val](h, result: Node[Key, Val]) =
       result.links[j] = h.links[Mhalf + j]
   else:
     for j in 0..<Mhalf:
-      when defined(gcArc) or defined(gcOrc):
+      when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc):
         result.vals[j] = move h.vals[Mhalf + j]
       else:
         shallowCopy(result.vals[j], h.vals[Mhalf + j])
@@ -91,7 +91,7 @@ proc insert[Key, Val](h: Node[Key, Val], key: Key, val: Val): Node[Key, Val] =
       if less(key, h.keys[j]): break
       inc j
     for i in countdown(h.entries, j+1):
-      when defined(gcArc) or defined(gcOrc):
+      when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc):
         h.vals[i] = move h.vals[i-1]
       else:
         shallowCopy(h.vals[i], h.vals[i-1])

compiler/ccgcalls.nim

@@ -156,7 +156,7 @@ proc reifiedOpenArray(n: PNode): bool {.inline.} =
   else:
     result = true
 
-proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope, Rope) =
+proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareForMutation = false): (Rope, Rope) =
   var a, b, c: TLoc
   initLocExpr(p, q[1], a)
   initLocExpr(p, q[2], b)
@@ -164,6 +164,8 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope,
   # but first produce the required index checks:
   if optBoundsCheck in p.options:
     genBoundsCheck(p, a, b, c)
+  if prepareForMutation:
+    linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
   let ty = skipTypes(a.t, abstractVar+{tyPtr})
   let dest = getTypeDesc(p.module, destType)
   let lengthExpr = "($1)-($2)+1" % [rdLoc(c), rdLoc(b)]
@@ -267,7 +269,7 @@ proc withTmpIfNeeded(p: BProc, a: TLoc, needsTmp: bool): TLoc =
   # Bug https://github.com/status-im/nimbus-eth2/issues/1549
   # Aliasing is preferred over stack overflows.
   # Also don't regress for non ARC-builds, too risky.
-  if needsTmp and a.lode.typ != nil and p.config.selectedGC in {gcArc, gcOrc} and
+  if needsTmp and a.lode.typ != nil and p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc} and
       getSize(p.config, a.lode.typ) < 1024:
     getTemp(p, a.lode.typ, result, needsInit=false)
     genAssignment(p, result, a, {})
@@ -290,7 +292,8 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Rope; need
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
     var n = if n.kind != nkHiddenAddr: n else: n[0]
     openArrayLoc(p, param.typ, n, result)
-  elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
+  elif ccgIntroducedPtr(p.config, param, call[0].typ[0]) and 
+    (optByRef notin param.options or not p.module.compileToCpp):
     initLocExpr(p, n, a)
     if n.kind in {nkCharLit..nkNilLit}:
       addAddrLoc(p.config, literalsNeedsTmp(p, a), result)
@@ -473,6 +476,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
           discard "resetLoc(p, d)"
         pl.add(addrLoc(p.config, d))
         genCallPattern()
+        if canRaise: raiseExit(p)
       else:
         var tmp: TLoc
         getTemp(p, typ[0], tmp, needsInit=true)

compiler/ccgexprs.nim

@@ -382,7 +382,7 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
     else:
       linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)])
   of tyArray:
-    if containsGarbageCollectedRef(dest.t) and p.config.selectedGC notin {gcArc, gcOrc, gcHooks}:
+    if containsGarbageCollectedRef(dest.t) and p.config.selectedGC notin {gcArc, gcAtomicArc, gcOrc, gcHooks}:
       genGenericAsgn(p, dest, src, flags)
     else:
       linefmt(p, cpsStmts,
@@ -401,7 +401,7 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
     else:
       linefmt(p, cpsStmts,
            # bug #4799, keep the nimCopyMem for a while
-           #"#nimCopyMem((void*)$1, (NIM_CONST void*)$2, sizeof($1[0])*$1Len_0);$n",
+           #"#nimCopyMem((void*)$1, (NIM_CONST void*)$2, sizeof($1[0])*$1Len_0);\n",
            "$1 = $2;$n",
            [rdLoc(dest), rdLoc(src)])
   of tySet:
@@ -450,9 +450,10 @@ proc genDeepCopy(p: BProc; dest, src: TLoc) =
               [addrLoc(p.config, dest), rdLoc(src),
               genTypeInfoV1(p.module, dest.t, dest.lode.info)])
   of tyOpenArray, tyVarargs:
+    let source = addrLocOrTemp(src)
     linefmt(p, cpsStmts,
-         "#genericDeepCopyOpenArray((void*)$1, (void*)$2, $1Len_0, $3);$n",
-         [addrLoc(p.config, dest), addrLocOrTemp(src),
+         "#genericDeepCopyOpenArray((void*)$1, (void*)$2, $2->Field1, $3);$n",
+         [addrLoc(p.config, dest), source,
          genTypeInfoV1(p.module, dest.t, dest.lode.info)])
   of tySet:
     if mapSetType(p.config, ty) == ctArray:
@@ -752,23 +753,8 @@ proc isCppRef(p: BProc; typ: PType): bool {.inline.} =
       skipTypes(typ, abstractInstOwned).kind in {tyVar} and
       tfVarIsPtr notin skipTypes(typ, abstractInstOwned).flags
 
-proc derefBlock(p: BProc, e: PNode, d: var TLoc) =
-  # We transform (block: x)[] to (block: x[])
-  let e0 = e[0]
-  var n = shallowCopy(e0)
-  n.typ = e.typ
-  for i in 0 ..< e0.len - 1:
-    n[i] = e0[i]
-  n[e0.len-1] = newTreeIT(nkHiddenDeref, e.info, e.typ, e0[e0.len-1])
-  expr p, n, d
-
 proc genDeref(p: BProc, e: PNode, d: var TLoc) =
-  if e.kind == nkHiddenDeref and e[0].kind in {nkBlockExpr, nkBlockStmt}:
-    # bug #20107. Watch out to not deref the pointer too late.
-    derefBlock(p, e, d)
-    return
-
-  let mt = mapType(p.config, e[0].typ, mapTypeChooser(e[0]))
+  let mt = mapType(p.config, e[0].typ, mapTypeChooser(e[0]) == skParam)
   if mt in {ctArray, ctPtrToArray} and lfEnforceDeref notin d.flags:
     # XXX the amount of hacks for C's arrays is incredible, maybe we should
     # simply wrap them in a struct? --> Losing auto vectorization then?
@@ -836,7 +822,7 @@ proc genAddr(p: BProc, e: PNode, d: var TLoc) =
     initLocExpr(p, e[0], a)
     putIntoDest(p, d, e, "&" & a.r, a.storage)
     #Message(e.info, warnUser, "HERE NEW &")
-  elif mapType(p.config, e[0].typ, mapTypeChooser(e[0])) == ctArray or isCppRef(p, e.typ):
+  elif mapType(p.config, e[0].typ, mapTypeChooser(e[0]) == skParam) == ctArray or isCppRef(p, e.typ):
     expr(p, e[0], d)
   else:
     var a: TLoc
@@ -945,10 +931,17 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) =
     var discIndex = newRopeAppender()
     rdSetElemLoc(p.config, v, u.t, discIndex)
     if optTinyRtti in p.config.globalOptions:
-      # not sure how to use `genEnumToStr` here
-      if p.config.getStdlibVersion < (1, 5, 1):
-        const code = "{ #raiseFieldError($1); "
-        linefmt(p, cpsStmts, code, [strLit])
+      let base = disc.typ.skipTypes(abstractInst+{tyRange})
+      case base.kind
+      of tyEnum:
+        const code = "{ #raiseFieldErrorStr($1, $2); "
+        let toStrProc = getToStringProc(p.module.g.graph, base)
+        # XXX need to modify this logic for IC.
+        # need to analyze nkFieldCheckedExpr and marks procs "used" like range checks in dce
+        var toStr: TLoc
+        expr(p, newSymNode(toStrProc), toStr)
+        let enumStr = "$1($2)" % [rdLoc(toStr), rdLoc(v)]
+        linefmt(p, cpsStmts, code, [strLit, enumStr])
       else:
         const code = "{ #raiseFieldError2($1, (NI)$2); "
         linefmt(p, cpsStmts, code, [strLit, discIndex])
@@ -959,12 +952,8 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) =
       var firstLit = newRopeAppender()
       int64Literal(cast[int](first), firstLit)
       let discName = genTypeInfo(p.config, p.module, disc.sym.typ, e.info)
-      if p.config.getStdlibVersion < (1,5,1):
-        const code = "{ #raiseFieldError($1); "
-        linefmt(p, cpsStmts, code, [strLit])
-      else:
-        const code = "{ #raiseFieldError2($1, #reprDiscriminant(((NI)$2) + (NI)$3, $4)); "
-        linefmt(p, cpsStmts, code, [strLit, discIndex, firstLit, discName])
+      const code = "{ #raiseFieldError2($1, #reprDiscriminant(((NI)$2) + (NI)$3, $4)); "
+      linefmt(p, cpsStmts, code, [strLit, discIndex, firstLit, discName])
 
     raiseInstr(p, p.s(cpsStmts))
     linefmt p, cpsStmts, "}$n", []
@@ -1427,7 +1416,7 @@ proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) =
       p.module.s[cfsTypeInit3].addf("$1->finalizer = (void*)$2;$n", [ti, rdLoc(f)])
 
     if a.storage == OnHeap and usesWriteBarrier(p.config):
-      if canFormAcycle(a.t):
+      if canFormAcycle(p.module.g.graph, a.t):
         linefmt(p, cpsStmts, "if ($1) { #nimGCunrefRC1($1); $1 = NIM_NIL; }$n", [a.rdLoc])
       else:
         linefmt(p, cpsStmts, "if ($1) { #nimGCunrefNoCycle($1); $1 = NIM_NIL; }$n", [a.rdLoc])
@@ -1464,7 +1453,7 @@ proc genNewSeqAux(p: BProc, dest: TLoc, length: Rope; lenIsZero: bool) =
   var call: TLoc
   initLoc(call, locExpr, dest.lode, OnHeap)
   if dest.storage == OnHeap and usesWriteBarrier(p.config):
-    if canFormAcycle(dest.t):
+    if canFormAcycle(p.module.g.graph, dest.t):
       linefmt(p, cpsStmts, "if ($1) { #nimGCunrefRC1($1); $1 = NIM_NIL; }$n", [dest.rdLoc])
     else:
       linefmt(p, cpsStmts, "if ($1) { #nimGCunrefNoCycle($1); $1 = NIM_NIL; }$n", [dest.rdLoc])
@@ -1647,6 +1636,7 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) =
     return
   if d.k == locNone:
     getTemp(p, n.typ, d)
+  initLocExpr(p, n[1], a)
   # generate call to newSeq before adding the elements per hand:
   let L = toInt(lengthOrd(p.config, n[1].typ))
   if optSeqDestructors in p.config.globalOptions:
@@ -1658,7 +1648,6 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) =
     var lit = newRopeAppender()
     intLiteral(L, lit)
     genNewSeqAux(p, d, lit, L == 0)
-  initLocExpr(p, n[1], a)
   # bug #5007; do not produce excessive C source code:
   if L < 10:
     for i in 0..<L:
@@ -1704,7 +1693,7 @@ proc genNewFinalize(p: BProc, e: PNode) =
 
 proc genOfHelper(p: BProc; dest: PType; a: Rope; info: TLineInfo; result: var Rope) =
   if optTinyRtti in p.config.globalOptions:
-    let token = $genDisplayElem(MD5Digest(hashType(dest)))
+    let token = $genDisplayElem(MD5Digest(hashType(dest, p.config)))
     appcg(p.module, result, "#isObjDisplayCheck($#.m_type, $#, $#)", [a, getObjDepth(dest), token])
   else:
     # unfortunately 'genTypeInfoV1' sets tfObjHasKids as a side effect, so we
@@ -1895,8 +1884,17 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
         if op == mHigh: unaryExpr(p, e, d, "($1Len_0-1)")
         else: unaryExpr(p, e, d, "$1Len_0")
       else:
-        if op == mHigh: unaryExpr(p, e, d, "($1.Field1-1)")
-        else: unaryExpr(p, e, d, "$1.Field1")
+        let isDeref = a.kind in {nkHiddenDeref, nkDerefExpr}
+        if op == mHigh:
+          if isDeref:
+            unaryExpr(p, e, d, "($1->Field1-1)")
+          else:
+            unaryExpr(p, e, d, "($1.Field1-1)")
+        else:
+          if isDeref:
+            unaryExpr(p, e, d, "$1->Field1")
+          else:
+            unaryExpr(p, e, d, "$1.Field1")
   of tyCstring:
     if op == mHigh: unaryExpr(p, e, d, "($1 ? (#nimCStrLen($1)-1) : -1)")
     else: unaryExpr(p, e, d, "($1 ? #nimCStrLen($1) : 0)")
@@ -1921,10 +1919,6 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     else: putIntoDest(p, d, e, rope(lengthOrd(p.config, typ)))
   else: internalError(p.config, e.info, "genArrayLen()")
 
-proc makePtrType(baseType: PType; idgen: IdGenerator): PType =
-  result = newType(tyPtr, nextTypeId idgen, baseType.owner)
-  addSonSkipIntLit(result, baseType, idgen)
-
 proc makeAddr(n: PNode; idgen: IdGenerator): PNode =
   if n.kind == nkHiddenAddr:
     result = n
@@ -1947,7 +1941,7 @@ proc genSetLengthSeq(p: BProc, e: PNode, d: var TLoc) =
 
   initLoc(call, locCall, e, OnHeap)
   if not p.module.compileToCpp:
-    const setLenPattern = "($3) #setLengthSeqV2(&($1)->Sup, $4, $2)"
+    const setLenPattern = "($3) #setLengthSeqV2(($1)?&($1)->Sup:NIM_NIL, $4, $2)"
     call.r = ropecg(p.module, setLenPattern, [
       rdLoc(a), rdLoc(b), getTypeDesc(p.module, t),
       genTypeInfoV1(p.module, t.skipTypes(abstractInst), e.info)])
@@ -2126,7 +2120,7 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     of mCard:
       var a: TLoc
       initLocExpr(p, e[1], a)
-      putIntoDest(p, d, e, ropecg(p.module, "#cardSet($1, $2)", [rdCharLoc(a), size]))
+      putIntoDest(p, d, e, ropecg(p.module, "#cardSet($1, $2)", [addrLoc(p.config, a), size]))
     of mLtSet, mLeSet:
       getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i) # our counter
       initLocExpr(p, e[1], a)
@@ -2207,8 +2201,15 @@ proc genCast(p: BProc, e: PNode, d: var TLoc) =
     var lbl = p.labels.rope
     var tmp: TLoc
     tmp.r = "LOC$1.source" % [lbl]
-    linefmt(p, cpsLocals, "union { $1 source; $2 dest; } LOC$3;$n",
-      [getTypeDesc(p.module, e[1].typ), getTypeDesc(p.module, e.typ), lbl])
+    let destsize = getSize(p.config, destt)
+    let srcsize = getSize(p.config, srct)
+
+    if destsize > srcsize:
+      linefmt(p, cpsLocals, "union { $1 dest; $2 source; } LOC$3;$n #nimZeroMem(&LOC$3, sizeof(LOC$3));$n",
+        [getTypeDesc(p.module, e.typ), getTypeDesc(p.module, e[1].typ), lbl])
+    else:
+      linefmt(p, cpsLocals, "union { $1 source; $2 dest; } LOC$3;$n",
+        [getTypeDesc(p.module, e[1].typ), getTypeDesc(p.module, e.typ), lbl])
     tmp.k = locExpr
     tmp.lode = lodeTyp srct
     tmp.storage = OnStack
@@ -2339,7 +2340,7 @@ proc genWasMoved(p: BProc; n: PNode) =
   if p.withinBlockLeaveActions > 0 and notYetAlive(n1):
     discard
   else:
-    initLocExpr(p, n1, a)
+    initLocExpr(p, n1, a, {lfEnforceDeref})
     resetLoc(p, a)
     #linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n",
     #  [addrLoc(p.config, a), getTypeDesc(p.module, a.t)])
@@ -2356,8 +2357,19 @@ proc genMove(p: BProc; n: PNode; d: var TLoc) =
     linefmt(p, cpsStmts, "}$n$1.len = $2.len; $1.p = $2.p;$n", [rdLoc(a), rdLoc(src)])
   else:
     if d.k == locNone: getTemp(p, n.typ, d)
-    genAssignment(p, d, a, {})
-    resetLoc(p, a)
+    if p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc}:
+      genAssignment(p, d, a, {})
+      var op = getAttachedOp(p.module.g.graph, n.typ, attachedWasMoved)
+      if op == nil:
+        resetLoc(p, a)
+      else:
+        let addrExp = makeAddr(n[1], p.module.idgen)
+        let wasMovedCall = newTreeI(nkCall, n.info, newSymNode(op), addrExp)
+        genCall(p, wasMovedCall, d)
+    else:
+      let flags = if not canMove(p, n[1], d): {needToCopy} else: {}
+      genAssignment(p, d, a, flags)
+      resetLoc(p, a)
 
 proc genDestroy(p: BProc; n: PNode) =
   if optSeqDestructors in p.config.globalOptions:
@@ -2408,7 +2420,10 @@ proc genDispose(p: BProc; n: PNode) =
       lineCg(p, cpsStmts, ["#nimDestroyAndDispose($#)", rdLoc(a)])
 
 proc genSlice(p: BProc; e: PNode; d: var TLoc) =
-  let (x, y) = genOpenArraySlice(p, e, e.typ, e.typ.lastSon)
+  let (x, y) = genOpenArraySlice(p, e, e.typ, e.typ.lastSon,
+    prepareForMutation = e[1].kind == nkHiddenDeref and
+                         e[1].typ.skipTypes(abstractInst).kind == tyString and
+                         p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc})
   if d.k == locNone: getTemp(p, e.typ, d)
   linefmt(p, cpsStmts, "$1.Field0 = $2; $1.Field1 = $3;$n", [rdLoc(d), x, y])
   when false:
@@ -2515,10 +2530,10 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mNewSeqOfCap: genNewSeqOfCap(p, e, d)
   of mSizeOf:
     let t = e[1].typ.skipTypes({tyTypeDesc})
-    putIntoDest(p, d, e, "((NI)sizeof($1))" % [getTypeDesc(p.module, t, skVar)])
+    putIntoDest(p, d, e, "((NI)sizeof($1))" % [getTypeDesc(p.module, t, dkVar)])
   of mAlignOf:
     let t = e[1].typ.skipTypes({tyTypeDesc})
-    putIntoDest(p, d, e, "((NI)NIM_ALIGNOF($1))" % [getTypeDesc(p.module, t, skVar)])
+    putIntoDest(p, d, e, "((NI)NIM_ALIGNOF($1))" % [getTypeDesc(p.module, t, dkVar)])
   of mOffsetOf:
     var dotExpr: PNode
     if e[1].kind == nkDotExpr:
@@ -2528,7 +2543,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     else:
       internalError(p.config, e.info, "unknown ast")
     let t = dotExpr[0].typ.skipTypes({tyTypeDesc})
-    let tname = getTypeDesc(p.module, t, skVar)
+    let tname = getTypeDesc(p.module, t, dkVar)
     let member =
       if t.kind == tyTuple:
         "Field" & rope(dotExpr[1].sym.position)
@@ -2590,7 +2605,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       let n = semparallel.liftParallel(p.module.g.graph, p.module.idgen, p.module.module, e)
       expr(p, n, d)
   of mDeepCopy:
-    if p.config.selectedGC in {gcArc, gcOrc} and optEnableDeepCopy notin p.config.globalOptions:
+    if p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc} and optEnableDeepCopy notin p.config.globalOptions:
       localError(p.config, e.info,
         "for --gc:arc|orc 'deepcopy' support has to be enabled with --deepcopy:on")
 
@@ -2607,6 +2622,8 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mAccessTypeField: genAccessTypeField(p, e, d)
   of mSlice: genSlice(p, e, d)
   of mTrace: discard "no code to generate"
+  of mEnsureMove:
+    expr(p, e[1], d)
   else:
     when defined(debugMagics):
       echo p.prc.name.s, " ", p.prc.id, " ", p.prc.flags, " ", p.prc.ast[genericParamsPos].kind
@@ -2677,15 +2694,30 @@ proc genTupleConstr(p: BProc, n: PNode, d: var TLoc) =
   if not handleConstExpr(p, n, d):
     let t = n.typ
     discard getTypeDesc(p.module, t) # so that any fields are initialized
-    if d.k == locNone: getTemp(p, t, d)
+
+    var tmp: TLoc
+    # bug #16331
+    let doesAlias = lhsDoesAlias(d.lode, n)
+    let dest = if doesAlias: addr(tmp) else: addr(d)
+    if doesAlias:
+      getTemp(p, n.typ, tmp)
+    elif d.k == locNone:
+      getTemp(p, n.typ, d)
+
     for i in 0..<n.len:
       var it = n[i]
       if it.kind == nkExprColonExpr: it = it[1]
-      initLoc(rec, locExpr, it, d.storage)
-      rec.r = "$1.Field$2" % [rdLoc(d), rope(i)]
+      initLoc(rec, locExpr, it, dest[].storage)
+      rec.r = "$1.Field$2" % [rdLoc(dest[]), rope(i)]
       rec.flags.incl(lfEnforceDeref)
       expr(p, it, rec)
 
+    if doesAlias:
+      if d.k == locNone:
+        d = tmp
+      else:
+        genAssignment(p, d, tmp, {})
+
 proc isConstClosure(n: PNode): bool {.inline.} =
   result = n[0].kind == nkSym and isRoutine(n[0].sym) and
       n[1].kind == nkNilLit
@@ -2776,7 +2808,7 @@ proc upConv(p: BProc, n: PNode, d: var TLoc) =
     rdMType(p, a, nilCheck, r)
     if optTinyRtti in p.config.globalOptions:
       let checkFor = $getObjDepth(dest)
-      let token = $genDisplayElem(MD5Digest(hashType(dest)))
+      let token = $genDisplayElem(MD5Digest(hashType(dest, p.config)))
       if nilCheck != "":
         linefmt(p, cpsStmts, "if ($1 && !#isObjDisplayCheck($2, $3, $4)){ #raiseObjectConversionError(); ",
                 [nilCheck, r, checkFor, token])
@@ -2843,7 +2875,7 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) =
     # expression not found in the cache:
     inc(p.module.labels)
     p.module.s[cfsData].addf("static NIM_CONST $1 $2 = ",
-         [getTypeDesc(p.module, t, skConst), tmp])
+         [getTypeDesc(p.module, t, dkConst), tmp])
     genBracedInit(p, n, isConst = true, t, p.module.s[cfsData])
     p.module.s[cfsData].addf(";$n", [])
 
@@ -2870,13 +2902,13 @@ proc genConstHeader(m, q: BModule; p: BProc, sym: PSym) =
     if not genConstSetup(p, sym): return
   assert(sym.loc.r != "", $sym.name.s & $sym.itemId)
   if m.hcrOn:
-    m.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]);
+    m.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, dkVar), sym.loc.r]);
     m.initProc.procSec(cpsLocals).addf(
       "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r,
-      getTypeDesc(m, sym.loc.t, skVar), getModuleDllPath(q, sym)])
+      getTypeDesc(m, sym.loc.t, dkVar), getModuleDllPath(q, sym)])
   else:
     let headerDecl = "extern NIM_CONST $1 $2;$n" %
-        [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]
+        [getTypeDesc(m, sym.loc.t, dkVar), sym.loc.r]
     m.s[cfsData].add(headerDecl)
     if sfExportc in sym.flags and p.module.g.generatedHeader != nil:
       p.module.g.generatedHeader.s[cfsData].add(headerDecl)
@@ -2892,7 +2924,7 @@ proc genConstDefinition(q: BModule; p: BProc; sym: PSym) =
   q.s[cfsData].add data
   if q.hcrOn:
     # generate the global pointer with the real name
-    q.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(q, sym.loc.t, skVar), sym.loc.r])
+    q.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(q, sym.loc.t, dkVar), sym.loc.r])
     # register it (but ignore the boolean result of hcrRegisterGlobal)
     q.initProc.procSec(cpsLocals).addf(
       "\thcrRegisterGlobal($1, \"$2\", sizeof($3), NULL, (void**)&$2);$n",
@@ -3200,6 +3232,8 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo; result: var Rope) =
     result.add "}"
   of tyTuple:
     result.add "{"
+    if p.vccAndC and t.len == 0:
+      result.add "0"
     for i in 0..<t.len:
       if i > 0: result.add ", "
       getDefaultValue(p, t[i], info, result)
@@ -3329,6 +3363,8 @@ proc genConstObjConstr(p: BProc; n: PNode; isConst: bool; result: var Rope) =
 
 proc genConstSimpleList(p: BProc, n: PNode; isConst: bool; result: var Rope) =
   result.add "{"
+  if p.vccAndC and n.len == 0 and n.typ.kind == tyArray:
+    getDefaultValue(p, n.typ[1], n.info, result)
   for i in 0..<n.len:
     let it = n[i]
     if i > 0: result.add ",\n"
@@ -3338,6 +3374,8 @@ proc genConstSimpleList(p: BProc, n: PNode; isConst: bool; result: var Rope) =
 
 proc genConstTuple(p: BProc, n: PNode; isConst: bool; tup: PType; result: var Rope) =
   result.add "{"
+  if p.vccAndC and n.len == 0:
+    result.add "0"
   for i in 0..<n.len:
     let it = n[i]
     if i > 0: result.add ",\n"
@@ -3371,18 +3409,19 @@ proc genConstSeq(p: BProc, n: PNode, t: PType; isConst: bool; result: var Rope)
 
 proc genConstSeqV2(p: BProc, n: PNode, t: PType; isConst: bool; result: var Rope) =
   let base = t.skipTypes(abstractInst)[0]
-  var data = rope"{"
-  for i in 0..<n.len:
-    if i > 0: data.addf(",$n", [])
-    genBracedInit(p, n[i], isConst, base, data)
-  data.add("}")
+  var data = rope""
+  if n.len > 0:
+    data.add(", {")
+    for i in 0..<n.len:
+      if i > 0: data.addf(",$n", [])
+      genBracedInit(p, n[i], isConst, base, data)
+    data.add("}")
 
   let payload = getTempName(p.module)
-
   appcg(p.module, cfsStrData,
     "static $5 struct {$n" &
     "  NI cap; $1 data[$2];$n" &
-    "} $3 = {$2 | NIM_STRLIT_FLAG, $4};$n", [
+    "} $3 = {$2 | NIM_STRLIT_FLAG$4};$n", [
     getTypeDesc(p.module, base), n.len, payload, data,
     if isConst: "const" else: ""])
   result.add "{$1, ($2*)&$3}" % [rope(n.len), getSeqPayloadType(p.module, t), payload]

compiler/ccgstmts.nim

@@ -8,7 +8,6 @@
 #
 
 # included from cgen.nim
-
 const
   RangeExpandLimit = 256      # do not generate ranges
                               # over 'RangeExpandLimit' elements
@@ -36,7 +35,9 @@ proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} =
   if n.kind == nkEmpty:
     result = false
   elif n.kind in nkCallKinds and n[0] != nil and n[0].typ != nil and n[0].typ.skipTypes(abstractInst).kind == tyProc:
-    if isInvalidReturnType(conf, n[0].typ, true):
+    if n[0].kind == nkSym and sfConstructor in n[0].sym.flags:
+      result = true
+    elif isInvalidReturnType(conf, n[0].typ, true):
       # var v = f()
       # is transformed into: var v;  f(addr v)
       # where 'f' **does not** initialize the result!
@@ -289,11 +290,28 @@ proc potentialValueInit(p: BProc; v: PSym; value: PNode; result: var Rope) =
     #echo "New code produced for ", v.name.s, " ", p.config $ value.info
     genBracedInit(p, value, isConst = false, v.typ, result)
 
+proc genCppVarForCtor(p: BProc, v: PSym; vn, value: PNode; decl: var Rope) =
+  var params = newRopeAppender()
+  var argsCounter = 0
+  let typ = skipTypes(value[0].typ, abstractInst)
+  assert(typ.kind == tyProc)
+  for i in 1..<value.len:
+    assert(typ.len == typ.n.len)
+    genOtherArg(p, value, i, typ, params, argsCounter)
+  if params.len == 0:
+    decl = runtimeFormat("$#;\n", [decl])
+  else:
+    decl = runtimeFormat("$#($#);\n", [decl, params])
+
 proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
   if sfGoto in v.flags:
     # translate 'var state {.goto.} = X' into 'goto LX':
     genGotoVar(p, value)
     return
+  let imm = isAssignedImmediately(p.config, value)
+  let isCppCtorCall = p.module.compileToCpp and imm and
+    value.kind in nkCallKinds and value[0].kind == nkSym and
+    v.typ.kind != tyPtr and sfConstructor in value[0].sym.flags
   var targetProc = p
   var valueAsRope = ""
   potentialValueInit(p, v, value, valueAsRope)
@@ -305,7 +323,11 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     if sfPure in v.flags:
       # v.owner.kind != skModule:
       targetProc = p.module.preInitProc
-    assignGlobalVar(targetProc, vn, valueAsRope)
+    if isCppCtorCall and not containsHiddenPointer(v.typ):
+      callGlobalVarCppCtor(targetProc, v, vn, value)
+    else:
+      assignGlobalVar(targetProc, vn, valueAsRope)
+
     # XXX: be careful here.
     # Global variables should not be zeromem-ed within loops
     # (see bug #20).
@@ -314,7 +336,6 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     # global variables will be initialized to zero.
     if valueAsRope.len == 0:
       var loc = v.loc
-
       # When the native TLS is unavailable, a global thread-local variable needs
       # one more layer of indirection in order to access the TLS block.
       # Only do this for complex types that may need a call to `objectInit`
@@ -328,31 +349,21 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
       genVarPrototype(p.module.g.generatedHeader, vn)
     registerTraverseProc(p, v)
   else:
-    let imm = isAssignedImmediately(p.config, value)
     if imm and p.module.compileToCpp and p.splitDecls == 0 and
-        not containsHiddenPointer(v.typ):
+        not containsHiddenPointer(v.typ) and
+        nimErrorFlagAccessed notin p.flags:
       # C++ really doesn't like things like 'Foo f; f = x' as that invokes a
       # parameterless constructor followed by an assignment operator. So we
       # generate better code here: 'Foo f = x;'
       genLineDir(p, vn)
-      let decl = localVarDecl(p, vn)
+      var decl = localVarDecl(p, vn)
       var tmp: TLoc
-      if value.kind in nkCallKinds and value[0].kind == nkSym and
-           sfConstructor in value[0].sym.flags:
-        var params = newRopeAppender()
-        var argsCounter = 0
-        let typ = skipTypes(value[0].typ, abstractInst)
-        assert(typ.kind == tyProc)
-        for i in 1..<value.len:
-          assert(typ.len == typ.n.len)
-          genOtherArg(p, value, i, typ, params, argsCounter)
-        if params.len == 0:
-          lineF(p, cpsStmts, "$#;$n", [decl])
-        else:
-          lineF(p, cpsStmts, "$#($#);$n", [decl, params])
+      if isCppCtorCall:
+        genCppVarForCtor(p, v, vn, value, decl)
+        line(p, cpsStmts, decl)
       else:
         initLocExprSingleUse(p, value, tmp)
-        lineF(p, cpsStmts, "$# = $#;$n", [decl, tmp.rdLoc])
+        lineF(p, cpsStmts, "$# = $#;\n", [decl, tmp.rdLoc])
       return
     assignLocalVar(p, vn)
     initLocalVar(p, v, imm)
@@ -379,7 +390,8 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
     startBlock(targetProc)
   if value.kind != nkEmpty and valueAsRope.len == 0:
     genLineDir(targetProc, vn)
-    loadInto(targetProc, vn, value, v.loc)
+    if not isCppCtorCall:
+      loadInto(targetProc, vn, value, v.loc)
   if forHcr:
     endBlock(targetProc)
 
@@ -614,7 +626,7 @@ proc genWhileStmt(p: BProc, t: PNode) =
       if (t[0].kind != nkIntLit) or (t[0].intVal == 0):
         lineF(p, cpsStmts, "if (!$1) goto ", [rdLoc(a)])
         assignLabel(p.blocks[p.breakIdx], p.s(cpsStmts))
-        lineF(p, cpsStmts, ";$n", [])
+        appcg(p, cpsStmts, ";$n", [])
       genStmts(p, loopBody)
 
       if optProfiler in p.options:
@@ -654,7 +666,7 @@ proc genParForStmt(p: BProc, t: PNode) =
     #initLoc(forLoopVar.loc, locLocalVar, forLoopVar.typ, onStack)
     #discard mangleName(forLoopVar)
     let call = t[1]
-    assert(call.len in {4, 5})
+    assert(call.len == 4 or call.len == 5)
     initLocExpr(p, call[1], rangeA)
     initLocExpr(p, call[2], rangeB)
 
@@ -767,11 +779,7 @@ proc genRaiseStmt(p: BProc, t: PNode) =
   else:
     finallyActions(p)
     genLineDir(p, t)
-    # reraise the last exception:
-    if p.config.exc == excCpp:
-      line(p, cpsStmts, "throw;\n")
-    else:
-      linefmt(p, cpsStmts, "#reraiseException();$n", [])
+    linefmt(p, cpsStmts, "#reraiseException();$n", [])
   raiseInstr(p, p.s(cpsStmts))
 
 template genCaseGenericBranch(p: BProc, b: PNode, e: TLoc,
@@ -997,7 +1005,7 @@ proc genRestoreFrameAfterException(p: BProc) =
 proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   #[ code to generate:
 
-    std::exception_ptr error = nullptr;
+    std::exception_ptr error;
     try {
       body;
     } catch (Exception e) {
@@ -1028,7 +1036,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   inc(p.labels, 2)
   let etmp = p.labels
 
-  p.procSec(cpsInit).add(ropecg(p.module, "\tstd::exception_ptr T$1_ = nullptr;", [etmp]))
+  lineCg(p, cpsStmts, "std::exception_ptr T$1_;$n", [etmp])
 
   let fin = if t[^1].kind == nkFinally: t[^1] else: nil
   p.nestedTryStmts.add((fin, false, 0.Natural))
@@ -1062,7 +1070,6 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
       if hasIf: lineF(p, cpsStmts, "else ", [])
       startBlock(p)
       # we handled the error:
-      linefmt(p, cpsStmts, "T$1_ = nullptr;$n", [etmp])
       expr(p, t[i][0], d)
       linefmt(p, cpsStmts, "#popCurrentException();$n", [])
       endBlock(p)
@@ -1082,7 +1089,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
           let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
           if optTinyRtti in p.config.globalOptions:
             let checkFor = $getObjDepth(typeNode.typ)
-            appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(typeNode.typ)))])
+            appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(typeNode.typ, p.config)))])
           else:
             let checkFor = genTypeInfoV1(p.module, typeNode.typ, typeNode.info)
             appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
@@ -1125,7 +1132,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
 
       if t[i].len == 1:
         # general except section:
-        startBlock(p, "catch (...) {", [])
+        startBlock(p, "catch (...) {$n", [])
         genExceptBranchBody(t[i][0])
         endBlock(p)
         catchAllPresent = true
@@ -1151,7 +1158,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   # general finally block:
   if t.len > 0 and t[^1].kind == nkFinally:
     if not catchAllPresent:
-      startBlock(p, "catch (...) {", [])
+      startBlock(p, "catch (...) {$n", [])
       genRestoreFrameAfterException(p)
       linefmt(p, cpsStmts, "T$1_ = std::current_exception();$n", [etmp])
       endBlock(p)
@@ -1301,7 +1308,8 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
         let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
         if optTinyRtti in p.config.globalOptions:
           let checkFor = $getObjDepth(t[i][j].typ)
-          appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor,  $genDisplayElem(MD5Digest(hashType(t[i][j].typ)))])
+          appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)",
+            [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config)))])
         else:
           let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
           appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
@@ -1446,7 +1454,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
         let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
         if optTinyRtti in p.config.globalOptions:
           let checkFor = $getObjDepth(t[i][j].typ)
-          appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor,  $genDisplayElem(MD5Digest(hashType(t[i][j].typ)))])
+          appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor,  $genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config)))])
         else:
           let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
           appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
@@ -1538,7 +1546,7 @@ proc determineSection(n: PNode): TCFileSection =
   result = cfsProcHeaders
   if n.len >= 1 and n[0].kind in {nkStrLit..nkTripleStrLit}:
     let sec = n[0].strVal
-    if sec.startsWith("/*TYPESECTION*/"): result = cfsTypes
+    if sec.startsWith("/*TYPESECTION*/"): result = cfsForwardTypes # TODO WORKAROUND
     elif sec.startsWith("/*VARSECTION*/"): result = cfsVars
     elif sec.startsWith("/*INCLUDESECTION*/"): result = cfsHeaders
 
@@ -1555,9 +1563,14 @@ proc genEmit(p: BProc, t: PNode) =
     line(p, cpsStmts, s)
 
 proc genPragma(p: BProc, n: PNode) =
-  for it in n.sons:
+  for i in 0..<n.len:
+    let it = n[i]
     case whichPragma(it)
     of wEmit: genEmit(p, it)
+    of wPush:
+      processPushBackendOption(p.optionsStack, p.options, n, i+1)
+    of wPop:
+      processPopBackendOption(p.optionsStack, p.options)
     else: discard
 
 
@@ -1575,6 +1588,8 @@ proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
         "#FieldDiscriminantCheck((NI)(NU)($1), (NI)(NU)($2), $3, $4);$n",
         [rdLoc(a), rdLoc(tmp), discriminatorTableName(p.module, t, field),
          lit])
+  if p.config.exc == excGoto:
+    raiseExit(p)
 
 when false:
   proc genCaseObjDiscMapping(p: BProc, e: PNode, t: PType, field: PSym; d: var TLoc) =
@@ -1599,7 +1614,7 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   initLocExpr(p, e[0], a)
   getTemp(p, a.t, tmp)
   expr(p, e[1], tmp)
-  if optTinyRtti notin p.config.globalOptions and p.inUncheckedAssignSection == 0:
+  if p.inUncheckedAssignSection == 0:
     let field = dotExpr[1].sym
     genDiscriminantCheck(p, a, tmp, dotExpr[0].typ, field)
     message(p.config, e.info, warnCaseTransition)
@@ -1616,10 +1631,11 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
     let le = e[0]
     let ri = e[1]
     var a: TLoc
-    discard getTypeDesc(p.module, le.typ.skipTypes(skipPtrs), skVar)
+    discard getTypeDesc(p.module, le.typ.skipTypes(skipPtrs), dkVar)
     initLoc(a, locNone, le, OnUnknown)
     a.flags.incl(lfEnforceDeref)
     a.flags.incl(lfPrepareForMutation)
+    genLineDir(p, le) # it can be a nkBracketExpr, which may raise
     expr(p, le, a)
     a.flags.excl(lfPrepareForMutation)
     if fastAsgn: incl(a.flags, lfNoDeepCopy)

compiler/ccgutils.nim

@@ -53,7 +53,7 @@ proc hashString*(conf: ConfigRef; s: string): BiggestInt =
     a = a + (a shl 3)
     a = a xor (a shr 11)
     a = a + (a shl 15)
-    result = cast[Hash](a)
+    result = cast[Hash](uint(a))
 
 template getUniqueType*(key: PType): PType = key
 
@@ -121,8 +121,11 @@ proc mapSetType(conf: ConfigRef; typ: PType): TCTypeKind =
 proc ccgIntroducedPtr*(conf: ConfigRef; s: PSym, retType: PType): bool =
   var pt = skipTypes(s.typ, typedescInst)
   assert skResult != s.kind
-
-  if tfByRef in pt.flags: return true
+  
+  #note precedence: params override types
+  if optByRef in s.options: return true
+  elif sfByCopy in s.flags: return false 
+  elif tfByRef in pt.flags: return true
   elif tfByCopy in pt.flags: return false
   case pt.kind
   of tyObject:

compiler/cgen.nim

@@ -12,10 +12,12 @@
 import
   ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets,
   nversion, nimsets, msgs, bitsets, idents, types,
-  ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth,
+  ccgutils, os, ropes, math, wordrecg, treetab, cgmeth,
   rodutils, renderer, cgendata, aliases,
   lowerings, tables, sets, ndi, lineinfos, pathutils, transf,
-  injectdestructors, astmsgs, modulepaths
+  injectdestructors, astmsgs, modulepaths, backendpragmas
+
+import pipelineutils
 
 when defined(nimPreviewSlimSystem):
   import std/assertions
@@ -59,12 +61,12 @@ proc findPendingModule(m: BModule, s: PSym): BModule =
     var ms = getModule(s)
     result = m.g.modules[ms.position]
 
-proc initLoc(result: var TLoc, k: TLocKind, lode: PNode, s: TStorageLoc) =
+proc initLoc(result: var TLoc, k: TLocKind, lode: PNode, s: TStorageLoc, flags: TLocFlags = {}) =
   result.k = k
   result.storage = s
   result.lode = lode
   result.r = ""
-  result.flags = {}
+  result.flags = flags
 
 proc fillLoc(a: var TLoc, k: TLocKind, lode: PNode, r: Rope, s: TStorageLoc) {.inline.} =
   # fills the loc if it is not already initialized
@@ -214,20 +216,20 @@ macro ropecg(m: BModule, frmt: static[FormatStr], args: untyped): Rope =
     elif frmt[i] == '#' and frmt[i+1] == '#':
       inc(i, 2)
       strLit.add("#")
-
-    var start = i
-    while i < frmt.len:
-      if frmt[i] != '$' and frmt[i] != '#': inc(i)
-      else: break
-    if i - 1 >= start:
-      strLit.add(substr(frmt, start, i - 1))
+    else:
+      strLit.add(frmt[i])
+      inc(i)
 
   flushStrLit()
   result.add newCall(ident"rope", resVar)
 
 proc addIndent(p: BProc; result: var Rope) =
-  for i in 0..<p.blocks.len:
-    result.add "\t".rope
+  var i = result.len
+  let newLen = i + p.blocks.len
+  result.setLen newLen
+  while i < newLen:
+    result[i] = '\t'
+    inc i
 
 template appcg(m: BModule, c: var Rope, frmt: FormatStr,
            args: untyped) =
@@ -267,11 +269,21 @@ proc safeLineNm(info: TLineInfo): int =
 proc genCLineDir(r: var Rope, filename: string, line: int; conf: ConfigRef) =
   assert line >= 0
   if optLineDir in conf.options and line > 0:
-    r.addf("$N#line $2 $1$N",
+    r.addf("\n#line $2 $1\n",
+        [rope(makeSingleLineCString(filename)), rope(line)])
+
+proc genCLineDir(r: var Rope, filename: string, line: int; p: BProc; info: TLineInfo; lastFileIndex: FileIndex) =
+  assert line >= 0
+  if optLineDir in p.config.options and line > 0:
+    if lastFileIndex == info.fileIndex:
+        r.addf("\n#line $1\n", [rope(line)])
+    else:
+      r.addf("\n#line $2 $1\n",
         [rope(makeSingleLineCString(filename)), rope(line)])
 
 proc genCLineDir(r: var Rope, info: TLineInfo; conf: ConfigRef) =
-  genCLineDir(r, toFullPath(conf, info), info.safeLineNm, conf)
+  if optLineDir in conf.options:
+    genCLineDir(r, toFullPath(conf, info), info.safeLineNm, conf)
 
 proc freshLineInfo(p: BProc; info: TLineInfo): bool =
   if p.lastLineInfo.line != info.line or
@@ -280,16 +292,29 @@ proc freshLineInfo(p: BProc; info: TLineInfo): bool =
     p.lastLineInfo.fileIndex = info.fileIndex
     result = true
 
+proc genCLineDir(r: var Rope, p: BProc, info: TLineInfo; conf: ConfigRef) =
+  if optLineDir in conf.options:
+    let lastFileIndex = p.lastLineInfo.fileIndex
+    if freshLineInfo(p, info):
+      genCLineDir(r, toFullPath(conf, info), info.safeLineNm, p, info, lastFileIndex)
+
 proc genLineDir(p: BProc, t: PNode) =
   let line = t.info.safeLineNm
 
   if optEmbedOrigSrc in p.config.globalOptions:
     p.s(cpsStmts).add("//" & sourceLine(p.config, t.info) & "\L")
-  genCLineDir(p.s(cpsStmts), toFullPath(p.config, t.info), line, p.config)
+  let lastFileIndex = p.lastLineInfo.fileIndex
+  let freshLine = freshLineInfo(p, t.info)
+  if freshLine:
+    genCLineDir(p.s(cpsStmts), toFullPath(p.config, t.info), line, p, t.info, lastFileIndex)
   if ({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and
       (p.prc == nil or sfPure notin p.prc.flags) and t.info.fileIndex != InvalidFileIdx:
-    if freshLineInfo(p, t.info):
-      linefmt(p, cpsStmts, "nimln_($1, $2);$n",
+      if freshLine:
+        if lastFileIndex == t.info.fileIndex:
+          linefmt(p, cpsStmts, "nimln_($1);\n",
+              [line])
+        else:
+          linefmt(p, cpsStmts, "nimlf_($1, $2);\n",
               [line, quotedFilename(p.config, t.info)])
 
 proc accessThreadLocalVar(p: BProc, s: PSym)
@@ -349,19 +374,19 @@ template mapTypeChooser(n: PNode): TSymKind =
 template mapTypeChooser(a: TLoc): TSymKind = mapTypeChooser(a.lode)
 
 proc addAddrLoc(conf: ConfigRef; a: TLoc; result: var Rope) =
-  if lfIndirect notin a.flags and mapType(conf, a.t, mapTypeChooser(a)) != ctArray:
+  if lfIndirect notin a.flags and mapType(conf, a.t, mapTypeChooser(a) == skParam) != ctArray:
     result.add "(&" & a.r & ")"
   else:
     result.add a.r
 
 proc addrLoc(conf: ConfigRef; a: TLoc): Rope =
-  if lfIndirect notin a.flags and mapType(conf, a.t, mapTypeChooser(a)) != ctArray:
+  if lfIndirect notin a.flags and mapType(conf, a.t, mapTypeChooser(a) == skParam) != ctArray:
     result = "(&" & a.r & ")"
   else:
     result = a.r
 
 proc byRefLoc(p: BProc; a: TLoc): Rope =
-  if lfIndirect notin a.flags and mapType(p.config, a.t, mapTypeChooser(a)) != ctArray and not
+  if lfIndirect notin a.flags and mapType(p.config, a.t, mapTypeChooser(a) == skParam) != ctArray and not
       p.module.compileToCpp:
     result = "(&" & a.r & ")"
   else:
@@ -413,7 +438,7 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: var TLoc,
         rawConstExpr(p, newNodeIT(nkType, a.lode.info, objType), tmp)
         linefmt(p, cpsStmts,
             "#nimCopyMem((void*)$1, (NIM_CONST void*)&$2, sizeof($3));$n",
-            [rdLoc(a), rdLoc(tmp), getTypeDesc(p.module, objType, mapTypeChooser(a))])
+            [rdLoc(a), rdLoc(tmp), getTypeDesc(p.module, objType, descKindFromSymKind mapTypeChooser(a))])
       else:
         rawConstExpr(p, newNodeIT(nkType, a.lode.info, t), tmp)
         genAssignment(p, a, tmp, {})
@@ -475,7 +500,7 @@ proc resetLoc(p: BProc, loc: var TLoc) =
       # so we use getTypeDesc here rather than rdLoc(loc)
       linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n",
               [addrLoc(p.config, loc),
-              getTypeDesc(p.module, loc.t, mapTypeChooser(loc))])
+              getTypeDesc(p.module, loc.t, descKindFromSymKind mapTypeChooser(loc))])
       # XXX: We can be extra clever here and call memset only
       # on the bytes following the m_type field?
       genObjectInit(p, cpsStmts, loc.t, loc, constructObj)
@@ -492,14 +517,14 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) =
       genRefAssign(p, loc, nilLoc)
     else:
       linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc),
-        getTypeDesc(p.module, typ, mapTypeChooser(loc))])
+        getTypeDesc(p.module, typ, descKindFromSymKind mapTypeChooser(loc))])
   else:
     if not isTemp or containsGarbageCollectedRef(loc.t):
       # don't use nimZeroMem for temporary values for performance if we can
       # avoid it:
-      if not isImportedCppType(typ):
+      if not isOrHasImportedCppType(typ):
         linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n",
-                [addrLoc(p.config, loc), getTypeDesc(p.module, typ, mapTypeChooser(loc))])
+                [addrLoc(p.config, loc), getTypeDesc(p.module, typ, descKindFromSymKind mapTypeChooser(loc))])
     genObjectInit(p, cpsStmts, loc.t, loc, constructObj)
 
 proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) =
@@ -517,7 +542,10 @@ proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) =
 proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) =
   inc(p.labels)
   result.r = "T" & rope(p.labels) & "_"
-  linefmt(p, cpsLocals, "$1 $2;$n", [getTypeDesc(p.module, t, skVar), result.r])
+  if p.module.compileToCpp and isOrHasImportedCppType(t):
+    linefmt(p, cpsLocals, "$1 $2{};$n", [getTypeDesc(p.module, t, dkVar), result.r])
+  else:
+    linefmt(p, cpsLocals, "$1 $2;$n", [getTypeDesc(p.module, t, dkVar), result.r])
   result.k = locTemp
   result.lode = lodeTyp t
   result.storage = OnStack
@@ -535,7 +563,7 @@ proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) =
 proc getTempCpp(p: BProc, t: PType, result: var TLoc; value: Rope) =
   inc(p.labels)
   result.r = "T" & rope(p.labels) & "_"
-  linefmt(p, cpsStmts, "$1 $2 = $3;$n", [getTypeDesc(p.module, t, skVar), result.r, value])
+  linefmt(p, cpsStmts, "$1 $2 = $3;$n", [getTypeDesc(p.module, t, dkVar), result.r, value])
   result.k = locTemp
   result.lode = lodeTyp t
   result.storage = OnStack
@@ -558,8 +586,11 @@ proc localVarDecl(p: BProc; n: PNode): Rope =
     if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy)
   if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0:
     result.addf("NIM_ALIGN($1) ", [rope(s.alignment)])
-  result.add getTypeDesc(p.module, s.typ, skVar)
-  if s.constraint.isNil:
+
+  genCLineDir(result, p, n.info, p.config)
+
+  result.add getTypeDesc(p.module, s.typ, dkVar)
+  if sfCodegenDecl notin s.flags:
     if sfRegister in s.flags: result.add(" register")
     #elif skipTypes(s.typ, abstractInst).kind in GcTypeKinds:
     #  decl.add(" GC_GUARD")
@@ -574,8 +605,8 @@ proc assignLocalVar(p: BProc, n: PNode) =
   #assert(s.loc.k == locNone) # not yet assigned
   # this need not be fulfilled for inline procs; they are regenerated
   # for each module that uses them!
-  let nl = if optLineDir in p.config.options: "" else: "\L"
-  let decl = localVarDecl(p, n) & ";" & nl
+  let nl = if optLineDir in p.config.options: "" else: "\n"
+  let decl = localVarDecl(p, n) & (if p.module.compileToCpp and isOrHasImportedCppType(n.typ): "{};" else: ";") & nl
   line(p, cpsLocals, decl)
 
 include ccgthreadvars
@@ -588,6 +619,40 @@ proc treatGlobalDifferentlyForHCR(m: BModule, s: PSym): bool =
       # and s.owner.kind == skModule # owner isn't always a module (global pragma on local var)
       # and s.loc.k == locGlobalVar  # loc isn't always initialized when this proc is used
 
+proc genGlobalVarDecl(p: BProc, n: PNode; td, value: Rope; decl: var Rope) =
+  let s = n.sym
+  if sfCodegenDecl notin s.flags:
+    if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0:
+      decl.addf "NIM_ALIGN($1) ", [rope(s.alignment)]
+    if p.hcrOn: decl.add("static ")
+    elif sfImportc in s.flags: decl.add("extern ")
+    elif lfExportLib in s.loc.flags: decl.add("N_LIB_EXPORT_VAR ")
+    else: decl.add("N_LIB_PRIVATE ")
+    if s.kind == skLet and value != "": decl.add("NIM_CONST ")
+    decl.add(td)
+    if p.hcrOn: decl.add("*")
+    if sfRegister in s.flags: decl.add(" register")
+    if sfVolatile in s.flags: decl.add(" volatile")
+    if sfNoalias in s.flags: decl.add(" NIM_NOALIAS")
+  else:
+    if value != "":
+      decl = runtimeFormat(s.cgDeclFrmt & " = $#;$n", [td, s.loc.r, value])
+    else:
+      decl = runtimeFormat(s.cgDeclFrmt & ";$n", [td, s.loc.r])
+
+proc genCppVarForCtor(p: BProc, v: PSym; vn, value: PNode; decl: var Rope)
+
+proc callGlobalVarCppCtor(p: BProc; v: PSym; vn, value: PNode) =
+  let s = vn.sym
+  fillBackendName(p.module, s)
+  fillLoc(s.loc, locGlobalVar, vn, OnHeap)
+  var decl: Rope
+  let td = getTypeDesc(p.module, vn.sym.typ, dkVar)
+  genGlobalVarDecl(p, vn, td, "", decl)
+  decl.add " " & $s.loc.r
+  genCppVarForCtor(p, v, vn, value, decl)
+  p.module.s[cfsVars].add decl
+
 proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
   let s = n.sym
   if s.loc.k == locNone:
@@ -613,29 +678,30 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
         internalError(p.config, n.info, ".threadvar variables cannot have a value")
     else:
       var decl: Rope = ""
-      var td = getTypeDesc(p.module, s.loc.t, skVar)
+      let td = getTypeDesc(p.module, s.loc.t, dkVar)
+      genGlobalVarDecl(p, n, td, value, decl)
       if s.constraint.isNil:
-        if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0:
-          decl.addf "NIM_ALIGN($1) ", [rope(s.alignment)]
-        if p.hcrOn: decl.add("static ")
-        elif sfImportc in s.flags: decl.add("extern ")
-        elif lfExportLib in s.loc.flags: decl.add("N_LIB_EXPORT_VAR ")
-        else: decl.add("N_LIB_PRIVATE ")
-        if s.kind == skLet and value != "": decl.add("NIM_CONST ")
-        decl.add(td)
-        if p.hcrOn: decl.add("*")
-        if sfRegister in s.flags: decl.add(" register")
-        if sfVolatile in s.flags: decl.add(" volatile")
-        if sfNoalias in s.flags: decl.add(" NIM_NOALIAS")
         if value != "":
-          decl.addf(" $1 = $2;$n", [s.loc.r, value])
+          if p.module.compileToCpp and value.startsWith "{{}":
+            # TODO: taking this branch, re"\{\{\}(,\s\{\})*\}" might be emitted, resulting in
+            # either warnings (GCC 12.2+) or errors (Clang 15, MSVC 19.3+) of C++11+ compilers **when
+            # explicit constructors are around** due to overload resolution rules in place [^0][^1][^2]
+            # *Workaround* here: have C++'s static initialization mechanism do the default init work,
+            # for us lacking a deeper knowledge of an imported object's constructors' ex-/implicitness
+            # (so far) *and yet* trying to achieve default initialization.
+            # Still, generating {}s in genConstObjConstr() just to omit them here is faaaar from ideal;
+            # need to figure out a better way, possibly by keeping around more data about the
+            # imported objects' contructors?
+            #
+            # [^0]: https://en.cppreference.com/w/cpp/language/aggregate_initialization
+            # [^1]: https://cplusplus.github.io/CWG/issues/1518.html
+            # [^2]: https://eel.is/c++draft/over.match.ctor
+            decl.addf(" $1;$n", [s.loc.r])
+          else:
+            decl.addf(" $1 = $2;$n", [s.loc.r, value])
         else:
           decl.addf(" $1;$n", [s.loc.r])
-      else:
-        if value != "":
-          decl = runtimeFormat(s.cgDeclFrmt & " = $#;$n", [td, s.loc.r, value])
-        else:
-          decl = runtimeFormat(s.cgDeclFrmt & ";$n", [td, s.loc.r])
+
       p.module.s[cfsVars].add(decl)
   if p.withinLoop > 0 and value == "":
     # fixes tests/run/tzeroarray:
@@ -656,7 +722,7 @@ proc getLabel(p: BProc): TLabel =
   result = "LA" & rope(p.labels) & "_"
 
 proc fixLabel(p: BProc, labl: TLabel) =
-  lineF(p, cpsStmts, "$1: ;$n", [labl])
+  p.s(cpsStmts).add("$1: ;$n" % [labl])
 
 proc genVarPrototype(m: BModule, n: PNode)
 proc requestConstImpl(p: BProc, sym: PSym)
@@ -669,8 +735,8 @@ proc genLiteral(p: BProc, n: PNode; result: var Rope)
 proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType; result: var Rope; argsCounter: var int)
 proc raiseExit(p: BProc)
 
-proc initLocExpr(p: BProc, e: PNode, result: var TLoc) =
-  initLoc(result, locNone, e, OnUnknown)
+proc initLocExpr(p: BProc, e: PNode, result: var TLoc, flags: TLocFlags = {}) =
+  initLoc(result, locNone, e, OnUnknown, flags)
   expr(p, e, result)
 
 proc initLocExprSingleUse(p: BProc, e: PNode, result: var TLoc) =
@@ -698,8 +764,12 @@ $1define nimfr_(proc, file) \
     struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename;NI len;VarSlot s[slots];} FR_; \
     FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = length; #nimFrame((TFrame*)&FR_);
 
-  $1define nimln_(n, file) \
+  $1define nimln_(n) \
+    FR_.line = n;
+
+  $1define nimlf_(n, file) \
     FR_.line = n; FR_.filename = file;
+
 """
   if p.module.s[cfsFrameDefines].len == 0:
     appcg(p.module, p.module.s[cfsFrameDefines], frameDefines, ["#"])
@@ -765,7 +835,7 @@ proc loadDynamicLib(m: BModule, lib: PLib) =
       initLoc(dest, locTemp, lib.path, OnStack)
       dest.r = getTempName(m)
       appcg(m, m.s[cfsDynLibInit],"$1 $2;$n",
-           [getTypeDesc(m, lib.path.typ, skVar), rdLoc(dest)])
+           [getTypeDesc(m, lib.path.typ, dkVar), rdLoc(dest)])
       expr(p, lib.path, dest)
 
       m.s[cfsVars].add(p.s(cpsLocals))
@@ -805,7 +875,7 @@ proc symInDynamicLib(m: BModule, sym: PSym) =
       params.add(rdLoc(a))
       params.add(", ")
     let load = "\t$1 = ($2) ($3$4));$n" %
-        [tmp, getTypeDesc(m, sym.typ, skVar), params, makeCString($extname)]
+        [tmp, getTypeDesc(m, sym.typ, dkVar), params, makeCString($extname)]
     var last = lastSon(n)
     if last.kind == nkHiddenStdConv: last = last[1]
     internalAssert(m.config, last.kind == nkStrLit)
@@ -819,8 +889,8 @@ proc symInDynamicLib(m: BModule, sym: PSym) =
   else:
     appcg(m, m.s[cfsDynLibInit],
         "\t$1 = ($2) #nimGetProcAddr($3, $4);$n",
-        [tmp, getTypeDesc(m, sym.typ, skVar), lib.name, makeCString($extname)])
-  m.s[cfsVars].addf("$2 $1;$n", [sym.loc.r, getTypeDesc(m, sym.loc.t, skVar)])
+        [tmp, getTypeDesc(m, sym.typ, dkVar), lib.name, makeCString($extname)])
+  m.s[cfsVars].addf("$2 $1;$n", [sym.loc.r, getTypeDesc(m, sym.loc.t, dkVar)])
 
 proc varInDynamicLib(m: BModule, sym: PSym) =
   var lib = sym.annex
@@ -832,9 +902,9 @@ proc varInDynamicLib(m: BModule, sym: PSym) =
   inc(m.labels, 2)
   appcg(m, m.s[cfsDynLibInit],
       "$1 = ($2*) #nimGetProcAddr($3, $4);$n",
-      [tmp, getTypeDesc(m, sym.typ, skVar), lib.name, makeCString($extname)])
+      [tmp, getTypeDesc(m, sym.typ, dkVar), lib.name, makeCString($extname)])
   m.s[cfsVars].addf("$2* $1;$n",
-      [sym.loc.r, getTypeDesc(m, sym.loc.t, skVar)])
+      [sym.loc.r, getTypeDesc(m, sym.loc.t, dkVar)])
 
 proc symInDynamicLibPartial(m: BModule, sym: PSym) =
   sym.loc.r = mangleDynLibProc(sym)
@@ -865,7 +935,9 @@ proc cgsymValue(m: BModule, name: string): Rope =
     result.addActualSuffixForHCR(m.module, sym)
 
 proc generateHeaders(m: BModule) =
-  m.s[cfsHeaders].add("\L#include \"nimbase.h\"\L")
+  var nimbase = m.config.nimbasePattern
+  if nimbase == "": nimbase = "nimbase.h"
+  m.s[cfsHeaders].addf("\L#include \"$1\"\L", [nimbase])
 
   for it in m.headerFiles:
     if it[0] == '#':
@@ -1062,7 +1134,7 @@ proc getProcTypeCast(m: BModule, prc: PSym): Rope =
 
 proc genProcBody(p: BProc; procBody: PNode) =
   genStmts(p, procBody) # modifies p.locals, p.init, etc.
-  if {nimErrorFlagAccessed, nimErrorFlagDeclared} * p.flags == {nimErrorFlagAccessed}:
+  if {nimErrorFlagAccessed, nimErrorFlagDeclared, nimErrorFlagDisabled} * p.flags == {nimErrorFlagAccessed}:
     p.flags.incl nimErrorFlagDeclared
     p.blocks[0].sections[cpsLocals].add(ropecg(p.module, "NIM_BOOL* nimErr_;$n", []))
     p.blocks[0].sections[cpsInit].add(ropecg(p.module, "nimErr_ = #nimErrorFlag();$n", []))
@@ -1070,10 +1142,13 @@ proc genProcBody(p: BProc; procBody: PNode) =
 proc isNoReturn(m: BModule; s: PSym): bool {.inline.} =
   sfNoReturn in s.flags and m.config.exc != excGoto
 
-proc genProcAux(m: BModule, prc: PSym) =
+proc genProcAux*(m: BModule, prc: PSym) =
   var p = newProc(prc, m)
   var header = newRopeAppender()
-  genProcHeader(m, prc, header)
+  if m.config.backend == backendCpp and {sfVirtual, sfConstructor} * prc.flags != {}:
+    genMemberProcHeader(m, prc, header)
+  else:
+    genProcHeader(m, prc, header)
   var returnStmt: Rope = ""
   assert(prc.ast != nil)
 
@@ -1081,12 +1156,15 @@ proc genProcAux(m: BModule, prc: PSym) =
   if sfInjectDestructors in prc.flags:
     procBody = injectDestructorCalls(m.g.graph, m.idgen, prc, procBody)
 
+  let tmpInfo = prc.info
+  discard freshLineInfo(p, prc.info)
+
   if sfPure notin prc.flags and prc.typ[0] != nil:
     if resultPos >= prc.ast.len:
       internalError(m.config, prc.info, "proc has no result symbol")
     let resNode = prc.ast[resultPos]
     let res = resNode.sym # get result symbol
-    if not isInvalidReturnType(m.config, prc.typ):
+    if not isInvalidReturnType(m.config, prc.typ) and sfConstructor notin prc.flags:
       if sfNoInit in prc.flags: incl(res.flags, sfNoInit)
       if sfNoInit in prc.flags and p.module.compileToCpp and (let val = easyResultAsgn(procBody); val != nil):
         var decl = localVarDecl(p, resNode)
@@ -1099,6 +1177,8 @@ proc genProcAux(m: BModule, prc: PSym) =
         assert(res.loc.r != "")
         initLocalVar(p, res, immediateAsgn=false)
       returnStmt = ropecg(p.module, "\treturn $1;$n", [rdLoc(res.loc)])
+    elif sfConstructor in prc.flags:
+      fillLoc(resNode.sym.loc, locParam, resNode, "this", OnHeap)
     else:
       fillResult(p.config, resNode, prc.typ)
       assignParam(p, res, prc.typ[0])
@@ -1123,6 +1203,8 @@ proc genProcAux(m: BModule, prc: PSym) =
   closureSetup(p, prc)
   genProcBody(p, procBody)
 
+  prc.info = tmpInfo
+
   var generatedProc: Rope
   generatedProc.genCLineDir prc.info, m.config
   if isNoReturn(p.module, prc):
@@ -1174,7 +1256,7 @@ proc requiresExternC(m: BModule; sym: PSym): bool {.inline.} =
 
 proc genProcPrototype(m: BModule, sym: PSym) =
   useHeader(m, sym)
-  if lfNoDecl in sym.loc.flags: return
+  if lfNoDecl in sym.loc.flags or {sfVirtual, sfConstructor} * sym.flags != {}: return
   if lfDynamicLib in sym.loc.flags:
     if sym.itemId.module != m.module.position and
         not containsOrIncl(m.declaredThings, sym.id):
@@ -1210,6 +1292,20 @@ proc genProcNoForward(m: BModule, prc: PSym) =
   if lfNoDecl in prc.loc.flags:
     fillProcLoc(m, prc.ast[namePos])
     genProcPrototype(m, prc)
+  elif lfDynamicLib in prc.loc.flags:
+    var q = findPendingModule(m, prc)
+    fillProcLoc(q, prc.ast[namePos])
+    genProcPrototype(m, prc)
+    if q != nil and not containsOrIncl(q.declaredThings, prc.id):
+      symInDynamicLib(q, prc)
+      # register the procedure even though it is in a different dynamic library and will not be
+      # reloadable (and has no _actual suffix) - other modules will need to be able to get it through
+      # the hcr dynlib (also put it in the DynLibInit section - right after it gets loaded)
+      if isReloadable(q, prc):
+        q.s[cfsDynLibInit].addf("\t$1 = ($2) hcrRegisterProc($3, \"$1\", (void*)$1);$n",
+            [prc.loc.r, getTypeDesc(q, prc.loc.t), getModuleDllPath(m, q.module)])
+    else:
+      symInDynamicLibPartial(m, prc)
   elif prc.typ.callConv == ccInline:
     # We add inline procs to the calling module to enable C based inlining.
     # This also means that a check with ``q.declaredThings`` is wrong, we need
@@ -1228,20 +1324,6 @@ proc genProcNoForward(m: BModule, prc: PSym) =
       #  prc.loc.r = mangleName(m, prc)
       genProcPrototype(m, prc)
       genProcAux(m, prc)
-  elif lfDynamicLib in prc.loc.flags:
-    var q = findPendingModule(m, prc)
-    fillProcLoc(q, prc.ast[namePos])
-    genProcPrototype(m, prc)
-    if q != nil and not containsOrIncl(q.declaredThings, prc.id):
-      symInDynamicLib(q, prc)
-      # register the procedure even though it is in a different dynamic library and will not be
-      # reloadable (and has no _actual suffix) - other modules will need to be able to get it through
-      # the hcr dynlib (also put it in the DynLibInit section - right after it gets loaded)
-      if isReloadable(q, prc):
-        q.s[cfsDynLibInit].addf("\t$1 = ($2) hcrRegisterProc($3, \"$1\", (void*)$1);$n",
-            [prc.loc.r, getTypeDesc(q, prc.loc.t), getModuleDllPath(m, q.module)])
-    else:
-      symInDynamicLibPartial(m, prc)
   elif sfImportc notin prc.flags:
     var q = findPendingModule(m, prc)
     fillProcLoc(q, prc.ast[namePos])
@@ -1305,14 +1387,14 @@ proc genVarPrototype(m: BModule, n: PNode) =
   if sym.owner.id != m.module.id:
     # else we already have the symbol generated!
     assert(sym.loc.r != "")
+    incl(m.declaredThings, sym.id)
     if sfThread in sym.flags:
       declareThreadVar(m, sym, true)
     else:
-      incl(m.declaredThings, sym.id)
       if sym.kind in {skLet, skVar, skField, skForVar} and sym.alignment > 0:
         m.s[cfsVars].addf "NIM_ALIGN($1) ", [rope(sym.alignment)]
       m.s[cfsVars].add(if m.hcrOn: "static " else: "extern ")
-      m.s[cfsVars].add(getTypeDesc(m, sym.loc.t, skVar))
+      m.s[cfsVars].add(getTypeDesc(m, sym.loc.t, dkVar))
       if m.hcrOn: m.s[cfsVars].add("*")
       if lfDynamicLib in sym.loc.flags: m.s[cfsVars].add("*")
       if sfRegister in sym.flags: m.s[cfsVars].add(" register")
@@ -1321,7 +1403,7 @@ proc genVarPrototype(m: BModule, n: PNode) =
       m.s[cfsVars].addf(" $1;$n", [sym.loc.r])
       if m.hcrOn: m.initProc.procSec(cpsLocals).addf(
         "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r,
-        getTypeDesc(m, sym.loc.t, skVar), getModuleDllPath(m, sym)])
+        getTypeDesc(m, sym.loc.t, dkVar), getModuleDllPath(m, sym)])
 
 proc addNimDefines(result: var Rope; conf: ConfigRef) {.inline.} =
   result.addf("#define NIM_INTBITS $1\L", [
@@ -1397,20 +1479,23 @@ proc genMainProc(m: BModule) =
                        [handle, strLit])
 
     preMainCode.add(loadLib("hcr_handle", "hcrGetProc"))
-    preMainCode.add("\tvoid* rtl_handle;\L")
-    preMainCode.add(loadLib("rtl_handle", "nimGC_setStackBottom"))
-    preMainCode.add(hcrGetProcLoadCode(m, "nimGC_setStackBottom", "nimrtl_", "rtl_handle", "nimGetProcAddr"))
-    preMainCode.add("\tinner = PreMain;\L")
-    preMainCode.add("\tinitStackBottomWith_actual((void *)&inner);\L")
-    preMainCode.add("\t(*inner)();\L")
+    if m.config.selectedGC in {gcArc, gcAtomicArc, gcOrc}:
+      preMainCode.add("\t$1PreMain();\L" % [rope m.config.nimMainPrefix])
+    else:
+      preMainCode.add("\tvoid* rtl_handle;\L")
+      preMainCode.add(loadLib("rtl_handle", "nimGC_setStackBottom"))
+      preMainCode.add(hcrGetProcLoadCode(m, "nimGC_setStackBottom", "nimrtl_", "rtl_handle", "nimGetProcAddr"))
+      preMainCode.add("\tinner = $1PreMain;\L" % [rope m.config.nimMainPrefix])
+      preMainCode.add("\tinitStackBottomWith_actual((void *)&inner);\L")
+      preMainCode.add("\t(*inner)();\L")
   else:
     preMainCode.add("\t$1PreMain();\L" % [rope m.config.nimMainPrefix])
 
   var posixCmdLine: Rope
   if optNoMain notin m.config.globalOptions:
-    posixCmdLine.add "\tN_LIB_PRIVATE int cmdCount;\L"
-    posixCmdLine.add "\tN_LIB_PRIVATE char** cmdLine;\L"
-    posixCmdLine.add "\tN_LIB_PRIVATE char** gEnv;\L"
+    posixCmdLine.add "N_LIB_PRIVATE int cmdCount;\L"
+    posixCmdLine.add "N_LIB_PRIVATE char** cmdLine;\L"
+    posixCmdLine.add "N_LIB_PRIVATE char** gEnv;\L"
 
   const
     # The use of a volatile function pointer to call Pre/NimMainInner
@@ -1423,15 +1508,15 @@ proc genMainProc(m: BModule) =
       "}$N$N" &
       "$4" &
       "N_LIB_PRIVATE void $3PreMain(void) {$N" &
-      "\t##if $5$N" & # 1 for volatile call, 0 for non-volatile
+      "##if $5$N" & # 1 for volatile call, 0 for non-volatile
       "\tvoid (*volatile inner)(void);$N" &
       "\tinner = $3PreMainInner;$N" &
       "$1" &
       "\t(*inner)();$N" &
-      "\t##else$N" &
+      "##else$N" &
       "$1" &
       "\t$3PreMainInner();$N" &
-      "\t##endif$N" &
+      "##endif$N" &
       "}$N$N"
 
     MainProcs =
@@ -1446,17 +1531,17 @@ proc genMainProc(m: BModule) =
 
     NimMainProc =
       "N_CDECL(void, $5NimMain)(void) {$N" &
-      "\t##if $6$N" & # 1 for volatile call, 0 for non-volatile
+      "##if $6$N" & # 1 for volatile call, 0 for non-volatile
       "\tvoid (*volatile inner)(void);$N" &
       "$4" &
       "\tinner = $5NimMainInner;$N" &
       "$2" &
       "\t(*inner)();$N" &
-      "\t##else$N" &
+      "##else$N" &
       "$4" &
       "$2" &
       "\t$5NimMainInner();$N" &
-      "\t##endif$N" &
+      "##endif$N" &
       "}$N$N"
 
     NimMainBody = NimMainInner & NimMainProc
@@ -1487,7 +1572,7 @@ proc genMainProc(m: BModule) =
     WinCDllMain =
       "BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fwdreason, $N" &
       "                    LPVOID lpvReserved) {$N" &
-      "\tif(fwdreason == DLL_PROCESS_ATTACH) {$N" & MainProcs & "}$N" &
+      "\tif (fwdreason == DLL_PROCESS_ATTACH) {$N" & MainProcs & "\t}$N" &
       "\treturn 1;$N}$N$N"
 
     PosixNimDllMain = WinNimDllMain
@@ -1521,11 +1606,11 @@ proc genMainProc(m: BModule) =
     m.includeHeader("<libc/component.h>")
 
   let initStackBottomCall =
-    if m.config.target.targetOS == osStandalone or m.config.selectedGC in {gcNone, gcArc, gcOrc}: "".rope
+    if m.config.target.targetOS == osStandalone or m.config.selectedGC in {gcNone, gcArc, gcAtomicArc, gcOrc}: "".rope
     else: ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", [])
   inc(m.labels)
 
-  let isVolatile = if m.config.selectedGC notin {gcNone, gcArc, gcOrc}: "1" else: "0"
+  let isVolatile = if m.config.selectedGC notin {gcNone, gcArc, gcAtomicArc, gcOrc}: "1" else: "0"
   appcg(m, m.s[cfsProcs], PreMainBody, [m.g.mainDatInit, m.g.otherModsInit, m.config.nimMainPrefix, posixCmdLine, isVolatile])
 
   if m.config.target.targetOS == osWindows and
@@ -1624,7 +1709,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) =
     hcrModuleMeta.addf("\t\"\"};$n", [])
     hcrModuleMeta.addf("$nN_LIB_EXPORT N_NIMCALL(void**, HcrGetImportedModules)() { return (void**)hcr_module_list; }$n", [])
     hcrModuleMeta.addf("$nN_LIB_EXPORT N_NIMCALL(char*, HcrGetSigHash)() { return \"$1\"; }$n$n",
-                          [($sigHash(m.module)).rope])
+                          [($sigHash(m.module, m.config)).rope])
     if sfMainModule in m.module.flags:
       g.mainModProcs.add(hcrModuleMeta)
       g.mainModProcs.addf("static void* hcr_handle;$N", [])
@@ -1665,7 +1750,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) =
   if sfSystemModule in m.module.flags:
     if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
       g.mainDatInit.add(ropecg(m, "\t#initThreadVarsEmulation();$N", []))
-    if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcOrc}:
+    if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcAtomicArc, gcOrc}:
       g.mainDatInit.add(ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", []))
 
   if m.s[cfsInitProc].len > 0:
@@ -1716,10 +1801,10 @@ proc hcrGetProcLoadCode(m: BModule, sym, prefix, handle, getProcFunc: string): R
   prc.typ.sym = nil
 
   if not containsOrIncl(m.declaredThings, prc.id):
-    m.s[cfsVars].addf("static $2 $1;$n", [prc.loc.r, getTypeDesc(m, prc.loc.t, skVar)])
+    m.s[cfsVars].addf("static $2 $1;$n", [prc.loc.r, getTypeDesc(m, prc.loc.t, dkVar)])
 
   result = "\t$1 = ($2) $3($4, $5);$n" %
-      [tmp, getTypeDesc(m, prc.typ, skVar), getProcFunc.rope, handle.rope, makeCString(prefix & sym)]
+      [tmp, getTypeDesc(m, prc.typ, dkVar), getProcFunc.rope, handle.rope, makeCString(prefix & sym)]
 
 proc genInitCode(m: BModule) =
   ## this function is called in cgenWriteModules after all modules are closed,
@@ -1801,6 +1886,9 @@ proc genInitCode(m: BModule) =
 
     if optStackTrace in m.initProc.options and preventStackTrace notin m.flags:
       prc.add(deinitFrame(m.initProc))
+  elif sfMainModule in m.module.flags and m.config.exc == excGoto:
+    if getCompilerProc(m.g.graph, "nimTestErrorFlag") != nil:
+      m.appcg(prc, "\t#nimTestErrorFlag();$n", [])
 
   prc.addf("}$N", [])
 
@@ -1855,8 +1943,6 @@ proc genModule(m: BModule, cfile: Cfile): Rope =
     openNamespaceNim(m.config.cppCustomNamespace, result)
   if m.s[cfsFrameDefines].len > 0:
     result.add(m.s[cfsFrameDefines])
-  else:
-    result.add("#define nimfr_(x, y)\n#define nimln_(x, y)\n")
 
   for i in cfsForwardTypes..cfsProcs:
     if m.s[i].len > 0:
@@ -1929,10 +2015,7 @@ template injectG() {.dirty.} =
     graph.backend = newModuleList(graph)
   let g = BModuleList(graph.backend)
 
-when not defined(nimHasSinkInference):
-  {.pragma: nosinks.}
-
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
+proc setupCgen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   injectG()
   result = newModule(g, module, graph.config)
   result.idgen = idgen
@@ -2000,7 +2083,7 @@ proc addHcrInitGuards(p: BProc, n: PNode, inInitGuard: var bool) =
 
 proc genTopLevelStmt*(m: BModule; n: PNode) =
   ## Also called from `ic/cbackend.nim`.
-  if passes.skipCodegen(m.config, n): return
+  if pipelineutils.skipCodegen(m.config, n): return
   m.initProc.options = initProcOptions(m)
   #softRnl = if optLineDir in m.config.options: noRnl else: rnl
   # XXX replicate this logic!
@@ -2013,12 +2096,6 @@ proc genTopLevelStmt*(m: BModule; n: PNode) =
   else:
     genProcBody(m.initProc, transformedN)
 
-proc myProcess(b: PPassContext, n: PNode): PNode =
-  result = n
-  if b != nil:
-    var m = BModule(b)
-    genTopLevelStmt(m, n)
-
 proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
   if optForceFullMake notin m.config.globalOptions:
     if not moduleHasChanged(m.g.graph, m.module):
@@ -2084,7 +2161,7 @@ proc updateCachedModule(m: BModule) =
   cf.flags = {CfileFlag.Cached}
   addFileToCompile(m.config, cf)
 
-proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
+proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode): PNode =
   ## Also called from IC.
   if sfMainModule in m.module.flags:
     # phase ordering problem here: We need to announce this
@@ -2095,7 +2172,7 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
     if {optGenStaticLib, optGenDynLib, optNoMain} * m.config.globalOptions == {}:
       for i in countdown(high(graph.globalDestructors), 0):
         n.add graph.globalDestructors[i]
-  if passes.skipCodegen(m.config, n): return
+  if pipelineutils.skipCodegen(m.config, n): return
   if moduleHasChanged(graph, m.module):
     # if the module is cached, we don't regenerate the main proc
     # nor the dispatchers? But if the dispatchers changed?
@@ -2106,7 +2183,10 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
 
     if m.hcrOn:
       # make sure this is pulled in (meaning hcrGetGlobal() is called for it during init)
-      cgsym(m, "programResult")
+      let sym = magicsys.getCompilerProc(m.g.graph, "programResult")
+      # ignore when not available, could be a module imported early in `system`
+      if sym != nil:
+        cgsymImpl m, sym
       if m.inHcrInitGuard:
         endBlock(m.initProc)
 
@@ -2123,25 +2203,18 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
         cgsym(m, "rawWrite")
 
       # raise dependencies on behalf of genMainProc
-      if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcOrc}:
+      if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcAtomicArc, gcOrc}:
         cgsym(m, "initStackBottomWith")
       if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
         cgsym(m, "initThreadVarsEmulation")
 
       if m.g.forwardedProcs.len == 0:
         incl m.flags, objHasKidsValid
-      let disp = generateMethodDispatchers(graph)
-      for x in disp: genProcAux(m, x.sym)
+      result = generateMethodDispatchers(graph, m.idgen)
 
   let mm = m
   m.g.modulesClosed.add mm
 
-
-proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
-  result = n
-  if b == nil: return
-  finalCodegenActions(graph, BModule(b), n)
-
 proc genForwardedProcs(g: BModuleList) =
   # Forward declared proc:s lack bodies when first encountered, so they're given
   # a second pass here
@@ -2168,5 +2241,3 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) =
     m.writeModule(pending=true)
   writeMapping(config, g.mapping)
   if g.generatedHeader != nil: writeHeader(g.generatedHeader)
-
-const cgenPass* = makePass(myOpen, myProcess, myClose)

compiler/cgendata.nim

@@ -86,6 +86,7 @@ type
     options*: TOptions        # options that should be used for code
                               # generation; this is the same as prc.options
                               # unless prc == nil
+    optionsStack*: seq[TOptions]
     module*: BModule          # used to prevent excessive parameter passing
     withinLoop*: int          # > 0 if we are within a loop
     splitDecls*: int          # > 0 if we are in some context for C++ that
@@ -172,6 +173,7 @@ type
 
 template config*(m: BModule): ConfigRef = m.g.config
 template config*(p: BProc): ConfigRef = p.module.g.config
+template vccAndC*(p: BProc): bool = p.module.config.cCompiler == ccVcc and p.module.config.backend == backendC
 
 proc includeHeader*(this: BModule; header: string) =
   if not this.headerFiles.contains header:
@@ -191,15 +193,15 @@ proc initBlock*(): TBlock =
     result.sections[i] = newRopeAppender()
 
 proc newProc*(prc: PSym, module: BModule): BProc =
-  new(result)
-  result.prc = prc
-  result.module = module
-  result.options = if prc != nil: prc.options
-                   else: module.config.options
-  result.blocks = @[initBlock()]
-  result.nestedTryStmts = @[]
-  result.finallySafePoints = @[]
-  result.sigConflicts = initCountTable[string]()
+  result = BProc(
+    prc: prc,
+    module: module,
+    options: if prc != nil: prc.options
+             else: module.config.options,
+    blocks: @[initBlock()],
+    sigConflicts: initCountTable[string]())
+  if optQuirky in result.options:
+    result.flags = {nimErrorFlagDisabled}
 
 proc newModuleList*(g: ModuleGraph): BModuleList =
   BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: int32]](),

compiler/cgmeth.nim

@@ -113,7 +113,7 @@ proc attachDispatcher(s: PSym, dispatcher: PNode) =
     s.ast[dispatcherPos] = dispatcher
 
 proc createDispatcher(s: PSym; g: ModuleGraph; idgen: IdGenerator): PSym =
-  var disp = copySym(s, nextSymId(idgen))
+  var disp = copySym(s, idgen)
   incl(disp.flags, sfDispatcher)
   excl(disp.flags, sfExported)
   let old = disp.typ
@@ -127,7 +127,7 @@ proc createDispatcher(s: PSym; g: ModuleGraph; idgen: IdGenerator): PSym =
   disp.loc.r = ""
   if s.typ[0] != nil:
     if disp.ast.len > resultPos:
-      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, nextSymId(idgen))
+      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, idgen)
     else:
       # We've encountered a method prototype without a filled-in
       # resultPos slot. We put a placeholder in there that will
@@ -213,7 +213,7 @@ proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
       a[j] = v
     if h == 1: break
 
-proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet): PSym =
+proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet; idgen: IdGenerator): PSym =
   var base = methods[0].ast[dispatcherPos].sym
   result = base
   var paramLen = base.typ.len
@@ -272,7 +272,7 @@ proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet): PS
   nilchecks.flags.incl nfTransf # should not be further transformed
   result.ast[bodyPos] = nilchecks
 
-proc generateMethodDispatchers*(g: ModuleGraph): PNode =
+proc generateMethodDispatchers*(g: ModuleGraph, idgen: IdGenerator): PNode =
   result = newNode(nkStmtList)
   for bucket in 0..<g.methods.len:
     var relevantCols = initIntSet()
@@ -282,4 +282,4 @@ proc generateMethodDispatchers*(g: ModuleGraph): PNode =
         # if multi-methods are not enabled, we are interested only in the first field
         break
     sortBucket(g.methods[bucket].methods, relevantCols)
-    result.add newSymNode(genDispatcher(g, g.methods[bucket].methods, relevantCols))
+    result.add newSymNode(genDispatcher(g, g.methods[bucket].methods, relevantCols, idgen))

compiler/closureiters.nim

@@ -183,7 +183,7 @@ proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
   ctx.newStateAssgn(newIntTypeNode(stateNo, ctx.g.getSysType(TLineInfo(), tyInt)))
 
 proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
-  result = newSym(skVar, getIdent(ctx.g.cache, name), nextSymId(ctx.idgen), ctx.fn, ctx.fn.info)
+  result = newSym(skVar, getIdent(ctx.g.cache, name), ctx.idgen, ctx.fn, ctx.fn.info)
   result.typ = typ
   assert(not typ.isNil)
 
@@ -855,7 +855,9 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
   case n.kind
   of nkReturnStmt:
     # We're somewhere in try, transform to finally unrolling
-    assert(ctx.nearestFinally != 0)
+    if ctx.nearestFinally == 0:
+      # return is within the finally
+      return
 
     result = newNodeI(nkStmtList, n.info)
 
@@ -1348,7 +1350,7 @@ proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
         let idefs = copyNode(it)
         for v in 0..it.len-3:
           if it[v].kind == nkSym:
-            let x = copySym(it[v].sym, nextSymId(c.idgen))
+            let x = copySym(it[v].sym, c.idgen)
             c.tab[it[v].sym.id] = x
             idefs.add newSymNode(x)
           else:
@@ -1389,7 +1391,7 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode =
       discard c.finallys.pop()
 
   of nkWhileStmt, nkBlockStmt:
-    if n.hasYields == false: return n
+    if not n.hasYields: return n
     c.blocks.add((n, c.finallys.len))
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
@@ -1431,9 +1433,9 @@ proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n:
     # Lambda lifting was not done yet. Use temporary :state sym, which will
     # be handled specially by lambda lifting. Local temp vars (if needed)
     # should follow the same logic.
-    ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), nextSymId(idgen), fn, fn.info)
+    ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), idgen, fn, fn.info)
     ctx.stateVarSym.typ = g.createClosureIterStateType(fn, idgen)
-  ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), nextSymId(idgen), fn, fn.info)
+  ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), idgen, fn, fn.info)
   var pc = PreprocessContext(finallys: @[], config: g.config, idgen: idgen)
   var n = preprocess(pc, n.toStmtList)
   #echo "transformed into ", n
@@ -1466,9 +1468,10 @@ proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n:
   result = ctx.transformStateAssignments(result)
   result = ctx.wrapIntoStateLoop(result)
 
-  # echo "TRANSFORM TO STATES: "
-  # echo renderTree(result)
+  when false:
+    echo "TRANSFORM TO STATES: "
+    echo renderTree(result)
 
-  # echo "exception table:"
-  # for i, e in ctx.exceptionTable:
-  #   echo i, " -> ", e
+    echo "exception table:"
+    for i, e in ctx.exceptionTable:
+      echo i, " -> ", e

compiler/cmdlinehelper.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-## Helpers for binaries that use compiler passes, e.g.: nim, nimsuggest, nimfix
+## Helpers for binaries that use compiler passes, e.g.: nim, nimsuggest
 
 import
   options, idents, nimconf, extccomp, commands, msgs,

compiler/commands.nim

@@ -238,9 +238,9 @@ proc processCompile(conf: ConfigRef; filename: string) =
   extccomp.addExternalFileToCompile(conf, found)
 
 const
-  errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found"
+  errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'atomicArc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
-  errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found"
+  errGuiConsoleOrLibExpectedButXFound = "'gui', 'console', 'lib' or 'staticlib' expected, but '$1' found"
   errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found"
 
 template warningOptionNoop(switch: string) =
@@ -262,6 +262,7 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
     of "go": result = conf.selectedGC == gcGo
     of "none": result = conf.selectedGC == gcNone
     of "stack", "regions": result = conf.selectedGC == gcRegions
+    of "atomicarc": result = conf.selectedGC == gcAtomicArc
     else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "opt":
     case arg.normalize
@@ -336,6 +337,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace)
   of "nilseqs", "nilchecks", "taintmode": warningOptionNoop(switch)
   of "panics": result = contains(conf.globalOptions, optPanics)
+  of "jsbigint64": result = contains(conf.globalOptions, optJsBigInt64)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc processPath(conf: ConfigRef; path: string, info: TLineInfo,
@@ -515,14 +517,7 @@ proc initOrcDefines*(conf: ConfigRef) =
   if conf.exc == excNone and conf.backend != backendCpp:
     conf.exc = excGoto
 
-proc registerArcOrc(pass: TCmdLinePass, conf: ConfigRef, isOrc: bool) =
-  if isOrc:
-    conf.selectedGC = gcOrc
-    defineSymbol(conf.symbols, "gcorc")
-  else:
-    conf.selectedGC = gcArc
-    defineSymbol(conf.symbols, "gcarc")
-
+proc registerArcOrc(pass: TCmdLinePass, conf: ConfigRef) =
   defineSymbol(conf.symbols, "gcdestructors")
   incl conf.globalOptions, optSeqDestructors
   incl conf.globalOptions, optTinyRtti
@@ -532,10 +527,11 @@ proc registerArcOrc(pass: TCmdLinePass, conf: ConfigRef, isOrc: bool) =
   if conf.exc == excNone and conf.backend != backendCpp:
     conf.exc = excGoto
 
-proc unregisterArcOrc(conf: ConfigRef) =
+proc unregisterArcOrc*(conf: ConfigRef) =
   undefSymbol(conf.symbols, "gcdestructors")
   undefSymbol(conf.symbols, "gcarc")
   undefSymbol(conf.symbols, "gcorc")
+  undefSymbol(conf.symbols, "gcatomicarc")
   undefSymbol(conf.symbols, "nimSeqsV2")
   undefSymbol(conf.symbols, "nimV2")
   excl conf.globalOptions, optSeqDestructors
@@ -561,9 +557,17 @@ proc processMemoryManagementOption(switch, arg: string, pass: TCmdLinePass,
       conf.selectedGC = gcMarkAndSweep
       defineSymbol(conf.symbols, "gcmarkandsweep")
     of "destructors", "arc":
-      registerArcOrc(pass, conf, false)
+      conf.selectedGC = gcArc
+      defineSymbol(conf.symbols, "gcarc")
+      registerArcOrc(pass, conf)
     of "orc":
-      registerArcOrc(pass, conf, true)
+      conf.selectedGC = gcOrc
+      defineSymbol(conf.symbols, "gcorc")
+      registerArcOrc(pass, conf)
+    of "atomicarc":
+      conf.selectedGC = gcAtomicArc
+      defineSymbol(conf.symbols, "gcatomicarc")
+      registerArcOrc(pass, conf)
     of "hooks":
       conf.selectedGC = gcHooks
       defineSymbol(conf.symbols, "gchooks")
@@ -610,6 +614,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       var path = processPath(conf, arg, info, notRelativeToProj=true)
       let nimbleDir = AbsoluteDir getEnv("NIMBLE_DIR")
       if not nimbleDir.isEmpty and pass == passPP:
+        path = nimbleDir / RelativeDir"pkgs2"
+        nimblePath(conf, path, info)
         path = nimbleDir / RelativeDir"pkgs"
       nimblePath(conf, path, info)
   of "nonimblepath", "nobabelpath":
@@ -649,6 +655,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "backend", "b":
     let backend = parseEnum(arg.normalize, TBackend.default)
     if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
+    if backend == backendJs: # bug #21209
+      conf.globalOptions.excl {optThreadAnalysis, optThreads}
+      if optRun in conf.globalOptions:
+        # for now, -r uses nodejs, so define nodejs
+        defineSymbol(conf.symbols, "nodejs")
     conf.backend = backend
   of "doccmd": conf.docCmd = arg
   of "define", "d":
@@ -796,6 +807,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       defineSymbol(conf.symbols, "dll")
     of "staticlib":
       incl(conf.globalOptions, optGenStaticLib)
+      incl(conf.globalOptions, optNoMain)
       excl(conf.globalOptions, optGenGuiApp)
       defineSymbol(conf.symbols, "library")
       defineSymbol(conf.symbols, "staticlib")
@@ -819,8 +831,16 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "header":
     if conf != nil: conf.headerFile = arg
     incl(conf.globalOptions, optGenIndex)
+  of "nimbasepattern":
+    if conf != nil: conf.nimbasePattern = arg
   of "index":
-    processOnOffSwitchG(conf, {optGenIndex}, arg, pass, info)
+    case arg.normalize
+    of "", "on": conf.globalOptions.incl {optGenIndex}
+    of "only": conf.globalOptions.incl {optGenIndexOnly, optGenIndex}
+    of "off": conf.globalOptions.excl {optGenIndex, optGenIndexOnly}
+    else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
+  of "noimportdoc":
+    processOnOffSwitchG(conf, {optNoImportdoc}, arg, pass, info)
   of "import":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
@@ -853,6 +873,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       setTarget(conf.target, conf.target.targetOS, cpu)
   of "run", "r":
     processOnOffSwitchG(conf, {optRun}, arg, pass, info)
+    if conf.backend == backendJs:
+      # for now, -r uses nodejs, so define nodejs
+      defineSymbol(conf.symbols, "nodejs")
   of "maxloopiterationsvm":
     expectArg(conf, switch, arg, pass, info)
     conf.maxLoopIterationsVM = parseInt(arg)
@@ -867,7 +890,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "verbosity":
     expectArg(conf, switch, arg, pass, info)
     let verbosity = parseInt(arg)
-    if verbosity notin {0..3}:
+    if verbosity notin 0..3:
       localError(conf, info, "invalid verbosity level: '$1'" % arg)
     conf.verbosity = verbosity
     var verb = NotesVerbosity[conf.verbosity]
@@ -1000,6 +1023,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectNoArg(conf, switch, arg, pass, info)
     conf.exc = low(ExceptionSystem)
     defineSymbol(conf.symbols, "noCppExceptions")
+  of "shownonexports":
+    expectNoArg(conf, switch, arg, pass, info)
+    showNonExportedFields(conf)
   of "exceptions":
     case arg.normalize
     of "cpp": conf.exc = excCpp
@@ -1054,29 +1080,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "expandarc":
     expectArg(conf, switch, arg, pass, info)
     conf.arcToExpand[arg] = "T"
-  of "useversion":
-    expectArg(conf, switch, arg, pass, info)
-    case arg
-    of "1.0":
-      defineSymbol(conf.symbols, "NimMajor", "1")
-      defineSymbol(conf.symbols, "NimMinor", "0")
-      # old behaviors go here:
-      defineSymbol(conf.symbols, "nimOldRelativePathBehavior")
-      undefSymbol(conf.symbols, "nimDoesntTrackDefects")
-      ast.eqTypeFlags.excl {tfGcSafe, tfNoSideEffect}
-      conf.globalOptions.incl optNimV1Emulation
-    of "1.2":
-      defineSymbol(conf.symbols, "NimMajor", "1")
-      defineSymbol(conf.symbols, "NimMinor", "2")
-      conf.globalOptions.incl optNimV12Emulation
-    of "1.6":
-      defineSymbol(conf.symbols, "NimMajor", "1")
-      defineSymbol(conf.symbols, "NimMinor", "6")
-      conf.globalOptions.incl optNimV16Emulation
-    else:
-      localError(conf, info, "unknown Nim version; currently supported values are: `1.0`, `1.2`")
-    # always be compatible with 1.x.100:
-    defineSymbol(conf.symbols, "NimPatch", "100")
   of "benchmarkvm":
     processOnOffSwitchG(conf, {optBenchmarkVM}, arg, pass, info)
   of "profilevm":
@@ -1090,6 +1093,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitchG(conf, {optPanics}, arg, pass, info)
     if optPanics in conf.globalOptions:
       defineSymbol(conf.symbols, "nimPanics")
+  of "jsbigint64":
+    processOnOffSwitchG(conf, {optJsBigInt64}, arg, pass, info)
   of "sourcemap": # xxx document in --fullhelp
     conf.globalOptions.incl optSourcemap
     conf.options.incl optLineDir

compiler/compiler.nimble

@@ -0,0 +1,28 @@
+include "../lib/system/compilation.nim"
+version = $NimMajor & "." & $NimMinor & "." & $NimPatch
+author = "Andreas Rumpf"
+description = "Compiler package providing the compiler sources as a library."
+license = "MIT"
+skipDirs = @["."]
+installDirs = @["compiler"]
+
+import os
+
+var compilerDir = ""
+
+before install:
+  rmDir("compiler")
+
+  let
+    files = listFiles(".")
+    dirs = listDirs(".")
+
+  mkDir("compiler")
+
+  for f in files:
+    cpFile(f, "compiler" / f)
+
+  for d in dirs:
+    cpDir(d, "compiler" / d)
+
+requires "nim"

compiler/concepts.nim

@@ -27,7 +27,7 @@ const
 proc declareSelf(c: PContext; info: TLineInfo) =
   ## Adds the magical 'Self' symbols to the current scope.
   let ow = getCurrOwner(c)
-  let s = newSym(skType, getIdent(c.cache, "Self"), nextSymId(c.idgen), ow, info)
+  let s = newSym(skType, getIdent(c.cache, "Self"), c.idgen, ow, info)
   s.typ = newType(tyTypeDesc, nextTypeId(c.idgen), ow)
   s.typ.flags.incl {tfUnresolved, tfPacked}
   s.typ.add newType(tyEmpty, nextTypeId(c.idgen), ow)
@@ -62,7 +62,7 @@ proc semConceptDecl(c: PContext; n: PNode): PNode =
       result[i] = n[i]
     result[^1] = semConceptDecl(c, n[^1])
   of nkCommentStmt:
-    discard
+    result = n
   else:
     localError(c.config, n.info, "unexpected construct in the new-styled concept: " & renderTree(n))
     result = n
@@ -274,7 +274,7 @@ proc matchSym(c: PContext; candidate: PSym, n: PNode; m: var MatchCon): bool =
 proc matchSyms(c: PContext, n: PNode; kinds: set[TSymKind]; m: var MatchCon): bool =
   ## Walk the current scope, extract candidates which the same name as 'n[namePos]',
   ## 'n' is the nkProcDef or similar from the concept that we try to match.
-  let candidates = searchInScopesFilterBy(c, n[namePos].sym.name, kinds)
+  let candidates = searchInScopesAllCandidatesFilterBy(c, n[namePos].sym.name, kinds)
   for candidate in candidates:
     #echo "considering ", typeToString(candidate.typ), " ", candidate.magic
     m.magic = candidate.magic
@@ -306,6 +306,8 @@ proc conceptMatchNode(c: PContext; n: PNode; m: var MatchCon): bool =
     result = matchSyms(c, n, {skMethod}, m)
   of nkIteratorDef:
     result = matchSyms(c, n, {skIterator}, m)
+  of nkCommentStmt:
+    result = true
   else:
     # error was reported earlier.
     result = false

compiler/condsyms.nim

@@ -25,7 +25,7 @@ proc undefSymbol*(symbols: StringTableRef; symbol: string) =
 #  result = if isDefined(symbol): gSymbols[symbol] else: nil
 
 iterator definedSymbolNames*(symbols: StringTableRef): string =
-  for key, val in pairs(symbols):
+  for key in keys(symbols):
     yield key
 
 proc countDefinedSymbols*(symbols: StringTableRef): int =
@@ -45,10 +45,8 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimNewTypedesc") # deadcode
   defineSymbol("nimrequiresnimframe") # deadcode
   defineSymbol("nimparsebiggestfloatmagic") # deadcode
-  defineSymbol("nimalias") # deadcode
   defineSymbol("nimlocks") # deadcode
-  defineSymbol("nimnode") # deadcode pending `nimnode` reference in opengl package
-    # refs https://github.com/nim-lang/opengl/pull/79
+  defineSymbol("nimnode") # deadcode
   defineSymbol("nimvarargstyped") # deadcode
   defineSymbol("nimtypedescfixed") # deadcode
   defineSymbol("nimKnowsNimvm") # deadcode
@@ -84,10 +82,24 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasSignatureHashInMacro") # deadcode
   defineSymbol("nimHasDefault") # deadcode
   defineSymbol("nimMacrosSizealignof") # deadcode
+  defineSymbol("nimNoZeroExtendMagic") # deadcode
+  defineSymbol("nimMacrosGetNodeId") # deadcode
+  defineSymbol("nimFixedForwardGeneric") # deadcode
+  defineSymbol("nimToOpenArrayCString") # deadcode
+  defineSymbol("nimHasUsed") # deadcode
+  defineSymbol("nimnomagic64") # deadcode
+  defineSymbol("nimNewShiftOps") # deadcode
+  defineSymbol("nimHasCursor") # deadcode
+  defineSymbol("nimAlignPragma") # deadcode
+  defineSymbol("nimHasExceptionsQuery") # deadcode
+  defineSymbol("nimHasIsNamedTuple") # deadcode
+  defineSymbol("nimHashOrdinalFixed") # deadcode
+  defineSymbol("nimHasSinkInference") # deadcode
+  defineSymbol("nimNewIntegerOps") # deadcode
+  defineSymbol("nimHasInvariant") # deadcode
+
+
 
-  # > 0.20.0
-  defineSymbol("nimNoZeroExtendMagic")
-  defineSymbol("nimMacrosGetNodeId")
   for f in Feature:
     defineSymbol("nimHas" & $f)
 
@@ -98,31 +110,18 @@ proc initDefines*(symbols: StringTableRef) =
 
   defineSymbol("nimFixedOwned")
   defineSymbol("nimHasStyleChecks")
-  defineSymbol("nimToOpenArrayCString")
-  defineSymbol("nimHasUsed")
-  defineSymbol("nimFixedForwardGeneric")
-  defineSymbol("nimnomagic64")
-  defineSymbol("nimNewShiftOps")
-  defineSymbol("nimHasCursor")
-  defineSymbol("nimAlignPragma")
-  defineSymbol("nimHasExceptionsQuery")
-  defineSymbol("nimHasIsNamedTuple")
-  defineSymbol("nimHashOrdinalFixed")
 
   when defined(nimHasLibFFI):
     # Renaming as we can't conflate input vs output define flags; e.g. this
     # will report the right thing regardless of whether user adds
     # `-d:nimHasLibFFI` in his user config.
-    defineSymbol("nimHasLibFFIEnabled")
+    defineSymbol("nimHasLibFFIEnabled") # deadcode
 
-  defineSymbol("nimHasSinkInference")
-  defineSymbol("nimNewIntegerOps")
-  defineSymbol("nimHasInvariant")
-  defineSymbol("nimHasStacktraceMsgs")
+  defineSymbol("nimHasStacktraceMsgs") # deadcode
   defineSymbol("nimDoesntTrackDefects")
-  defineSymbol("nimHasLentIterators")
-  defineSymbol("nimHasDeclaredMagic")
-  defineSymbol("nimHasStacktracesModule")
+  defineSymbol("nimHasLentIterators") # deadcode
+  defineSymbol("nimHasDeclaredMagic") # deadcode
+  defineSymbol("nimHasStacktracesModule") # deadcode
   defineSymbol("nimHasEffectTraitsModule")
   defineSymbol("nimHasCastPragmaBlocks")
   defineSymbol("nimHasDeclaredLocs")
@@ -133,8 +132,8 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasCustomLiterals")
   defineSymbol("nimHasUnifiedTuple")
   defineSymbol("nimHasIterable")
-  defineSymbol("nimHasTypeofVoid")
-  defineSymbol("nimHasDragonBox")
+  defineSymbol("nimHasTypeofVoid") # deadcode
+  defineSymbol("nimHasDragonBox") # deadcode
   defineSymbol("nimHasHintAll")
   defineSymbol("nimHasTrace")
   defineSymbol("nimHasEffectsOf")
@@ -146,8 +145,16 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasCallsitePragma")
   defineSymbol("nimHasAmbiguousEnumHint")
 
+  defineSymbol("nimHasWarnCastSizes") # deadcode
   defineSymbol("nimHasOutParams")
   defineSymbol("nimHasSystemRaisesDefect")
   defineSymbol("nimHasWarnUnnamedBreak")
   defineSymbol("nimHasGenericDefine")
   defineSymbol("nimHasDefineAliases")
+  defineSymbol("nimHasWarnBareExcept")
+  defineSymbol("nimHasDup")
+  defineSymbol("nimHasChecksums")
+  defineSymbol("nimHasSendable")
+  defineSymbol("nimAllowNonVarDestructor")
+  defineSymbol("nimHasQuirky")
+  defineSymbol("nimHasEnsureMove")

compiler/depends.nim

@@ -9,7 +9,7 @@
 
 # This module implements a dependency file generator.
 
-import options, ast, ropes, passes, pathutils, msgs, lineinfos
+import options, ast, ropes, pathutils, msgs, lineinfos
 
 import modulegraphs
 
@@ -63,12 +63,16 @@ proc toNimblePath(s: string, isStdlib: bool): string =
       sub.add "/pkgs/"
     var start = s.find(sub)
     if start < 0:
-      result = s
-    else:
-      start += sub.len
-      start += skipUntil(s, '/', start)
-      start += 1
-      result = pkgPrefix & s[start..^1]
+      sub[^1] = '2'
+      sub.add '/'
+      start = s.find(sub) # /pkgs2
+      if start < 0:
+        return s
+
+    start += sub.len
+    start += skipUntil(s, '/', start)
+    start += 1
+    result = pkgPrefix & s[start..^1]
 
 proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) =
   doAssert n.kind == nkSym, $n.kind
@@ -79,7 +83,7 @@ proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) =
   let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym))
   addDependencyAux(b, parent, child)
 
-proc addDotDependency(c: PPassContext, n: PNode): PNode =
+proc addDotDependency*(c: PPassContext, n: PNode): PNode =
   result = n
   let g = PGen(c)
   let b = Backend(g.graph.backend)
@@ -100,10 +104,7 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) =
       rope(project.splitFile.name), b.dotGraph],
             changeFileExt(project, "dot"))
 
-when not defined(nimHasSinkInference):
-  {.pragma: nosinks.}
-
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
+proc setupDependPass*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   var g: PGen
   new(g)
   g.module = module
@@ -112,6 +113,3 @@ proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext
   if graph.backend == nil:
     graph.backend = Backend(dotGraph: "")
   result = g
-
-const gendependPass* = makePass(open = myOpen, process = addDotDependency)
-

compiler/dfa.nim

@@ -264,7 +264,8 @@ proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) =
     c.blocks[i].raiseFixups.add lab1
   else:
     var trailingFinales: seq[PNode]
-    if c.inTryStmt > 0: #Ok, we are in a try, lets see which (if any) try's we break out from:
+    if c.inTryStmt > 0:
+      # Ok, we are in a try, lets see which (if any) try's we break out from:
       for b in countdown(c.blocks.high, i):
         if c.blocks[b].isTryBlock:
           trailingFinales.add c.blocks[b].finale
@@ -385,7 +386,8 @@ proc genCall(c: var Con; n: PNode) =
       # Pass by 'out' is a 'must def'. Good enough for a move optimizer.
       genDef(c, n[i])
   # every call can potentially raise:
-  if false: # c.inTryStmt > 0 and canRaiseConservative(n[0]):
+  if c.inTryStmt > 0 and canRaiseConservative(n[0]):
+    inc c.interestingInstructions
     # we generate the instruction sequence:
     # fork lab1
     # goto exceptionHandler (except or finally)
@@ -493,7 +495,7 @@ proc constructCfg*(s: PSym; body: PNode; root: PSym): ControlFlowGraph =
     gen(c, body)
     if root.kind == skResult:
       genImplicitReturn(c)
-  when defined(gcArc) or defined(gcOrc):
+  when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc):
     result = c.code # will move
   else:
     shallowCopy(result, c.code)

compiler/docgen.nim

@@ -7,13 +7,17 @@
 #    distribution, for details about the copyright.
 #
 
-# This is the documentation generator. Cross-references are generated
-# by knowing how the anchors are going to be named.
+## This is the Nim documentation generator. Cross-references are generated
+## by knowing how the anchors are going to be named.
+##
+## .. importdoc:: ../docgen.md
+##
+## For corresponding users' documentation see [Nim DocGen Tools Guide].
 
 import
   ast, strutils, strtabs, algorithm, sequtils, options, msgs, os, idents,
   wordrecg, syntaxes, renderer, lexer,
-  packages/docutils/[rst, rstgen, dochelpers],
+  packages/docutils/[rst, rstidx, rstgen, dochelpers],
   json, xmltree, trees, types,
   typesrenderer, astalgo, lineinfos, intsets,
   pathutils, tables, nimpaths, renderverbatim, osproc, packages
@@ -91,7 +95,7 @@ type
     jEntriesFinal: JsonNode    # final JSON after RST pass 2 and rendering
     types: TStrTable
     sharedState: PRstSharedState
-    standaloneDoc: bool
+    standaloneDoc: bool        # is markup (.rst/.md) document?
     conf*: ConfigRef
     cache*: IdentCache
     exampleCounter: int
@@ -108,13 +112,16 @@ type
 proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst)
 proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str)
 
-proc addRstFileIndex(d: PDoc, info: lineinfos.TLineInfo): rstast.FileIndex =
+proc addRstFileIndex(d: PDoc, fileIndex: lineinfos.FileIndex): rstast.FileIndex =
   let invalid = rstast.FileIndex(-1)
-  result = d.nimToRstFid.getOrDefault(info.fileIndex, default = invalid)
+  result = d.nimToRstFid.getOrDefault(fileIndex, default = invalid)
   if result == invalid:
-    let fname = toFullPath(d.conf, info)
+    let fname = toFullPath(d.conf, fileIndex)
     result = addFilename(d.sharedState, fname)
-    d.nimToRstFid[info.fileIndex] = result
+    d.nimToRstFid[fileIndex] = result
+
+proc addRstFileIndex(d: PDoc, info: lineinfos.TLineInfo): rstast.FileIndex =
+  addRstFileIndex(d, info.fileIndex)
 
 proc cmpDecimalsIgnoreCase(a, b: string): int =
   ## For sorting with correct handling of cases like 'uint8' and 'uint16'.
@@ -225,7 +232,7 @@ proc attachToType(d: PDoc; p: PSym): PSym =
   if params.len > 0: check(0)
   for i in 2..<params.len: check(i)
 
-template declareClosures =
+template declareClosures(currentFilename: AbsoluteFile, destFile: string) =
   proc compilerMsgHandler(filename: string, line, col: int,
                           msgKind: rst.MsgKind, arg: string) {.gcsafe.} =
     # translate msg kind:
@@ -249,9 +256,19 @@ template declareClosures =
     of mwBrokenLink: k = warnRstBrokenLink
     of mwUnsupportedLanguage: k = warnRstLanguageXNotSupported
     of mwUnsupportedField: k = warnRstFieldXNotSupported
+    of mwUnusedImportdoc: k = warnRstUnusedImportdoc
     of mwRstStyle: k = warnRstStyle
     {.gcsafe.}:
-      globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg)
+      let errorsAsWarnings = (roPreferMarkdown in d.sharedState.options) and
+          not d.standaloneDoc  # not tolerate errors in .rst/.md files
+      if whichMsgClass(msgKind) == mcError and errorsAsWarnings:
+        liMessage(conf, newLineInfo(conf, AbsoluteFile filename, line, col),
+                  k, arg, doNothing, instLoc(), ignoreError=true)
+        # when our Markdown parser fails, we currently can only terminate the
+        # parsing (and then we will return monospaced text instead of markup):
+        raiseRecoverableError("")
+      else:
+        globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg)
 
   proc docgenFindFile(s: string): string {.gcsafe.} =
     result = options.findFile(conf, s).string
@@ -259,10 +276,29 @@ template declareClosures =
       result = getCurrentDir() / s
       if not fileExists(result): result = ""
 
+  proc docgenFindRefFile(targetRelPath: string):
+         tuple[targetPath: string, linkRelPath: string] {.gcsafe.} =
+    let fromDir = splitFile(destFile).dir  # dir where we reference from
+    let basedir = os.splitFile(currentFilename.string).dir
+    let outDirPath: RelativeFile =
+        presentationPath(conf, AbsoluteFile(basedir / targetRelPath))
+          # use presentationPath because `..` path can be be mangled to `_._`
+    result.targetPath = string(conf.outDir / outDirPath)
+    if not fileExists(result.targetPath):
+      # this can happen if targetRelPath goes to parent directory `OUTDIR/..`.
+      # Trying it, this may cause ambiguities, but allows us to insert
+      # "packages" into each other, which is actually used in Nim repo itself.
+      let destPath = fromDir / targetRelPath
+      if destPath != result.targetPath and fileExists(destPath):
+        result.targetPath = destPath
+
+    result.linkRelPath = relativePath(result.targetPath.splitFile.dir,
+                                      fromDir).replace('\\', '/')
+
+
 proc parseRst(text: string,
               line, column: int,
               conf: ConfigRef, sharedState: PRstSharedState): PRstNode =
-  declareClosures()
   result = rstParsePass1(text, line, column, sharedState)
 
 proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
@@ -283,8 +319,10 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
                     outExt: string = HtmlExt, module: PSym = nil,
                     standaloneDoc = false, preferMarkdown = true,
                     hasToc = true): PDoc =
-  declareClosures()
+  let destFile = getOutFile2(conf, presentationPath(conf, filename), outExt, false).string
   new(result)
+  let d = result  # pass `d` to `declareClosures`:
+  declareClosures(currentFilename = filename, destFile = destFile)
   result.module = module
   result.conf = conf
   result.cache = cache
@@ -298,7 +336,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   result.hasToc = hasToc
   result.sharedState = newRstSharedState(
       options, filename.string,
-      docgenFindFile, compilerMsgHandler, hasToc)
+      docgenFindFile, docgenFindRefFile, compilerMsgHandler, hasToc)
   initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
                    conf.configVars, filename.string,
                    docgenFindFile, compilerMsgHandler)
@@ -373,7 +411,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
       if gotten != status:
         rawMessage(conf, errGenerated, "snippet failed: cmd: '$1' status: $2 expected: $3 output: $4" % [cmd, $gotten, $status, output])
   result.emitted = initIntSet()
-  result.destFile = getOutFile2(conf, presentationPath(conf, filename), outExt, false).string
+  result.destFile = destFile
   result.thisDir = result.destFile.AbsoluteFile.splitFile.dir
 
 template dispA(conf: ConfigRef; dest: var string, xml, tex: string,
@@ -390,16 +428,19 @@ proc getVarIdx(varnames: openArray[string], id: string): int =
 proc genComment(d: PDoc, n: PNode): PRstNode =
   if n.comment.len > 0:
     d.sharedState.currFileIdx = addRstFileIndex(d, n.info)
-    result = parseRst(n.comment,
-                      toLinenumber(n.info),
-                      toColumn(n.info) + DocColOffset,
-                      d.conf, d.sharedState)
+    try:
+      result = parseRst(n.comment,
+                        toLinenumber(n.info),
+                        toColumn(n.info) + DocColOffset,
+                        d.conf, d.sharedState)
+    except ERecoverableError:
+      result = newRstNode(rnLiteralBlock, @[newRstLeaf(n.comment)])
 
 proc genRecCommentAux(d: PDoc, n: PNode): PRstNode =
   if n == nil: return nil
   result = genComment(d, n)
   if result == nil:
-    if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef,
+    if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef, nkTypeClassTy,
                   nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkSinkAsgn, nkHiddenStdConv}:
       # notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}:
       for i in 0..<n.len:
@@ -537,10 +578,13 @@ proc runAllExamples(d: PDoc) =
     # most useful semantics is that `docCmd` comes after `rdoccmd`, so that we can (temporarily) override
     # via command line
     # D20210224T221756:here
-    let cmd = "$nim $backend -r --lib:$libpath --warning:UnusedImport:off --path:$path --nimcache:$nimcache $rdoccmd $docCmd $file" % [
+    var pathArgs = "--path:$path" % [ "path", quoteShell(d.conf.projectPath) ]
+    for p in d.conf.searchPaths:
+      pathArgs = "$args --path:$path" % [ "args", pathArgs, "path", quoteShell(p) ]
+    let cmd = "$nim $backend -r --lib:$libpath --warning:UnusedImport:off $pathArgs --nimcache:$nimcache $rdoccmd $docCmd $file" % [
       "nim", quoteShell(os.getAppFilename()),
       "backend", $d.conf.backend,
-      "path", quoteShell(d.conf.projectPath),
+      "pathArgs", pathArgs,
       "libpath", quoteShell(d.conf.libpath),
       "nimcache", quoteShell(outputDir),
       "file", quoteShell(outp),
@@ -575,7 +619,6 @@ proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, c
   let useRenderModule = false
   let loc = d.conf.toFileLineCol(n.info)
   let code = extractRunnableExamplesSource(d.conf, n)
-  let codeIndent = extractRunnableExamplesSource(d.conf, n, indent = 2)
 
   if d.conf.errorCounter > 0:
     return (rdoccmd, code)
@@ -591,6 +634,7 @@ proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, c
     docComment.comment = comment
     var runnableExamples = newTree(nkStmtList,
         docComment,
+        newTree(nkImportStmt, newStrNode(nkStrLit, "std/assertions")),
         newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
     runnableExamples.info = n.info
     for a in n.lastSon: runnableExamples.add a
@@ -604,6 +648,7 @@ proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, c
   else:
     var code2 = code
     if code.len > 0 and "codeReordering" notin code:
+      let codeIndent = extractRunnableExamplesSource(d.conf, n, indent = 2)
       # hacky but simplest solution, until we devise a way to make `{.line.}`
       # work without introducing a scope
       code2 = """
@@ -614,6 +659,7 @@ $#
 #[
 $#
 ]#
+import std/assertions
 import $#
 $#
 """ % [comment, d.filename.quoted, code2]
@@ -765,21 +811,24 @@ proc isVisible(d: PDoc; n: PNode): bool =
   elif n.kind == nkPragmaExpr:
     result = isVisible(d, n[0])
 
-proc getName(d: PDoc, n: PNode, splitAfter = -1): string =
+proc getName(n: PNode): string =
   case n.kind
-  of nkPostfix: result = getName(d, n[1], splitAfter)
-  of nkPragmaExpr: result = getName(d, n[0], splitAfter)
-  of nkSym: result = esc(d.target, n.sym.renderDefinitionName, splitAfter)
-  of nkIdent: result = esc(d.target, n.ident.s, splitAfter)
+  of nkPostfix: result = getName(n[1])
+  of nkPragmaExpr: result = getName(n[0])
+  of nkSym: result = n.sym.renderDefinitionName
+  of nkIdent: result = n.ident.s
   of nkAccQuoted:
-    result = esc(d.target, "`")
-    for i in 0..<n.len: result.add(getName(d, n[i], splitAfter))
-    result.add esc(d.target, "`")
+    result = "`"
+    for i in 0..<n.len: result.add(getName(n[i]))
+    result = "`"
   of nkOpenSymChoice, nkClosedSymChoice:
-    result = getName(d, n[0], splitAfter)
+    result = getName(n[0])
   else:
     result = ""
 
+proc getNameEsc(d: PDoc, n: PNode): string =
+  esc(d.target, getName(n))
+
 proc getNameIdent(cache: IdentCache; n: PNode): PIdent =
   case n.kind
   of nkPostfix: result = getNameIdent(cache, n[1])
@@ -928,6 +977,13 @@ proc symbolPriority(k: TSymKind): int =
     else: 0  # including skProc which have higher priority
     # documentation itself has even higher priority 1
 
+proc getTypeKind(n: PNode): string =
+  case n[2].kind
+  of nkEnumTy: "enum"
+  of nkObjectTy: "object"
+  of nkTupleTy: "tuple"
+  else: ""
+
 proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol =
   ## Converts symbol info (names/types/parameters) in `n` into format
   ## `LangSymbol` convenient for ``rst.nim``/``dochelpers.nim``.
@@ -971,17 +1027,13 @@ proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol =
         if kind != tkSpaces:
           result.generics.add(literal.nimIdentNormalize)
 
-  if k == skType:
-    case n[2].kind
-    of nkEnumTy: result.symTypeKind = "enum"
-    of nkObjectTy: result.symTypeKind = "object"
-    of nkTupleTy: result.symTypeKind = "tuple"
-    else: discard
+  if k == skType: result.symTypeKind = getTypeKind(n)
 
-proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
+proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags, nonExports: bool = false) =
   if (docFlags != kForceExport) and not isVisible(d, nameNode): return
   let
-    name = getName(d, nameNode)
+    name = getName(nameNode)
+    nameEsc = esc(d.target, name)
   var plainDocstring = getPlainDocstring(n) # call here before genRecComment!
   var result = ""
   var literal, plainName = ""
@@ -1008,9 +1060,12 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   inc(d.id)
   let
     plainNameEsc = esc(d.target, plainName.strip)
-    detailedName = k.toHumanStr & " " & (
+    typeDescr =
+      if k == skType and getTypeKind(n) != "": getTypeKind(n)
+      else: k.toHumanStr
+    detailedName = typeDescr & " " & (
         if k in routineKinds: plainName else: name)
-    uniqueName = if k in routineKinds: plainNameEsc else: name
+    uniqueName = if k in routineKinds: plainNameEsc else: nameEsc
     sortName = if k in routineKinds: plainName.strip else: name
     cleanPlainSymbol = renderPlainSymbolName(nameNode)
     complexSymbol = complexName(k, n, cleanPlainSymbol)
@@ -1024,11 +1079,16 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   let lineinfo = rstast.TLineInfo(
       line: nameNode.info.line, col: nameNode.info.col,
       fileIndex: addRstFileIndex(d, nameNode.info))
-  addAnchorNim(d.sharedState, refn = symbolOrId, tooltip = detailedName,
-               rstLangSymbol, priority = symbolPriority(k), info = lineinfo)
+  addAnchorNim(d.sharedState, external = false, refn = symbolOrId,
+               tooltip = detailedName, langSym = rstLangSymbol,
+               priority = symbolPriority(k), info = lineinfo,
+               module = addRstFileIndex(d, FileIndex d.module.position))
 
-  nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,
-    renderDocComments, renderSyms, renderExpandUsing}, symbolOrIdEnc)
+  let renderFlags =
+    if nonExports: {renderNoBody, renderNoComments, renderDocComments, renderSyms,
+      renderExpandUsing, renderNonExportedFields}
+    else: {renderNoBody, renderNoComments, renderDocComments, renderSyms, renderExpandUsing}
+  nodeToHighlightedHtml(d, n, result, renderFlags, symbolOrIdEnc)
 
   let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
 
@@ -1060,8 +1120,10 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
       if e.sym.kind != skEnumField: continue
       let plain = renderPlainSymbolName(e)
       let symbolOrId = d.newUniquePlainSymbol(plain)
-      setIndexTerm(d[], external, symbolOrId, plain, nameNode.sym.name.s & '.' & plain,
-        xmltree.escape(getPlainDocstring(e).docstringSummary))
+      setIndexTerm(d[], ieNim, htmlFile = external, id = symbolOrId,
+                   term = plain, linkTitle = nameNode.sym.name.s & '.' & plain,
+                   linkDesc = xmltree.escape(getPlainDocstring(e).docstringSummary),
+                   line = n.info.line.int)
 
   d.tocSimple[k].add TocItem(
     sortName: sortName,
@@ -1076,25 +1138,23 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
       "itemSymOrID", symbolOrId.replace(",", ",<wbr>"),
       "itemSymOrIDEnc", symbolOrIdEnc])
 
-  # Ironically for types the complexSymbol is *cleaner* than the plainName
-  # because it doesn't include object fields or documentation comments. So we
-  # use the plain one for callable elements, and the complex for the rest.
-  var linkTitle = changeFileExt(extractFilename(d.filename), "") & ": "
-  if n.kind in routineDefs: linkTitle.add(xmltree.escape(plainName.strip))
-  else: linkTitle.add(xmltree.escape(complexSymbol.strip))
-
-  setIndexTerm(d[], external, symbolOrId, name, linkTitle,
-    xmltree.escape(plainDocstring.docstringSummary))
+  setIndexTerm(d[], ieNim, htmlFile = external, id = symbolOrId, term = name,
+               linkTitle = detailedName,
+               linkDesc = xmltree.escape(plainDocstring.docstringSummary),
+               line = n.info.line.int)
   if k == skType and nameNode.kind == nkSym:
     d.types.strTableAdd nameNode.sym
 
-proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem =
+proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind, nonExports = false): JsonItem =
   if not isVisible(d, nameNode): return
   var
-    name = getName(d, nameNode)
+    name = getNameEsc(d, nameNode)
     comm = genRecComment(d, n)
     r: TSrcGen
-  initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments, renderExpandUsing})
+    renderFlags = {renderNoBody, renderNoComments, renderDocComments, renderExpandUsing}
+  if nonExports:
+    renderFlags.incl renderNonExportedFields
+  initTokRender(r, n, renderFlags)
   result.json = %{ "name": %name, "type": %($k), "line": %n.info.line.int,
                    "col": %n.info.col}
   if comm != nil:
@@ -1279,12 +1339,13 @@ proc documentRaises*(cache: IdentCache; n: PNode) =
     if p5 != nil: n[pragmasPos].add p5
     if p6 != nil: n[pragmasPos].add p6
 
-proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
+proc generateDoc*(d: PDoc, n, orig: PNode, config: ConfigRef, docFlags: DocFlags = kDefault) =
   ## Goes through nim nodes recursively and collects doc comments.
   ## Main function for `doc`:option: command,
   ## which is implemented in ``docgen2.nim``.
   template genItemAux(skind) =
     genItem(d, n, n[namePos], skind, docFlags)
+  let showNonExports = optShowNonExportedFields in config.globalOptions
   case n.kind
   of nkPragma:
     let pragmaNode = findPragma(n, wDeprecated)
@@ -1311,20 +1372,20 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
       if n[i].kind != nkCommentStmt:
         # order is always 'type var let const':
         genItem(d, n[i], n[i][0],
-                succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags)
+                succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags, showNonExports)
   of nkStmtList:
-    for i in 0..<n.len: generateDoc(d, n[i], orig)
+    for i in 0..<n.len: generateDoc(d, n[i], orig, config)
   of nkWhenStmt:
     # generate documentation for the first branch only:
     if not checkForFalse(n[0][0]):
-      generateDoc(d, lastSon(n[0]), orig)
+      generateDoc(d, lastSon(n[0]), orig, config)
   of nkImportStmt:
     for it in n: traceDeps(d, it)
   of nkExportStmt:
     for it in n:
       if it.kind == nkSym:
         if d.module != nil and d.module == it.sym.owner:
-          generateDoc(d, it.sym.ast, orig, kForceExport)
+          generateDoc(d, it.sym.ast, orig, config, kForceExport)
         elif it.sym.ast != nil:
           exportSym(d, it.sym)
   of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept"
@@ -1337,9 +1398,31 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
 
 proc overloadGroupName(s: string, k: TSymKind): string =
   ## Turns a name like `f` into anchor `f-procs-all`
-  #s & " " & k.toHumanStr & "s all"
   s & "-" & k.toHumanStr & "s-all"
 
+proc setIndexTitle(d: PDoc, useMetaTitle: bool) =
+  let titleKind = if d.standaloneDoc: ieMarkupTitle else: ieNimTitle
+  let external = AbsoluteFile(d.destFile)
+    .relativeTo(d.conf.outDir, '/')
+    .changeFileExt(HtmlExt)
+    .string
+  var term, linkTitle: string
+  if useMetaTitle and d.meta[metaTitle].len != 0:
+    term = d.meta[metaTitleRaw]
+    linkTitle = d.meta[metaTitleRaw]
+  else:
+    let filename = extractFilename(d.filename)
+    term =
+      if d.standaloneDoc: filename  # keep .rst/.md extension
+      else: changeFileExt(filename, "")  # rm .nim extension
+    linkTitle =
+      if d.standaloneDoc: term  # keep .rst/.md extension
+      else: canonicalImport(d.conf, AbsoluteFile d.filename)
+  if not d.standaloneDoc:
+    linkTitle = "module " & linkTitle
+  setIndexTerm(d[], titleKind, htmlFile = external, id = "",
+               term = term, linkTitle = linkTitle)
+
 proc finishGenerateDoc*(d: var PDoc) =
   ## Perform 2nd RST pass for resolution of links/footnotes/headings...
   # copy file map `filenames` to ``rstgen.nim`` for its warnings
@@ -1352,7 +1435,24 @@ proc finishGenerateDoc*(d: var PDoc) =
       firstRst = fragment.rst
       break
   d.hasToc = d.hasToc or d.sharedState.hasToc
-  preparePass2(d.sharedState, firstRst)
+  # in --index:only mode we do NOT want to load other .idx, only write ours:
+  let importdoc = optGenIndexOnly notin d.conf.globalOptions and
+                  optNoImportdoc notin d.conf.globalOptions
+  preparePass2(d.sharedState, firstRst, importdoc)
+
+  if optGenIndexOnly in d.conf.globalOptions:
+    # Top-level doc.comments may contain titles and :idx: statements:
+    for fragment in d.modDescPre:
+      if fragment.isRst:
+        traverseForIndex(d[], fragment.rst)
+    setIndexTitle(d, useMetaTitle = d.standaloneDoc)
+    # Symbol-associated doc.comments may contain :idx: statements:
+    for k in TSymKind:
+      for _, overloadChoices in d.section[k].secItems:
+        for item in overloadChoices:
+          for fragment in item.descRst:
+            if fragment.isRst:
+              traverseForIndex(d[], fragment.rst)
 
   # add anchors to overload groups before RST resolution
   for k in TSymKind:
@@ -1362,13 +1462,25 @@ proc finishGenerateDoc*(d: var PDoc) =
           let refn = overloadGroupName(plainName, k)
           let tooltip = "$1 ($2 overloads)" % [
                       k.toHumanStr & " " & plainName, $overloadChoices.len]
-          addAnchorNim(d.sharedState, refn, tooltip,
+          let name = nimIdentBackticksNormalize(plainName)
+          # save overload group to ``.idx``
+          let external = d.destFile.AbsoluteFile.relativeTo(d.conf.outDir, '/').
+                         changeFileExt(HtmlExt).string
+          setIndexTerm(d[], ieNimGroup, htmlFile = external, id = refn,
+                       term = name, linkTitle = k.toHumanStr,
+                       linkDesc = "", line = overloadChoices[0].info.line.int)
+          if optGenIndexOnly in d.conf.globalOptions: continue
+          addAnchorNim(d.sharedState, external=false, refn, tooltip,
                        LangSymbol(symKind: k.toHumanStr,
-                                  name: nimIdentBackticksNormalize(plainName),
+                                  name: name,
                                   isGroup: true),
                        priority = symbolPriority(k),
                        # select index `0` just to have any meaningful warning:
-                       info = overloadChoices[0].info)
+                       info = overloadChoices[0].info,
+                       module = addRstFileIndex(d, FileIndex d.module.position))
+
+  if optGenIndexOnly in d.conf.globalOptions:
+    return
 
   # Finalize fragments of ``.nim`` or ``.rst`` file
   proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) =
@@ -1421,10 +1533,13 @@ proc finishGenerateDoc*(d: var PDoc) =
 
     d.jEntriesFinal.add entry.json # generates docs
 
+  setIndexTitle(d, useMetaTitle = d.standaloneDoc)
+  completePass2(d.sharedState)
+
 proc add(d: PDoc; j: JsonItem) =
   if j.json != nil or j.rst != nil: d.jEntriesPre.add j
 
-proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
+proc generateJson*(d: PDoc, n: PNode, config: ConfigRef, includeComments: bool = true) =
   case n.kind
   of nkPragma:
     let doctypeNode = findPragma(n, wDoctype)
@@ -1456,18 +1571,18 @@ proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
       if n[i].kind != nkCommentStmt:
         # order is always 'type var let const':
         d.add genJsonItem(d, n[i], n[i][0],
-                succ(skType, ord(n.kind)-ord(nkTypeSection)))
+                succ(skType, ord(n.kind)-ord(nkTypeSection)), optShowNonExportedFields in config.globalOptions)
   of nkStmtList:
     for i in 0..<n.len:
-      generateJson(d, n[i], includeComments)
+      generateJson(d, n[i], config, includeComments)
   of nkWhenStmt:
     # generate documentation for the first branch only:
     if not checkForFalse(n[0][0]):
-      generateJson(d, lastSon(n[0]), includeComments)
+      generateJson(d, lastSon(n[0]), config, includeComments)
   else: discard
 
 proc genTagsItem(d: PDoc, n, nameNode: PNode, k: TSymKind): string =
-  result = getName(d, nameNode) & "\n"
+  result = getNameEsc(d, nameNode) & "\n"
 
 proc generateTags*(d: PDoc, n: PNode, r: var string) =
   case n.kind
@@ -1574,13 +1689,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
   # Extract the title. Non API modules generate an entry in the index table.
   if d.meta[metaTitle].len != 0:
     title = d.meta[metaTitle]
-    let external = AbsoluteFile(d.destFile)
-      .relativeTo(d.conf.outDir, '/')
-      .changeFileExt(HtmlExt)
-      .string
-    setIndexTerm(d[], external, "", title)
   else:
-    # Modules get an automatic title for the HTML, but no entry in the index.
     title = canonicalImport(d.conf, AbsoluteFile d.filename)
   title = esc(d.target, title)
   var subtitle = ""
@@ -1619,11 +1728,17 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
     code = content
   result = code
 
+proc indexFile(d: PDoc): AbsoluteFile =
+  let dir = d.conf.outDir
+  result = dir / changeFileExt(presentationPath(d.conf,
+                                                AbsoluteFile d.filename),
+                               IndexExt)
+  let (finalDir, _, _) = result.string.splitFile
+  createDir(finalDir)
+
 proc generateIndex*(d: PDoc) =
   if optGenIndex in d.conf.globalOptions:
-    let dir = d.conf.outDir
-    createDir(dir)
-    let dest = dir / changeFileExt(presentationPath(d.conf, AbsoluteFile d.filename), IndexExt)
+    let dest = indexFile(d)
     writeIndexFile(d[], dest.string)
 
 proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
@@ -1634,6 +1749,9 @@ proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
         d.conf.outFile = splitPath(d.conf.outFile.string)[1].RelativeFile
 
 proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) =
+  if optGenIndexOnly in d.conf.globalOptions:
+    d.conf.outFile = indexFile(d).relativeTo(d.conf.outDir)  # just for display
+    return
   runAllExamples(d)
   var content = genOutFile(d, groupedToc)
   if optStdout in d.conf.globalOptions:
@@ -1698,7 +1816,7 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
   var ast = parseFile(conf.projectMainIdx, cache, conf)
   if ast == nil: return
   var d = newDocumentor(conf.projectFull, cache, conf, hasToc = true)
-  generateDoc(d, ast, ast)
+  generateDoc(d, ast, ast, conf)
   finishGenerateDoc(d)
   writeOutput(d)
   generateIndex(d)
@@ -1709,13 +1827,16 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef;
   var filen = addFileExt(filename, "txt")
   var d = newDocumentor(filen, cache, conf, outExt, standaloneDoc = true,
                         preferMarkdown = preferMarkdown, hasToc = false)
-  let rst = parseRst(readFile(filen.string),
-                     line=LineRstInit, column=ColRstInit,
-                     conf, d.sharedState)
-  d.modDescPre = @[ItemFragment(isRst: true, rst: rst)]
-  finishGenerateDoc(d)
-  writeOutput(d)
-  generateIndex(d)
+  try:
+    let rst = parseRst(readFile(filen.string),
+                      line=LineRstInit, column=ColRstInit,
+                      conf, d.sharedState)
+    d.modDescPre = @[ItemFragment(isRst: true, rst: rst)]
+    finishGenerateDoc(d)
+    writeOutput(d)
+    generateIndex(d)
+  except ERecoverableError:
+    discard "already reported the error"
 
 proc commandRst2Html*(cache: IdentCache, conf: ConfigRef,
                       preferMarkdown=false) =
@@ -1734,7 +1855,7 @@ proc commandJson*(cache: IdentCache, conf: ConfigRef) =
                           status: int; content: string) {.gcsafe.} =
     localError(conf, newLineInfo(conf, AbsoluteFile d.filename, -1, -1),
                warnUser, "the ':test:' attribute is not supported by this backend")
-  generateJson(d, ast)
+  generateJson(d, ast, conf)
   finishGenerateDoc(d)
   let json = d.jEntriesFinal
   let content = pretty(json)
@@ -1746,7 +1867,7 @@ proc commandJson*(cache: IdentCache, conf: ConfigRef) =
     let filename = getOutFile(conf, RelativeFile conf.projectName, JsonExt)
     try:
       writeFile(filename, content)
-    except:
+    except IOError:
       rawMessage(conf, errCannotOpenFile, filename.string)
 
 proc commandTags*(cache: IdentCache, conf: ConfigRef) =
@@ -1768,10 +1889,12 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) =
     let filename = getOutFile(conf, RelativeFile conf.projectName, TagsExt)
     try:
       writeFile(filename, content)
-    except:
+    except IOError:
       rawMessage(conf, errCannotOpenFile, filename.string)
 
 proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") =
+  if optGenIndexOnly in conf.globalOptions:
+    return
   var content = mergeIndexes(dir)
 
   var outFile = outFile
@@ -1789,7 +1912,7 @@ proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"")
 
   try:
     writeFile(filename, code)
-  except:
+  except IOError:
     rawMessage(conf, errCannotOpenFile, filename.string)
 
 proc commandBuildIndexJson*(conf: ConfigRef, dir: string, outFile = RelativeFile"") =
@@ -1803,5 +1926,5 @@ proc commandBuildIndexJson*(conf: ConfigRef, dir: string, outFile = RelativeFile
 
   try:
     writeFile(filename, $body)
-  except:
+  except IOError:
     rawMessage(conf, errCannotOpenFile, filename.string)

compiler/docgen2.nim

@@ -11,7 +11,7 @@
 # semantic checking.
 
 import
-  options, ast, msgs, passes, docgen, lineinfos, pathutils, packages
+  options, ast, msgs, docgen, lineinfos, pathutils, packages
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -38,25 +38,25 @@ template closeImpl(body: untyped) {.dirty.} =
     except IOError:
       discard
 
-proc close(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
+proc closeDoc*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
     writeOutput(g.doc, useWarning, groupedToc)
 
-proc closeJson(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
+proc closeJson*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
     writeOutputJson(g.doc, useWarning)
 
-proc processNode(c: PPassContext, n: PNode): PNode =
+proc processNode*(c: PPassContext, n: PNode): PNode =
   result = n
   var g = PGen(c)
   if shouldProcess(g):
-    generateDoc(g.doc, n, n)
+    generateDoc(g.doc, n, n, g.config)
 
-proc processNodeJson(c: PPassContext, n: PNode): PNode =
+proc processNodeJson*(c: PPassContext, n: PNode): PNode =
   result = n
   var g = PGen(c)
   if shouldProcess(g):
-    generateJson(g.doc, n, false)
+    generateJson(g.doc, n, g.config, false)
 
 template myOpenImpl(ext: untyped) {.dirty.} =
   var g: PGen
@@ -68,20 +68,11 @@ template myOpenImpl(ext: untyped) {.dirty.} =
   g.doc = d
   result = g
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc openHtml*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(HtmlExt)
 
-proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc openTex*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(TexExt)
 
-proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc openJson*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(JsonExt)
-
-const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
-const docgen2TexPass* = makePass(open = myOpenTex, process = processNode,
-                                 close = close)
-const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
-                                  close = closeJson)
-
-proc finishDoc2Pass*(project: string) =
-  discard

compiler/enumtostr.nim

@@ -6,12 +6,12 @@ when defined(nimPreviewSlimSystem):
 
 
 proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
-  result = newSym(skProc, getIdent(g.cache, "$"), nextSymId idgen, t.owner, info)
+  result = newSym(skProc, getIdent(g.cache, "$"), idgen, t.owner, info)
 
-  let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info)
+  let dest = newSym(skParam, getIdent(g.cache, "e"), idgen, result, info)
   dest.typ = t
 
-  let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), idgen, result, info)
   res.typ = getSysType(g, info, tyString)
 
   result.typ = newType(tyProc, nextTypeId idgen, t.owner)
@@ -67,12 +67,12 @@ proc searchObjCase(t: PType; field: PSym): PNode =
   doAssert result != nil
 
 proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
-  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), nextSymId idgen, t.owner, info)
+  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), idgen, t.owner, info)
 
-  let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info)
+  let dest = newSym(skParam, getIdent(g.cache, "e"), idgen, result, info)
   dest.typ = field.typ
 
-  let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), idgen, result, info)
   res.typ = getSysType(g, info, tyUInt8)
 
   result.typ = newType(tyProc, nextTypeId idgen, t.owner)

compiler/evalffi.nim

@@ -77,7 +77,7 @@ proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode =
       theAddr = dllhandle.symAddr(name.cstring)
     if theAddr.isNil: globalError(conf, sym.info,
       "cannot import symbol: " & name & " from " & libPathMsg)
-    result.intVal = cast[ByteAddress](theAddr)
+    result.intVal = cast[int](theAddr)
 
 proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type =
   if t == nil: return addr libffi.type_void
@@ -113,7 +113,7 @@ proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI
 template rd(typ, p: untyped): untyped = (cast[ptr typ](p))[]
 template wr(typ, p, v: untyped): untyped = (cast[ptr typ](p))[] = v
 template `+!`(x, y: untyped): untyped =
-  cast[pointer](cast[ByteAddress](x) + y)
+  cast[pointer](cast[int](x) + y)
 
 proc packSize(conf: ConfigRef, v: PNode, typ: PType): int =
   ## computes the size of the blob
@@ -369,13 +369,13 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
       # in their unboxed representation so nothing it to be unpacked:
       result = n
     else:
-      awi(nkPtrLit, cast[ByteAddress](p))
+      awi(nkPtrLit, cast[int](p))
   of tyPtr, tyRef, tyVar, tyLent:
     let p = rd(pointer, x)
     if p.isNil:
       setNil()
     elif n == nil or n.kind == nkPtrLit:
-      awi(nkPtrLit, cast[ByteAddress](p))
+      awi(nkPtrLit, cast[int](p))
     elif n != nil and n.len == 1:
       internalAssert(conf, n.kind == nkRefTy)
       n[0] = unpack(conf, p, typ.lastSon, n[0])

compiler/evaltempl.nim

@@ -10,7 +10,7 @@
 ## Template evaluation engine. Now hygienic.
 
 import
-  strutils, options, ast, astalgo, msgs, renderer, lineinfos, idents
+  strutils, options, ast, astalgo, msgs, renderer, lineinfos, idents, trees
 
 type
   TemplCtx = object
@@ -49,7 +49,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
         internalAssert c.config, sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
         if x == nil:
-          x = copySym(s, nextSymId(c.idgen))
+          x = copySym(s, c.idgen)
           # sem'check needs to set the owner properly later, see bug #9476
           x.owner = nil # c.genSymOwner
           #if x.kind == skParam and x.owner.kind == skModule:

compiler/extccomp.nim

@@ -14,13 +14,15 @@
 
 import ropes, platform, condsyms, options, msgs, lineinfos, pathutils, modulepaths
 
-import std/[os, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar, parseutils]
+import std/[os, osproc, streams, sequtils, times, strtabs, json, jsonutils, sugar, parseutils]
 
 import std / strutils except addf
 
 when defined(nimPreviewSlimSystem):
   import std/syncio
 
+import ../dist/checksums/src/checksums/sha1
+
 type
   TInfoCCProp* = enum         # properties of the C compiler:
     hasSwitchRange,           # CC allows ranges in switch statements (GNU C)
@@ -328,7 +330,9 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
                      platform.OS[conf.target.targetOS].name & '.' &
                      CC[c].name & fullSuffix
     result = getConfigVar(conf, fullCCname)
-    if result.len == 0:
+    if existsConfigVar(conf, fullCCname):
+      result = getConfigVar(conf, fullCCname)
+    else:
       # not overridden for this cross compilation setting?
       result = getConfigVar(conf, CC[c].name & fullSuffix)
   else:
@@ -528,7 +532,7 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool =
     var exe = getConfigVar(conf, conf.cCompiler, ".exe")
     if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
     # NOTE: should we need the full version, use -dumpfullversion
-    let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except: ("", 1)
+    let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except IOError, OSError, ValueError: ("", 1)
     if exitCode == 0:
       var major: int
       discard parseInt(s, major)
@@ -576,7 +580,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
 
     compilePattern = joinPath(conf.cCompilerPath, exe)
   else:
-    compilePattern = getCompilerExe(conf, c, isCpp)
+    compilePattern = exe
 
   includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)]))
 
@@ -830,10 +834,22 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) =
   else:
     writeFile(linkerArgs, args)
   try:
-    execLinkCmd(conf, cmd.substr(0, last) & " @" & linkerArgs)
+    when defined(macosx):
+      execLinkCmd(conf, "xargs " & cmd.substr(0, last) & " < " & linkerArgs)
+    else:
+      execLinkCmd(conf, cmd.substr(0, last) & " @" & linkerArgs)
   finally:
     removeFile(linkerArgs)
 
+proc linkViaShellScript(conf: ConfigRef; cmd: string) =
+  let linkerScript = conf.projectName & "_" & "linkerScript.sh"
+  writeFile(linkerScript, cmd)
+  let shell = getEnv("SHELL")
+  try:
+    execLinkCmd(conf, shell & " " & linkerScript)
+  finally:
+    removeFile(linkerScript)
+
 proc getObjFilePath(conf: ConfigRef, f: Cfile): string =
   if noAbsolutePaths(conf): f.obj.extractFilename
   else: f.obj.string
@@ -851,6 +867,20 @@ proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string =
     else:
       result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name)
 
+proc preventLinkCmdMaxCmdLen(conf: ConfigRef, linkCmd: string) =
+  # Prevent linkcmd from exceeding the maximum command line length.
+  # Windows's command line limit is about 8K (8191 characters) so C compilers on
+  # Windows support a feature where the command line can be passed via ``@linkcmd``
+  # to them.
+  const MaxCmdLen = when defined(windows): 8_000 elif defined(macosx): 260_000 else: 32_000
+  if linkCmd.len > MaxCmdLen:
+    when defined(macosx):
+      linkViaShellScript(conf, linkCmd)
+    else:
+      linkViaResponseFile(conf, linkCmd)
+  else:
+    execLinkCmd(conf, linkCmd)
+
 proc callCCompiler*(conf: ConfigRef) =
   var
     linkCmd: string
@@ -927,14 +957,7 @@ proc callCCompiler*(conf: ConfigRef) =
       linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true)
       extraCmds = getExtraCmds(conf, mainOutput)
       if optCompileOnly notin conf.globalOptions:
-        const MaxCmdLen = when defined(windows): 8_000 else: 32_000
-        if linkCmd.len > MaxCmdLen:
-          # Windows's command line limit is about 8K (don't laugh...) so C compilers on
-          # Windows support a feature where the command line can be passed via ``@linkcmd``
-          # to them.
-          linkViaResponseFile(conf, linkCmd)
-        else:
-          execLinkCmd(conf, linkCmd)
+        preventLinkCmdMaxCmdLen(conf, linkCmd)
         for cmd in extraCmds:
           execExternalProgram(conf, cmd, hintExecuting)
   else:
@@ -1018,7 +1041,7 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: Absolute
 proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) =
   var bcache: BuildCache
   try: bcache.fromJson(jsonFile.string.parseFile)
-  except:
+  except ValueError, KeyError, JsonKindError:
     let e = getCurrentException()
     conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" %
       [e.msg, e.getStackTrace(), jsonFile.string]
@@ -1035,7 +1058,7 @@ proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) =
     cmds.add cmd
     prettyCmds.add displayProgressCC(conf, name, cmd)
   execCmdsInParallel(conf, cmds, prettyCb)
-  execLinkCmd(conf, bcache.linkcmd)
+  preventLinkCmdMaxCmdLen(conf, bcache.linkcmd)
   for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting)
 
 proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope =

compiler/gorgeimpl.nim

@@ -9,12 +9,14 @@
 
 ## Module that implements ``gorge`` for the compiler.
 
-import msgs, std / sha1, os, osproc, streams, options,
+import msgs, os, osproc, streams, options,
   lineinfos, pathutils
 
 when defined(nimPreviewSlimSystem):
   import std/syncio
 
+import ../dist/checksums/src/checksums/sha1
+
 proc readOutput(p: Process): (string, int) =
   result[0] = ""
   var output = p.outputStream

compiler/ic/cbackend.nim

@@ -50,7 +50,10 @@ proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var Alive
     let n = unpackTree(g, m.module.position, m.fromDisk.topLevel, p)
     cgen.genTopLevelStmt(bmod, n)
 
-  finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info))
+  let disps = finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info))
+  if disps != nil:
+    for disp in disps:
+      genProcAux(bmod, disp.sym)
   m.fromDisk.backendFlags = cgen.whichInitProcs(bmod)
 
 proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) =

compiler/ic/ic.nim

@@ -7,13 +7,15 @@
 #    distribution, for details about the copyright.
 #
 
-import hashes, tables, intsets, std/sha1
+import hashes, tables, intsets
 import packed_ast, bitabs, rodfiles
 import ".." / [ast, idents, lineinfos, msgs, ropes, options,
   pathutils, condsyms, packages, modulepaths]
 #import ".." / [renderer, astalgo]
 from os import removeFile, isAbsolute
 
+import ../../dist/checksums/src/checksums/sha1
+
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions, formatfloat]
 
@@ -372,7 +374,7 @@ proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib
   if l.isNil: return
   result.kind = l.kind
   result.generated = l.generated
-  result.isOverriden = l.isOverriden
+  result.isOverridden = l.isOverridden
   result.name = toLitId($l.name, m)
   storeNode(result, l, path)
 
@@ -390,7 +392,7 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
     assert sfForward notin s.flags
 
     var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic,
-      position: s.position, offset: s.offset, options: s.options,
+      position: s.position, offset: s.offset, disamb: s.disamb, options: s.options,
       name: s.name.s.toLitId(m))
 
     storeNode(p, s, ast)
@@ -830,6 +832,7 @@ proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
     options: s.options,
     position: if s.kind in {skForVar, skVar, skLet, skTemp}: 0 else: s.position,
     offset: if s.kind in routineKinds: defaultOffset else: s.offset,
+    disamb: s.disamb,
     name: getIdent(c.cache, g[si].fromDisk.strings[s.name])
   )
 
@@ -847,7 +850,7 @@ proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph;
   if l.name.int == 0:
     result = nil
   else:
-    result = PLib(generated: l.generated, isOverriden: l.isOverriden,
+    result = PLib(generated: l.generated, isOverridden: l.isOverridden,
                   kind: l.kind, name: rope g[si].fromDisk.strings[l.name])
     loadAstBody(l, path)
 

compiler/ic/packed_ast.nim

@@ -45,7 +45,7 @@ type
   PackedLib* = object
     kind*: TLibKind
     generated*: bool
-    isOverriden*: bool
+    isOverridden*: bool
     name*: LitId
     path*: NodeId
 
@@ -63,7 +63,8 @@ type
     alignment*: int # for alignment
     options*: TOptions
     position*: int
-    offset*: int
+    offset*: int32
+    disamb*: int32
     externalName*: LitId # instead of TLoc
     locFlags*: TLocFlags
     annex*: PackedLib

compiler/importer.nim

@@ -228,10 +228,15 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; im
     for i in 0..n.safeLen-1:
       importForwarded(c, n[i], exceptSet, fromMod, importSet)
 
+proc addUnique[T](x: var seq[T], y: sink T) {.noSideEffect.} =
+  for i in 0..high(x):
+    if x[i] == y: return
+  x.add y
+    
 proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool): PSym =
   result = realModule
   template createModuleAliasImpl(ident): untyped =
-    createModuleAlias(realModule, nextSymId c.idgen, ident, n.info, c.config.options)
+    createModuleAlias(realModule, c.idgen, ident, n.info, c.config.options)
   if n.kind != nkImportAs: discard
   elif n.len != 2 or n[1].kind != nkIdent:
     localError(c.config, n.info, "module alias must be an identifier")
@@ -245,6 +250,7 @@ proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool)
     result.options.incl optImportHidden
   c.unusedImports.add((result, n.info))
   c.importModuleMap[result.id] = realModule.id
+  c.importModuleLookup.mgetOrPut(result.name.id, @[]).addUnique realModule.id
 
 proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] =
   var ret: typeof(result)
@@ -323,22 +329,24 @@ proc evalImport*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   for i in 0..<n.len:
     let it = n[i]
-    if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
-      let sep = it[0]
-      let dir = it[1]
-      var imp = newNodeI(nkInfix, it.info)
-      imp.add sep
-      imp.add dir
-      imp.add sep # dummy entry, replaced in the loop
-      for x in it[2]:
+    if it.kind in {nkInfix, nkPrefix} and it[^1].kind == nkBracket:
+      let lastPos = it.len - 1
+      var imp = copyNode(it)
+      newSons(imp, it.len)
+      for i in 0 ..< lastPos: imp[i] = it[i]
+      imp[lastPos] = imp[0] # dummy entry, replaced in the loop
+      for x in it[lastPos]:
         # transform `a/b/[c as d]` to `/a/b/c as d`
         if x.kind == nkInfix and x[0].ident.s == "as":
-          let impAs = copyTree(x)
-          imp[2] = x[1]
+          var impAs = copyNode(x)
+          newSons(impAs, 3)
+          impAs[0] = x[0]
+          imp[lastPos] = x[1]
           impAs[1] = imp
-          impMod(c, imp, result)
+          impAs[2] = x[2]
+          impMod(c, impAs, result)
         else:
-          imp[2] = x
+          imp[lastPos] = x
           impMod(c, imp, result)
     else:
       impMod(c, it, result)

compiler/injectdestructors.nim

@@ -17,12 +17,12 @@ import
   intsets, strtabs, ast, astalgo, msgs, renderer, magicsys, types, idents,
   strutils, options, lowerings, tables, modulegraphs,
   lineinfos, parampatterns, sighashes, liftdestructors, optimizer,
-  varpartitions, aliasanalysis, dfa
+  varpartitions, aliasanalysis, dfa, wordrecg
 
 when defined(nimPreviewSlimSystem):
   import std/assertions
 
-from trees import exprStructuralEquivalent, getRoot
+from trees import exprStructuralEquivalent, getRoot, whichPragma
 
 type
   Con = object
@@ -35,6 +35,8 @@ type
     idgen: IdGenerator
     body: PNode
     otherUsage: TLineInfo
+    inUncheckedAssignSection: int
+    inEnsureMove: int
 
   Scope = object # we do scope-based memory management.
     # a scope is comparable to an nkStmtListExpr like
@@ -65,11 +67,11 @@ proc hasDestructor(c: Con; t: PType): bool {.inline.} =
   result = ast.hasDestructor(t)
   when toDebug.len > 0:
     # for more effective debugging
-    if not result and c.graph.config.selectedGC in {gcArc, gcOrc}:
+    if not result and c.graph.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
       assert(not containsGarbageCollectedRef(t))
 
 proc getTemp(c: var Con; s: var Scope; typ: PType; info: TLineInfo): PNode =
-  let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), nextSymId c.idgen, c.owner, info)
+  let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), c.idgen, c.owner, info)
   sym.typ = typ
   s.vars.add(sym)
   result = newSymNode(sym)
@@ -185,9 +187,12 @@ template isUnpackedTuple(n: PNode): bool =
   ## hence unpacked tuples themselves don't need to be destroyed
   (n.kind == nkSym and n.sym.kind == skTemp and n.sym.typ.kind == tyTuple)
 
-proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
+proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string; inferredFromCopy = false) =
   var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">"
-  if (opname == "=" or opname == "=copy") and ri != nil:
+  if inferredFromCopy:
+    m.add ", which is inferred from unavailable '=copy'"
+
+  if (opname == "=" or opname == "=copy" or opname == "=dup") and ri != nil:
     m.add "; requires a copy because it's not the last read of '"
     m.add renderTree(ri)
     m.add '\''
@@ -209,15 +214,19 @@ proc makePtrType(c: var Con, baseType: PType): PType =
   addSonSkipIntLit(result, baseType, c.idgen)
 
 proc genOp(c: var Con; op: PSym; dest: PNode): PNode =
-  let addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
-  addrExp.add(dest)
+  var addrExp: PNode
+  if op.typ != nil and op.typ.len > 1 and op.typ[1].kind != tyVar:
+    addrExp = dest
+  else:
+    addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
+    addrExp.add(dest)
   result = newTree(nkCall, newSymNode(op), addrExp)
 
 proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode =
   var op = getAttachedOp(c.graph, t, kind)
   if op == nil or op.ast.isGenericRoutine:
     # give up and find the canonical type instead:
-    let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
+    let h = sighashes.hashType(t, c.graph.config, {CoType, CoConsiderOwned, CoDistinct})
     let canon = c.graph.canonTypes.getOrDefault(h)
     if canon != nil:
       op = getAttachedOp(c.graph, canon, kind)
@@ -248,7 +257,17 @@ proc canBeMoved(c: Con; t: PType): bool {.inline.} =
 proc isNoInit(dest: PNode): bool {.inline.} =
   result = dest.kind == nkSym and sfNoInit in dest.sym.flags
 
-proc genSink(c: var Con; dest, ri: PNode; flags: set[MoveOrCopyFlag] = {}): PNode =
+proc deepAliases(dest, ri: PNode): bool =
+  case ri.kind
+  of nkCallKinds, nkStmtListExpr, nkBracket, nkTupleConstr, nkObjConstr,
+     nkCast, nkConv, nkObjUpConv, nkObjDownConv:
+    for r in ri:
+      if deepAliases(dest, r): return true
+    return false
+  else:
+    return aliases(dest, ri) != no
+
+proc genSink(c: var Con; s: var Scope; dest, ri: PNode; flags: set[MoveOrCopyFlag] = {}): PNode =
   if (c.inLoopCond == 0 and (isUnpackedTuple(dest) or IsDecl in flags or
       (isAnalysableFieldAccess(dest, c.owner) and isFirstWrite(dest, c)))) or
       isNoInit(dest):
@@ -262,7 +281,14 @@ proc genSink(c: var Con; dest, ri: PNode; flags: set[MoveOrCopyFlag] = {}): PNod
     else:
       # the default is to use combination of `=destroy(dest)` and
       # and copyMem(dest, source). This is efficient.
-      result = newTree(nkStmtList, c.genDestroy(dest), newTree(nkFastAsgn, dest, ri))
+      if deepAliases(dest, ri):
+        # consider: x = x + y, it is wrong to destroy the destination first!
+        # tmp to support self assignments
+        let tmp = c.getTemp(s, dest.typ, dest.info)
+        result = newTree(nkStmtList, newTree(nkFastAsgn, tmp, dest), newTree(nkFastAsgn, dest, ri),
+                         c.genDestroy(tmp))
+      else:
+        result = newTree(nkStmtList, c.genDestroy(dest), newTree(nkFastAsgn, dest, ri))
 
 proc isCriticalLink(dest: PNode): bool {.inline.} =
   #[
@@ -287,13 +313,13 @@ proc isCriticalLink(dest: PNode): bool {.inline.} =
 proc finishCopy(c: var Con; result, dest: PNode; isFromSink: bool) =
   if c.graph.config.selectedGC == gcOrc:
     let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink, tyDistinct})
-    if cyclicType(t):
+    if cyclicType(c.graph, t):
       result.add boolLit(c.graph, result.info, isFromSink or isCriticalLink(dest))
 
 proc genMarkCyclic(c: var Con; result, dest: PNode) =
   if c.graph.config.selectedGC == gcOrc:
     let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink, tyDistinct})
-    if cyclicType(t):
+    if cyclicType(c.graph, t):
       if t.kind == tyRef:
         result.add callCodegenProc(c.graph, "nimMarkCyclic", dest.info, dest)
       else:
@@ -307,6 +333,9 @@ proc genCopyNoCheck(c: var Con; dest, ri: PNode; a: TTypeAttachedOp): PNode =
   assert ri.typ != nil
 
 proc genCopy(c: var Con; dest, ri: PNode; flags: set[MoveOrCopyFlag]): PNode =
+  if c.inEnsureMove > 0:
+    localError(c.graph.config, ri.info, errFailedMove, "cannot move '" & $ri &
+                                                      "', which introduces an implicit copy")
   let t = dest.typ
   if tfHasOwned in t.flags and ri.kind != nkNilLit:
     # try to improve the error message here:
@@ -335,30 +364,38 @@ proc genDiscriminantAsgn(c: var Con; s: var Scope; n: PNode): PNode =
 
   if hasDestructor(c, objType):
     if getAttachedOp(c.graph, objType, attachedDestructor) != nil and
-        sfOverriden in getAttachedOp(c.graph, objType, attachedDestructor).flags:
+        sfOverridden in getAttachedOp(c.graph, objType, attachedDestructor).flags:
       localError(c.graph.config, n.info, errGenerated, """Assignment to discriminant for objects with user defined destructor is not supported, object must have default destructor.
 It is best to factor out piece of object that needs custom destructor into separate object or not use discriminator assignment""")
       result.add newTree(nkFastAsgn, le, tmp)
       return
 
     # generate: if le != tmp: `=destroy`(le)
-    let branchDestructor = produceDestructorForDiscriminator(c.graph, objType, leDotExpr[1].sym, n.info, c.idgen)
-    let cond = newNodeIT(nkInfix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
-    cond.add newSymNode(getMagicEqSymForType(c.graph, le.typ, n.info))
-    cond.add le
-    cond.add tmp
-    let notExpr = newNodeIT(nkPrefix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
-    notExpr.add newSymNode(createMagic(c.graph, c.idgen, "not", mNot))
-    notExpr.add cond
-    result.add newTree(nkIfStmt, newTree(nkElifBranch, notExpr, c.genOp(branchDestructor, le)))
+    if c.inUncheckedAssignSection != 0:
+      let branchDestructor = produceDestructorForDiscriminator(c.graph, objType, leDotExpr[1].sym, n.info, c.idgen)
+      let cond = newNodeIT(nkInfix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
+      cond.add newSymNode(getMagicEqSymForType(c.graph, le.typ, n.info))
+      cond.add le
+      cond.add tmp
+      let notExpr = newNodeIT(nkPrefix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
+      notExpr.add newSymNode(createMagic(c.graph, c.idgen, "not", mNot))
+      notExpr.add cond
+      result.add newTree(nkIfStmt, newTree(nkElifBranch, notExpr, c.genOp(branchDestructor, le)))
   result.add newTree(nkFastAsgn, le, tmp)
 
 proc genWasMoved(c: var Con, n: PNode): PNode =
-  result = newNodeI(nkCall, n.info)
-  result.add(newSymNode(createMagic(c.graph, c.idgen, "wasMoved", mWasMoved)))
-  result.add copyTree(n) #mWasMoved does not take the address
-  #if n.kind != nkSym:
-  #  message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")")
+  let typ = n.typ.skipTypes({tyGenericInst, tyAlias, tySink})
+  let op = getAttachedOp(c.graph, n.typ, attachedWasMoved)
+  if op != nil:
+    if sfError in op.flags:
+      c.checkForErrorPragma(n.typ, n, "=wasMoved")
+    result = genOp(c, op, n)
+  else:
+    result = newNodeI(nkCall, n.info)
+    result.add(newSymNode(createMagic(c.graph, c.idgen, "`=wasMoved`", mWasMoved)))
+    result.add copyTree(n) #mWasMoved does not take the address
+    #if n.kind != nkSym:
+    #  message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")")
 
 proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode =
   result = newNodeI(nkCall, info)
@@ -367,13 +404,13 @@ proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode =
 
 proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode =
   # generate: (let tmp = v; reset(v); tmp)
-  if not hasDestructor(c, n.typ):
+  if (not hasDestructor(c, n.typ)) and c.inEnsureMove == 0:
     assert n.kind != nkSym or not hasDestructor(c, n.sym.typ)
     result = copyTree(n)
   else:
     result = newNodeIT(nkStmtListExpr, n.info, n.typ)
 
-    var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), nextSymId c.idgen, c.owner, n.info)
+    var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), c.idgen, c.owner, n.info)
     temp.typ = n.typ
     var v = newNodeI(nkLetSection, n.info)
     let tempAsNode = newSymNode(temp)
@@ -386,7 +423,8 @@ proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode =
 
     result.add v
     let nn = skipConv(n)
-    c.genMarkCyclic(result, nn)
+    if hasDestructor(c, n.typ):
+      c.genMarkCyclic(result, nn)
     let wasMovedCall = c.genWasMoved(nn)
     result.add wasMovedCall
     result.add tempAsNode
@@ -399,17 +437,41 @@ proc passCopyToSink(n: PNode; c: var Con; s: var Scope): PNode =
   result = newNodeIT(nkStmtListExpr, n.info, n.typ)
   let tmp = c.getTemp(s, n.typ, n.info)
   if hasDestructor(c, n.typ):
-    result.add c.genWasMoved(tmp)
-    var m = c.genCopy(tmp, n, {})
-    m.add p(n, c, s, normal)
-    c.finishCopy(m, n, isFromSink = true)
-    result.add m
+    let typ = n.typ.skipTypes({tyGenericInst, tyAlias, tySink})
+    let op = getAttachedOp(c.graph, typ, attachedDup)
+    if op != nil and tfHasOwned notin typ.flags:
+      if sfError in op.flags:
+        c.checkForErrorPragma(n.typ, n, "=dup")
+      else:
+        let copyOp = getAttachedOp(c.graph, typ, attachedAsgn)
+        if copyOp != nil and sfError in copyOp.flags and
+           sfOverridden notin op.flags:
+          c.checkForErrorPragma(n.typ, n, "=dup", inferredFromCopy = true)
+
+      let src = p(n, c, s, normal)
+      var newCall = newTreeIT(nkCall, src.info, src.typ,
+            newSymNode(op),
+            src)
+      c.finishCopy(newCall, n, isFromSink = true)
+      result.add newTreeI(nkFastAsgn,
+          src.info, tmp,
+          newCall
+      )
+    else:
+      result.add c.genWasMoved(tmp)
+      var m = c.genCopy(tmp, n, {})
+      m.add p(n, c, s, normal)
+      c.finishCopy(m, n, isFromSink = true)
+      result.add m
     if isLValue(n) and not isCapturedVar(n) and n.typ.skipTypes(abstractInst).kind != tyRef and c.inSpawn == 0:
       message(c.graph.config, n.info, hintPerformance,
         ("passing '$1' to a sink parameter introduces an implicit copy; " &
         "if possible, rearrange your program's control flow to prevent it") % $n)
+    if c.inEnsureMove > 0:
+      localError(c.graph.config, n.info, errFailedMove,
+        ("cannot move '$1', passing '$1' to a sink parameter introduces an implicit copy") % $n)
   else:
-    if c.graph.config.selectedGC in {gcArc, gcOrc}:
+    if c.graph.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
       assert(not containsManagedMemory(n.typ))
     if n.typ.skipTypes(abstractInst).kind in {tyOpenArray, tyVarargs}:
       localError(c.graph.config, n.info, "cannot create an implicit openArray copy to be passed to a sink parameter")
@@ -445,14 +507,14 @@ proc ensureDestruction(arg, orig: PNode; c: var Con; s: var Scope): PNode =
     # This was already done in the sink parameter handling logic.
     result = newNodeIT(nkStmtListExpr, arg.info, arg.typ)
     let tmp = c.getTemp(s, arg.typ, arg.info)
-    result.add c.genSink(tmp, arg, {IsDecl})
+    result.add c.genSink(s, tmp, arg, {IsDecl})
     result.add tmp
     s.final.add c.genDestroy(tmp)
   else:
     result = arg
 
 proc cycleCheck(n: PNode; c: var Con) =
-  if c.graph.config.selectedGC != gcArc: return
+  if c.graph.config.selectedGC notin {gcArc, gcAtomicArc}: return
   var value = n[1]
   if value.kind == nkClosure:
     value = value[1]
@@ -653,6 +715,24 @@ template handleNestedTempl(n, processCall: untyped, willProduceStmt = false,
   of nkWhen: # This should be a "when nimvm" node.
     result = copyTree(n)
     result[1][0] = processCall(n[1][0], s)
+
+  of nkPragmaBlock:
+    var inUncheckedAssignSection = 0
+    let pragmaList = n[0]
+    for pi in pragmaList:
+      if whichPragma(pi) == wCast:
+        case whichPragma(pi[1])
+        of wUncheckedAssign:
+          inUncheckedAssignSection = 1
+        else:
+          discard
+    result = shallowCopy(n)
+    inc c.inUncheckedAssignSection, inUncheckedAssignSection
+    for i in 0 ..< n.len-1:
+      result[i] = p(n[i], c, s, normal)
+    result[^1] = maybeVoid(n[^1], s)
+    dec c.inUncheckedAssignSection, inUncheckedAssignSection
+
   else: assert(false)
 
 proc pRaiseStmt(n: PNode, c: var Con; s: var Scope): PNode =
@@ -682,7 +762,7 @@ proc pRaiseStmt(n: PNode, c: var Con; s: var Scope): PNode =
 
 proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSingleUsedTemp}): PNode =
   if n.kind in {nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkIfStmt,
-                nkIfExpr, nkCaseStmt, nkWhen, nkWhileStmt, nkParForStmt, nkTryStmt}:
+                nkIfExpr, nkCaseStmt, nkWhen, nkWhileStmt, nkParForStmt, nkTryStmt, nkPragmaBlock}:
     template process(child, s): untyped = p(child, c, s, mode)
     handleNestedTempl(n, process, tmpFlags = tmpFlags)
   elif mode == sinkArg:
@@ -693,7 +773,15 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
       result = passCopyToSink(n, c, s)
     elif n.kind in {nkBracket, nkObjConstr, nkTupleConstr, nkClosure, nkNilLit} +
          nkCallKinds + nkLiterals:
-      result = p(n, c, s, consumed)
+      if n.kind in nkCallKinds and n[0].kind == nkSym:
+        if n[0].sym.magic == mEnsureMove:
+          inc c.inEnsureMove
+          result = p(n[1], c, s, sinkArg)
+          dec c.inEnsureMove
+        else:
+          result = p(n, c, s, consumed)
+      else:
+        result = p(n, c, s, consumed)
     elif ((n.kind == nkSym and isSinkParam(n.sym)) or isAnalysableFieldAccess(n, c.owner)) and
         isLastRead(n, c, s) and not (n.kind == nkSym and isCursor(n)):
       # Sinked params can be consumed only once. We need to reset the memory
@@ -765,6 +853,12 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
       if mode == normal and isRefConstr:
         result = ensureDestruction(result, n, c, s)
     of nkCallKinds:
+      if n[0].kind == nkSym and n[0].sym.magic == mEnsureMove:
+        inc c.inEnsureMove
+        result = p(n[1], c, s, sinkArg)
+        dec c.inEnsureMove
+        return
+
       let inSpawn = c.inSpawn
       if n[0].kind == nkSym and n[0].sym.magic == mSpawn:
         c.inSpawn.inc
@@ -795,7 +889,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
 
       if n[0].kind == nkSym and n[0].sym.magic in {mNew, mNewFinalize}:
         result[0] = copyTree(n[0])
-        if c.graph.config.selectedGC in {gcHooks, gcArc, gcOrc}:
+        if c.graph.config.selectedGC in {gcHooks, gcArc, gcAtomicArc, gcOrc}:
           let destroyOld = c.genDestroy(result[1])
           result = newTree(nkStmtList, destroyOld, result)
       else:
@@ -870,7 +964,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
        nkTypeOfExpr, nkMixinStmt, nkBindStmt:
       result = n
 
-    of nkStringToCString, nkCStringToString, nkChckRangeF, nkChckRange64, nkChckRange, nkPragmaBlock:
+    of nkStringToCString, nkCStringToString, nkChckRangeF, nkChckRange64, nkChckRange:
       result = shallowCopy(n)
       for i in 0 ..< n.len:
         result[i] = p(n[i], c, s, normal)
@@ -952,7 +1046,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
 
 proc sameLocation*(a, b: PNode): bool =
   proc sameConstant(a, b: PNode): bool =
-    a.kind in nkLiterals and a.intVal == b.intVal
+    a.kind in nkLiterals and b.kind in nkLiterals and a.intVal == b.intVal
 
   const nkEndPoint = {nkSym, nkDotExpr, nkCheckedFieldExpr, nkBracketExpr}
   if a.kind in nkEndPoint and b.kind in nkEndPoint:
@@ -976,9 +1070,9 @@ proc sameLocation*(a, b: PNode): bool =
     of nkHiddenStdConv, nkHiddenSubConv: sameLocation(a[1], b)
     else: false
 
-proc genFieldAccessSideEffects(c: var Con; dest, ri: PNode; flags: set[MoveOrCopyFlag] = {}): PNode =
+proc genFieldAccessSideEffects(c: var Con; s: var Scope; dest, ri: PNode; flags: set[MoveOrCopyFlag] = {}): PNode =
   # with side effects
-  var temp = newSym(skLet, getIdent(c.graph.cache, "bracketTmp"), nextSymId c.idgen, c.owner, ri[1].info)
+  var temp = newSym(skLet, getIdent(c.graph.cache, "bracketTmp"), c.idgen, c.owner, ri[1].info)
   temp.typ = ri[1].typ
   var v = newNodeI(nkLetSection, ri[1].info)
   let tempAsNode = newSymNode(temp)
@@ -993,14 +1087,22 @@ proc genFieldAccessSideEffects(c: var Con; dest, ri: PNode; flags: set[MoveOrCop
   newAccess.add ri[0]
   newAccess.add tempAsNode
 
-  var snk = c.genSink(dest, newAccess, flags)
+  var snk = c.genSink(s, dest, newAccess, flags)
   result = newTree(nkStmtList, v, snk, c.genWasMoved(newAccess))
 
 proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, flags: set[MoveOrCopyFlag] = {}): PNode =
+  var ri = ri
+  var isEnsureMove = 0
+  if ri.kind in nkCallKinds and ri[0].kind == nkSym and ri[0].sym.magic == mEnsureMove:
+    ri = ri[1]
+    isEnsureMove = 1
   if sameLocation(dest, ri):
     # rule (self-assignment-removal):
     result = newNodeI(nkEmpty, dest.info)
-  elif isCursor(dest):
+  elif isCursor(dest) or dest.typ.kind in {tyOpenArray, tyVarargs}:
+    # hoisted openArray parameters might end up here
+    # openArray types don't have a lifted assignment operation (it's empty)
+    # bug #22132
     case ri.kind:
     of nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt:
       template process(child, s): untyped = moveOrCopy(dest, child, c, s, flags)
@@ -1009,53 +1111,60 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, flags: set[MoveOrCopy
     else:
       result = newTree(nkFastAsgn, dest, p(ri, c, s, normal))
   else:
-    case ri.kind
+    let ri2 = if ri.kind == nkWhen: ri[1][0] else: ri
+    case ri2.kind
     of nkCallKinds:
-      result = c.genSink(dest, p(ri, c, s, consumed), flags)
+      result = c.genSink(s, dest, p(ri, c, s, consumed), flags)
     of nkBracketExpr:
       if isUnpackedTuple(ri[0]):
         # unpacking of tuple: take over the elements
-        result = c.genSink(dest, p(ri, c, s, consumed), flags)
+        result = c.genSink(s, dest, p(ri, c, s, consumed), flags)
       elif isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c, s):
         if aliases(dest, ri) == no:
           # Rule 3: `=sink`(x, z); wasMoved(z)
           if isAtom(ri[1]):
-            var snk = c.genSink(dest, ri, flags)
+            var snk = c.genSink(s, dest, ri, flags)
             result = newTree(nkStmtList, snk, c.genWasMoved(ri))
           else:
-            result = genFieldAccessSideEffects(c, dest, ri, flags)
+            result = genFieldAccessSideEffects(c, s, dest, ri, flags)
         else:
-          result = c.genSink(dest, destructiveMoveVar(ri, c, s), flags)
+          result = c.genSink(s, dest, destructiveMoveVar(ri, c, s), flags)
       else:
+        inc c.inEnsureMove, isEnsureMove
         result = c.genCopy(dest, ri, flags)
+        dec c.inEnsureMove, isEnsureMove
         result.add p(ri, c, s, consumed)
         c.finishCopy(result, dest, isFromSink = false)
     of nkBracket:
       # array constructor
       if ri.len > 0 and isDangerousSeq(ri.typ):
+        inc c.inEnsureMove, isEnsureMove
         result = c.genCopy(dest, ri, flags)
+        dec c.inEnsureMove, isEnsureMove
         result.add p(ri, c, s, consumed)
         c.finishCopy(result, dest, isFromSink = false)
       else:
-        result = c.genSink(dest, p(ri, c, s, consumed), flags)
+        result = c.genSink(s, dest, p(ri, c, s, consumed), flags)
     of nkObjConstr, nkTupleConstr, nkClosure, nkCharLit..nkNilLit:
-      result = c.genSink(dest, p(ri, c, s, consumed), flags)
+      result = c.genSink(s, dest, p(ri, c, s, consumed), flags)
     of nkSym:
       if isSinkParam(ri.sym) and isLastRead(ri, c, s):
         # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, flags)
+        let snk = c.genSink(s, dest, ri, flags)
         result = newTree(nkStmtList, snk, c.genWasMoved(ri))
       elif ri.sym.kind != skParam and ri.sym.owner == c.owner and
           isLastRead(ri, c, s) and canBeMoved(c, dest.typ) and not isCursor(ri):
         # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, flags)
+        let snk = c.genSink(s, dest, ri, flags)
         result = newTree(nkStmtList, snk, c.genWasMoved(ri))
       else:
+        inc c.inEnsureMove, isEnsureMove
         result = c.genCopy(dest, ri, flags)
+        dec c.inEnsureMove, isEnsureMove
         result.add p(ri, c, s, consumed)
         c.finishCopy(result, dest, isFromSink = false)
     of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv, nkCast:
-      result = c.genSink(dest, p(ri, c, s, sinkArg), flags)
+      result = c.genSink(s, dest, p(ri, c, s, sinkArg), flags)
     of nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt:
       template process(child, s): untyped = moveOrCopy(dest, child, c, s, flags)
       # We know the result will be a stmt so we use that fact to optimize
@@ -1066,10 +1175,12 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, flags: set[MoveOrCopy
       if isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c, s) and
           canBeMoved(c, dest.typ):
         # Rule 3: `=sink`(x, z); wasMoved(z)
-        let snk = c.genSink(dest, ri, flags)
+        let snk = c.genSink(s, dest, ri, flags)
         result = newTree(nkStmtList, snk, c.genWasMoved(ri))
       else:
+        inc c.inEnsureMove, isEnsureMove
         result = c.genCopy(dest, ri, flags)
+        dec c.inEnsureMove, isEnsureMove
         result.add p(ri, c, s, consumed)
         c.finishCopy(result, dest, isFromSink = false)
 

compiler/installer.ini

@@ -65,11 +65,12 @@ Files: "compiler"
 Files: "doc"
 Files: "doc/html"
 Files: "tools"
-Files: "tools/nim-gdb.py"
+Files: "tools/debug/nim-gdb.py"
 Files: "nimpretty"
 Files: "testament"
 Files: "nimsuggest"
 Files: "nimsuggest/tests/*.nim"
+Files: "changelogs/*.md"
 
 [Lib]
 Files: "lib"
@@ -77,6 +78,7 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/checksums"
 
 Files: "tests"
 
@@ -90,6 +92,7 @@ Files: "bin/nimgrab.exe"
 Files: "bin/nimpretty.exe"
 Files: "bin/testament.exe"
 Files: "bin/nim-gdb.bat"
+Files: "bin/atlas.exe"
 
 Files: "koch.exe"
 Files: "finish.exe"
@@ -120,6 +123,7 @@ Files: "bin/nim"
 InstallScript: "yes"
 UninstallScript: "yes"
 Files: "bin/nim-gdb"
+Files: "build_all.sh"
 
 
 [InnoSetup]

compiler/int128.nim

@@ -33,26 +33,27 @@ template high*(t: typedesc[Int128]): Int128 = Max
 proc `$`*(a: Int128): string
 
 proc toInt128*[T: SomeInteger | bool](arg: T): Int128 =
-  when T is bool: result.sdata(0) = int32(arg)
-  elif T is SomeUnsignedInt:
-    when sizeof(arg) <= 4:
-      result.udata[0] = uint32(arg)
+  {.noSideEffect.}:
+    when T is bool: result.sdata(0) = int32(arg)
+    elif T is SomeUnsignedInt:
+      when sizeof(arg) <= 4:
+        result.udata[0] = uint32(arg)
+      else:
+        result.udata[0] = uint32(arg and T(0xffffffff))
+        result.udata[1] = uint32(arg shr 32)
+    elif sizeof(arg) <= 4:
+      result.sdata(0) = int32(arg)
+      if arg < 0: # sign extend
+        result.sdata(1) = -1
+        result.sdata(2) = -1
+        result.sdata(3) = -1
     else:
-      result.udata[0] = uint32(arg and T(0xffffffff))
-      result.udata[1] = uint32(arg shr 32)
-  elif sizeof(arg) <= 4:
-    result.sdata(0) = int32(arg)
-    if arg < 0: # sign extend
-      result.sdata(1) = -1
-      result.sdata(2) = -1
-      result.sdata(3) = -1
-  else:
-    let tmp = int64(arg)
-    result.udata[0] = uint32(tmp and 0xffffffff)
-    result.sdata(1) = int32(tmp shr 32)
-    if arg < 0: # sign extend
-      result.sdata(2) = -1
-      result.sdata(3) = -1
+      let tmp = int64(arg)
+      result.udata[0] = uint32(tmp and 0xffffffff)
+      result.sdata(1) = int32(tmp shr 32)
+      if arg < 0: # sign extend
+        result.sdata(2) = -1
+        result.sdata(3) = -1
 
 template isNegative(arg: Int128): bool =
   arg.sdata(3) < 0

compiler/isolation_check.nim

@@ -72,6 +72,69 @@ proc isValueOnlyType(t: PType): bool =
   proc wrap(t: PType): bool {.nimcall.} = t.kind in {tyRef, tyPtr, tyVar, tyLent}
   result = not types.searchTypeFor(t, wrap)
 
+type
+  SearchResult = enum
+    NotFound, Abort, Found
+
+proc containsDangerousRefAux(t: PType; marker: var IntSet): SearchResult
+
+proc containsDangerousRefAux(n: PNode; marker: var IntSet): SearchResult =
+  result = NotFound
+  case n.kind
+  of nkRecList:
+    for i in 0..<n.len:
+      result = containsDangerousRefAux(n[i], marker)
+      if result == Found: return result
+  of nkRecCase:
+    assert(n[0].kind == nkSym)
+    result = containsDangerousRefAux(n[0], marker)
+    if result == Found: return result
+    for i in 1..<n.len:
+      case n[i].kind
+      of nkOfBranch, nkElse:
+        result = containsDangerousRefAux(lastSon(n[i]), marker)
+        if result == Found: return result
+      else: discard
+  of nkSym:
+    result = containsDangerousRefAux(n.sym.typ, marker)
+  else: discard
+
+proc containsDangerousRefAux(t: PType; marker: var IntSet): SearchResult =
+  result = NotFound
+  if t == nil: return result
+  if containsOrIncl(marker, t.id): return result
+
+  if t.kind == tyRef or (t.kind == tyProc and t.callConv == ccClosure):
+    result = Found
+  elif tfSendable in t.flags:
+    result = Abort
+  else:
+    # continue the type traversal:
+    result = NotFound
+
+  if result != NotFound: return result
+  case t.kind
+  of tyObject:
+    if t[0] != nil:
+      result = containsDangerousRefAux(t[0].skipTypes(skipPtrs), marker)
+    if result == NotFound: result = containsDangerousRefAux(t.n, marker)
+  of tyGenericInst, tyDistinct, tyAlias, tySink:
+    result = containsDangerousRefAux(lastSon(t), marker)
+  of tyArray, tySet, tyTuple, tySequence:
+    for i in 0..<t.len:
+      result = containsDangerousRefAux(t[i], marker)
+      if result == Found: return result
+  else:
+    discard
+
+proc containsDangerousRef(t: PType): bool =
+  # a `ref` type is "dangerous" if it occurs not within a type that is like `Isolated[T]`.
+  # For example:
+  # `ref int` # dangerous
+  # `Isolated[ref int]` # not dangerous
+  var marker = initIntSet()
+  result = containsDangerousRefAux(t, marker) == Found
+
 proc canAlias*(arg, ret: PType): bool =
   if isValueOnlyType(arg):
     # can alias only with addr(arg.x) and we don't care if it is not safe
@@ -80,12 +143,15 @@ proc canAlias*(arg, ret: PType): bool =
     var marker = initIntSet()
     result = canAlias(arg, ret, marker)
 
+const
+  SomeVar = {skForVar, skParam, skVar, skLet, skConst, skResult, skTemp}
+
 proc containsVariable(n: PNode): bool =
   case n.kind
   of nodesToIgnoreSet:
     result = false
   of nkSym:
-    result = n.sym.kind in {skForVar, skParam, skVar, skLet, skConst, skResult, skTemp}
+    result = n.sym.kind in SomeVar
   else:
     for ch in n:
       if containsVariable(ch): return true
@@ -109,7 +175,7 @@ proc checkIsolate*(n: PNode): bool =
           discard "fine, it is isolated already"
         else:
           let argType = n[i].typ
-          if argType != nil and not isCompileTimeOnly(argType) and containsTyRef(argType):
+          if argType != nil and not isCompileTimeOnly(argType) and containsDangerousRef(argType):
             if argType.canAlias(n.typ) or containsVariable(n[i]):
               # bug #19013: Alias information is not enough, we need to check for potential
               # "overlaps". I claim the problem can only happen by reading again from a location
@@ -143,6 +209,12 @@ proc checkIsolate*(n: PNode): bool =
         result = checkIsolate(n[^1])
       else:
         result = false
+    of nkSym:
+      result = true
+      if n.sym.kind in SomeVar:
+        let argType = n.typ
+        if argType != nil and not isCompileTimeOnly(argType) and containsDangerousRef(argType):
+          result = false
     else:
       # unanalysable expression:
       result = false

compiler/jsgen.nim

@@ -31,9 +31,11 @@ implements the required case distinction.
 import
   ast, trees, magicsys, options,
   nversion, msgs, idents, types,
-  ropes, passes, ccgutils, wordrecg, renderer,
+  ropes, ccgutils, wordrecg, renderer,
   cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils,
-  transf, injectdestructors, sourcemap, astmsgs
+  transf, injectdestructors, sourcemap, astmsgs, backendpragmas
+
+import pipelineutils
 
 import json, sets, math, tables, intsets
 import strutils except addf
@@ -98,6 +100,7 @@ type
     prc: PSym
     globals, locals, body: Rope
     options: TOptions
+    optionsStack: seq[TOptions]
     module: BModule
     g: PGlobals
     generatedParamCopies: IntSet
@@ -107,6 +110,7 @@ type
     extraIndent: int
     up: PProc     # up the call chain; required for closure support
     declaredGlobals: IntSet
+    previousFileName: string  # For frameInfo inside templates.
 
 template config*(p: PProc): ConfigRef = p.module.config
 
@@ -261,7 +265,7 @@ proc mangleName(m: BModule, s: PSym): Rope =
       if m.config.hcrOn:
         # When hot reloading is enabled, we must ensure that the names
         # of functions and types will be preserved across rebuilds:
-        result.add(idOrSig(s, m.module.name.s, m.sigConflicts))
+        result.add(idOrSig(s, m.module.name.s, m.sigConflicts, m.config))
       else:
         result.add("_")
         result.add(rope(s.id))
@@ -552,28 +556,36 @@ template binaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string,
   r.res = frmt % [a, b, tmp, tmp2]
   r.kind = resExpr
 
-proc unsignedTrimmerJS(size: BiggestInt): Rope =
+proc unsignedTrimmer(size: BiggestInt): string =
   case size
-  of 1: rope"& 0xff"
-  of 2: rope"& 0xffff"
-  of 4: rope">>> 0"
-  else: rope""
+  of 1: "& 0xff"
+  of 2: "& 0xffff"
+  of 4: ">>> 0"
+  else: ""
 
-
-template unsignedTrimmer(size: BiggestInt): Rope =
-  size.unsignedTrimmerJS
+proc signedTrimmer(size: BiggestInt): string =
+  # sign extension is done by shifting to the left and then back to the right
+  "<< $1 >> $1" % [$(32 - size * 8)]
 
 proc binaryUintExpr(p: PProc, n: PNode, r: var TCompRes, op: string,
                     reassign: static[bool] = false) =
   var x, y: TCompRes
   gen(p, n[1], x)
   gen(p, n[2], y)
-  let trimmer = unsignedTrimmer(n[1].typ.skipTypes(abstractRange).size)
+  let size = n[1].typ.skipTypes(abstractRange).size
   when reassign:
     let (a, tmp) = maybeMakeTempAssignable(p, n[1], x)
-    r.res = "$1 = (($5 $2 $3) $4)" % [a, rope op, y.rdLoc, trimmer, tmp]
+    if size == 8 and optJsBigInt64 in p.config.globalOptions:
+      r.res = "$1 = BigInt.asUintN(64, ($4 $2 $3))" % [a, rope op, y.rdLoc, tmp]
+    else:
+      let trimmer = unsignedTrimmer(size)
+      r.res = "$1 = (($5 $2 $3) $4)" % [a, rope op, y.rdLoc, trimmer, tmp]
   else:
-    r.res = "(($1 $2 $3) $4)" % [x.rdLoc, rope op, y.rdLoc, trimmer]
+    if size == 8 and optJsBigInt64 in p.config.globalOptions:
+      r.res = "BigInt.asUintN(64, ($1 $2 $3))" % [x.rdLoc, rope op, y.rdLoc]
+    else:
+      let trimmer = unsignedTrimmer(size)
+      r.res = "(($1 $2 $3) $4)" % [x.rdLoc, rope op, y.rdLoc, trimmer]
   r.kind = resExpr
 
 template ternaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
@@ -614,32 +626,103 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   template applyFormat(frmtA, frmtB) =
     if i == 0: applyFormat(frmtA) else: applyFormat(frmtB)
 
+  template bitwiseExpr(op: string) =
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    if typ.kind in {tyUInt, tyUInt32}:
+      r.res = "(($1 $2 $3) >>> 0)" % [xLoc, op, yLoc]
+    else:
+      r.res = "($1 $2 $3)" % [xLoc, op, yLoc]
+
   case op
-  of mAddI: applyFormat("addInt($1, $2)", "($1 + $2)")
-  of mSubI: applyFormat("subInt($1, $2)", "($1 - $2)")
-  of mMulI: applyFormat("mulInt($1, $2)", "($1 * $2)")
-  of mDivI: applyFormat("divInt($1, $2)", "Math.trunc($1 / $2)")
-  of mModI: applyFormat("modInt($1, $2)", "Math.trunc($1 % $2)")
+  of mAddI:
+    if i == 0:
+      if n[1].typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+        useMagic(p, "addInt64")
+        applyFormat("addInt64($1, $2)")
+      else:
+        applyFormat("addInt($1, $2)")
+    else:
+      applyFormat("($1 + $2)")
+  of mSubI:
+    if i == 0:
+      if n[1].typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+        useMagic(p, "subInt64")
+        applyFormat("subInt64($1, $2)")
+      else:
+        applyFormat("subInt($1, $2)")
+    else:
+      applyFormat("($1 - $2)")
+  of mMulI:
+    if i == 0:
+      if n[1].typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+        useMagic(p, "mulInt64")
+        applyFormat("mulInt64($1, $2)")
+      else:
+        applyFormat("mulInt($1, $2)")
+    else:
+      applyFormat("($1 * $2)")
+  of mDivI:
+    if n[1].typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+      useMagic(p, "divInt64")
+      applyFormat("divInt64($1, $2)", "$1 / $2")
+    else:
+      applyFormat("divInt($1, $2)", "Math.trunc($1 / $2)")
+  of mModI:
+    if n[1].typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+      useMagic(p, "modInt64")
+      applyFormat("modInt64($1, $2)", "$1 % $2")
+    else:
+      applyFormat("modInt($1, $2)", "Math.trunc($1 % $2)")
   of mSucc: applyFormat("addInt($1, $2)", "($1 + $2)")
   of mPred: applyFormat("subInt($1, $2)", "($1 - $2)")
   of mAddF64: applyFormat("($1 + $2)", "($1 + $2)")
   of mSubF64: applyFormat("($1 - $2)", "($1 - $2)")
   of mMulF64: applyFormat("($1 * $2)", "($1 * $2)")
   of mDivF64: applyFormat("($1 / $2)", "($1 / $2)")
-  of mShrI: applyFormat("", "")
+  of mShrI:
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    if typ.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+      applyFormat("BigInt.asIntN(64, BigInt.asUintN(64, $1) >> BigInt($2))")
+    elif typ.kind == tyUInt64 and optJsBigInt64 in p.config.globalOptions:
+      applyFormat("($1 >> BigInt($2))")
+    else:
+      if typ.kind in {tyInt..tyInt32}:
+        let trimmerU = unsignedTrimmer(typ.size)
+        let trimmerS = signedTrimmer(typ.size)
+        r.res = "((($1 $2) >>> $3) $4)" % [xLoc, trimmerU, yLoc, trimmerS]
+      else:
+        applyFormat("($1 >>> $2)")
   of mShlI:
-    if n[1].typ.size <= 4:
-      applyFormat("($1 << $2)", "($1 << $2)")
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    if typ.size == 8:
+      if typ.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+        applyFormat("BigInt.asIntN(64, $1 << BigInt($2))")
+      elif typ.kind == tyUInt64 and optJsBigInt64 in p.config.globalOptions:
+        applyFormat("BigInt.asUintN(64, $1 << BigInt($2))")
+      else:
+        applyFormat("($1 * Math.pow(2, $2))")
     else:
-      applyFormat("($1 * Math.pow(2, $2))", "($1 * Math.pow(2, $2))")
+      if typ.kind in {tyUInt..tyUInt32}:
+        let trimmer = unsignedTrimmer(typ.size)
+        r.res = "(($1 << $2) $3)" % [xLoc, yLoc, trimmer]
+      else:
+        let trimmer = signedTrimmer(typ.size)
+        r.res = "(($1 << $2) $3)" % [xLoc, yLoc, trimmer]
   of mAshrI:
-    if n[1].typ.size <= 4:
-      applyFormat("($1 >> $2)", "($1 >> $2)")
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    if typ.size == 8:
+      if optJsBigInt64 in p.config.globalOptions:
+        applyFormat("($1 >> BigInt($2))")
+      else:
+        applyFormat("Math.floor($1 / Math.pow(2, $2))")
     else:
-      applyFormat("Math.floor($1 / Math.pow(2, $2))", "Math.floor($1 / Math.pow(2, $2))")
-  of mBitandI: applyFormat("($1 & $2)", "($1 & $2)")
-  of mBitorI: applyFormat("($1 | $2)", "($1 | $2)")
-  of mBitxorI: applyFormat("($1 ^ $2)", "($1 ^ $2)")
+      if typ.kind in {tyUInt..tyUInt32}:
+        applyFormat("($1 >>> $2)")
+      else:
+        applyFormat("($1 >> $2)")
+  of mBitandI: bitwiseExpr("&")
+  of mBitorI: bitwiseExpr("|")
+  of mBitxorI: bitwiseExpr("^")
   of mMinI: applyFormat("nimMin($1, $2)", "nimMin($1, $2)")
   of mMaxI: applyFormat("nimMax($1, $2)", "nimMax($1, $2)")
   of mAddU: applyFormat("", "")
@@ -675,7 +758,16 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mAbsI: applyFormat("absInt($1)", "Math.abs($1)")
   of mNot: applyFormat("!($1)", "!($1)")
   of mUnaryPlusI: applyFormat("+($1)", "+($1)")
-  of mBitnotI: applyFormat("~($1)", "~($1)")
+  of mBitnotI:
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    if typ.kind in {tyUInt..tyUInt64}:
+      if typ.size == 8 and optJsBigInt64 in p.config.globalOptions:
+        applyFormat("BigInt.asUintN(64, ~($1))")
+      else:
+        let trimmer = unsignedTrimmer(typ.size)
+        r.res = "(~($1) $2)" % [xLoc, trimmer]
+    else:
+      applyFormat("~($1)")
   of mUnaryPlusF64: applyFormat("+($1)", "+($1)")
   of mUnaryMinusF64: applyFormat("-($1)", "-($1)")
   of mCharToStr: applyFormat("nimCharToStr($1)", "nimCharToStr($1)")
@@ -694,17 +786,14 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mMulU: binaryUintExpr(p, n, r, "*")
   of mDivU:
     binaryUintExpr(p, n, r, "/")
-    if n[1].typ.skipTypes(abstractRange).size == 8:
+    if optJsBigInt64 notin p.config.globalOptions and
+        n[1].typ.skipTypes(abstractRange).size == 8:
+      # bigint / already truncates
       r.res = "Math.trunc($1)" % [r.res]
   of mDivI:
     arithAux(p, n, r, op)
   of mModI:
     arithAux(p, n, r, op)
-  of mShrI:
-    var x, y: TCompRes
-    gen(p, n[1], x)
-    gen(p, n[2], y)
-    r.res = "($1 >>> $2)" % [x.rdLoc, y.rdLoc]
   of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr, mStrToStr, mEnumToStr:
     arithAux(p, n, r, op)
   of mEqRef:
@@ -732,10 +821,16 @@ proc genLineDir(p: PProc, n: PNode) =
   let line = toLinenumber(n.info)
   if line < 0:
     return
+  if optEmbedOrigSrc in p.config.globalOptions:
+    lineF(p, "//$1$n", [sourceLine(p.config, n.info)])
   if optLineDir in p.options or optLineDir in p.config.options:
     lineF(p, "$1", [lineDir(p.config, n.info, line)])
   if hasFrameInfo(p):
     lineF(p, "F.line = $1;$n", [rope(line)])
+    let currentFileName = toFilename(p.config, n.info)
+    if p.previousFileName != currentFileName:
+      lineF(p, "F.filename = $1;$n", [makeJSString(currentFileName)])
+      p.previousFileName = currentFileName
 
 proc genWhileStmt(p: PProc, n: PNode) =
   var cond: TCompRes
@@ -1759,15 +1854,25 @@ proc createObjInitList(p: PProc, typ: PType, excludedFieldIDs: IntSet, output: v
     createRecordVarAux(p, t.n, excludedFieldIDs, output)
     t = t[0]
 
-proc arrayTypeForElemType(typ: PType): string =
+proc arrayTypeForElemType(conf: ConfigRef; typ: PType): string =
   let typ = typ.skipTypes(abstractRange)
   case typ.kind
   of tyInt, tyInt32: "Int32Array"
   of tyInt16: "Int16Array"
   of tyInt8: "Int8Array"
+  of tyInt64:
+    if optJsBigInt64 in conf.globalOptions:
+      "BigInt64Array"
+    else:
+      ""
   of tyUInt, tyUInt32: "Uint32Array"
   of tyUInt16: "Uint16Array"
   of tyUInt8, tyChar, tyBool: "Uint8Array"
+  of tyUInt64:
+    if optJsBigInt64 in conf.globalOptions:
+      "BigUint64Array"
+    else:
+      ""
   of tyFloat32: "Float32Array"
   of tyFloat64, tyFloat: "Float64Array"
   of tyEnum:
@@ -1781,11 +1886,18 @@ proc arrayTypeForElemType(typ: PType): string =
 proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   var t = skipTypes(typ, abstractInst)
   case t.kind
-  of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar:
+  of tyInt8..tyInt32, tyUInt8..tyUInt32, tyEnum, tyChar:
+    result = putToSeq("0", indirect)
+  of tyInt, tyUInt:
     if $t.sym.loc.r == "bigint":
       result = putToSeq("0n", indirect)
     else:
       result = putToSeq("0", indirect)
+  of tyInt64, tyUInt64:
+    if optJsBigInt64 in p.config.globalOptions:
+      result = putToSeq("0n", indirect)
+    else:
+      result = putToSeq("0", indirect)
   of tyFloat..tyFloat128:
     result = putToSeq("0.0", indirect)
   of tyRange, tyGenericInst, tyAlias, tySink, tyOwned, tyLent:
@@ -1799,7 +1911,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   of tyArray:
     let length = toInt(lengthOrd(p.config, t))
     let e = elemType(t)
-    let jsTyp = arrayTypeForElemType(e)
+    let jsTyp = arrayTypeForElemType(p.config, e)
     if jsTyp.len > 0:
       result = "new $1($2)" % [rope(jsTyp), rope(length)]
     elif length > 32:
@@ -1974,7 +2086,11 @@ proc genNewSeq(p: PProc, n: PNode) =
 
 proc genOrd(p: PProc, n: PNode, r: var TCompRes) =
   case skipTypes(n[1].typ, abstractVar + abstractRange).kind
-  of tyEnum, tyInt..tyUInt64, tyChar: gen(p, n[1], r)
+  of tyEnum, tyInt..tyInt32, tyUInt..tyUInt32, tyChar: gen(p, n[1], r)
+  of tyInt64, tyUInt64:
+    if optJsBigInt64 in p.config.globalOptions:
+      unaryExpr(p, n, r, "", "Number($1)")
+    else: gen(p, n[1], r)
   of tyBool: unaryExpr(p, n, r, "", "($1 ? 1 : 0)")
   else: internalError(p.config, n.info, "genOrd")
 
@@ -2086,6 +2202,13 @@ proc genMove(p: PProc; n: PNode; r: var TCompRes) =
   genReset(p, n)
   #lineF(p, "$1 = $2;$n", [dest.rdLoc, src.rdLoc])
 
+proc genDup(p: PProc; n: PNode; r: var TCompRes) =
+  var a: TCompRes
+  r.kind = resVal
+  r.res = p.getTemp()
+  gen(p, n[1], a)
+  lineF(p, "$1 = $2;$n", [r.rdLoc, a.rdLoc])
+
 proc genJSArrayConstr(p: PProc, n: PNode, r: var TCompRes) =
   var a: TCompRes
   r.res = rope("[")
@@ -2197,14 +2320,34 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
       r.res = "($1).length - 1" % [x.rdLoc]
     r.kind = resExpr
   of mInc:
-    if n[1].typ.skipTypes(abstractRange).kind in {tyUInt..tyUInt64}:
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    case typ.kind
+    of tyUInt..tyUInt32:
       binaryUintExpr(p, n, r, "+", true)
+    of tyUInt64:
+      if optJsBigInt64 in p.config.globalOptions:
+        binaryExpr(p, n, r, "", "$1 = BigInt.asUintN(64, $3 + BigInt($2))", true)
+      else: binaryUintExpr(p, n, r, "+", true)
+    elif typ.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+      if optOverflowCheck notin p.options:
+        binaryExpr(p, n, r, "", "$1 = BigInt.asIntN(64, $3 + BigInt($2))", true)
+      else: binaryExpr(p, n, r, "addInt64", "$1 = addInt64($3, BigInt($2))", true)
     else:
       if optOverflowCheck notin p.options: binaryExpr(p, n, r, "", "$1 += $2")
       else: binaryExpr(p, n, r, "addInt", "$1 = addInt($3, $2)", true)
   of ast.mDec:
-    if n[1].typ.skipTypes(abstractRange).kind in {tyUInt..tyUInt64}:
+    let typ = n[1].typ.skipTypes(abstractVarRange)
+    case typ.kind
+    of tyUInt..tyUInt32:
       binaryUintExpr(p, n, r, "-", true)
+    of tyUInt64:
+      if optJsBigInt64 in p.config.globalOptions:
+        binaryExpr(p, n, r, "", "$1 = BigInt.asUintN(64, $3 - BigInt($2))", true)
+      else: binaryUintExpr(p, n, r, "+", true)
+    elif typ.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+      if optOverflowCheck notin p.options:
+        binaryExpr(p, n, r, "", "$1 = BigInt.asIntN(64, $3 - BigInt($2))", true)
+      else: binaryExpr(p, n, r, "subInt64", "$1 = subInt64($3, BigInt($2))", true)
     else:
       if optOverflowCheck notin p.options: binaryExpr(p, n, r, "", "$1 -= $2")
       else: binaryExpr(p, n, r, "subInt", "$1 = subInt($3, $2)", true)
@@ -2260,6 +2403,10 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
     r.kind = resExpr
   of mMove:
     genMove(p, n, r)
+  of mDup:
+    genDup(p, n, r)
+  of mEnsureMove:
+    gen(p, n[1], r)
   else:
     genCall(p, n, r)
     #else internalError(p.config, e.info, 'genMagic: ' + magicToStr[op]);
@@ -2298,7 +2445,7 @@ proc genArrayConstr(p: PProc, n: PNode, r: var TCompRes) =
   ## Nim sequence maps to JS array.
   var t = skipTypes(n.typ, abstractInst)
   let e = elemType(t)
-  let jsTyp = arrayTypeForElemType(e)
+  let jsTyp = arrayTypeForElemType(p.config, e)
   if skipTypes(n.typ, abstractVarRange).kind != tySequence and jsTyp.len > 0:
     # generate typed array
     # for example Nim generates `new Uint8Array([1, 2, 3])` for `[byte(1), 2, 3]`
@@ -2379,7 +2526,29 @@ proc genConv(p: PProc, n: PNode, r: var TCompRes) =
     r.res = "(!!($1))" % [r.res]
     r.kind = resExpr
   elif toInt:
-    r.res = "(($1) | 0)" % [r.res]
+    if src.kind in {tyInt64, tyUInt64} and optJsBigInt64 in p.config.globalOptions:
+      r.res = "Number($1)" % [r.res]
+    else:
+      r.res = "(($1) | 0)" % [r.res]
+  elif dest.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+    if fromInt or fromUint or src.kind in {tyBool, tyChar, tyEnum}:
+      r.res = "BigInt($1)" % [r.res]
+    elif src.kind in {tyFloat..tyFloat64}:
+      r.res = "BigInt(Math.trunc($1))" % [r.res]
+    elif src.kind == tyUInt64:
+      r.res = "BigInt.asIntN(64, $1)" % [r.res]
+  elif dest.kind == tyUInt64 and optJsBigInt64 in p.config.globalOptions:
+    if fromUint or src.kind in {tyBool, tyChar, tyEnum}:
+      r.res = "BigInt($1)" % [r.res]
+    elif fromInt: # could be negative
+      r.res = "BigInt.asUintN(64, BigInt($1))" % [r.res]
+    elif src.kind in {tyFloat..tyFloat64}:
+      r.res = "BigInt.asUintN(64, BigInt(Math.trunc($1)))" % [r.res]
+    elif src.kind == tyInt64:
+      r.res = "BigInt.asUintN(64, $1)" % [r.res]
+  elif toUint or dest.kind in tyFloat..tyFloat64:
+    if src.kind in {tyInt64, tyUInt64} and optJsBigInt64 in p.config.globalOptions:
+      r.res = "Number($1)" % [r.res]
   else:
     # TODO: What types must we handle here?
     discard
@@ -2390,7 +2559,11 @@ proc upConv(p: PProc, n: PNode, r: var TCompRes) =
 proc genRangeChck(p: PProc, n: PNode, r: var TCompRes, magic: string) =
   var a, b: TCompRes
   gen(p, n[0], r)
-  if optRangeCheck notin p.options or (skipTypes(n.typ, abstractVar).kind in {tyUInt..tyUInt64} and
+  let src = skipTypes(n[0].typ, abstractVarRange)
+  let dest = skipTypes(n.typ, abstractVarRange)
+  if src.kind in {tyInt64, tyUInt64} and dest.kind notin {tyInt64, tyUInt64} and optJsBigInt64 in p.config.globalOptions:
+    r.res = "Number($1)" % [r.res]
+  if optRangeCheck notin p.options or (dest.kind in {tyUInt..tyUInt64} and
       checkUnsignedConversions notin p.config.legacyFeatures):
     discard "XXX maybe emit masking instructions here"
   else:
@@ -2484,17 +2657,18 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
   if prc.typ[0] != nil and sfPure notin prc.flags:
     resultSym = prc.ast[resultPos].sym
     let mname = mangleName(p.module, resultSym)
-    let returnAddress = not isIndirect(resultSym) and
+    # otherwise uses "fat pointers"
+    let useRawPointer = not isIndirect(resultSym) and
       resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef, tyOwned} and
         mapType(p, resultSym.typ) == etyBaseIndex
-    if returnAddress:
+    if useRawPointer:
       resultAsgn = p.indentLine(("var $# = null;$n") % [mname])
       resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname])
     else:
       let resVar = createVar(p, resultSym.typ, isIndirect(resultSym))
       resultAsgn = p.indentLine(("var $# = $#;$n") % [mname, resVar])
     gen(p, prc.ast[resultPos], a)
-    if returnAddress:
+    if mapType(p, resultSym.typ) == etyBaseIndex:
       returnStmt = "return [$#, $#];$n" % [a.address, a.res]
     else:
       returnStmt = "return $#;$n" % [a.res]
@@ -2556,9 +2730,14 @@ proc genStmt(p: PProc, n: PNode) =
   if r.res != "": lineF(p, "$#;$n", [r.res])
 
 proc genPragma(p: PProc, n: PNode) =
-  for it in n.sons:
+  for i in 0..<n.len:
+    let it = n[i]
     case whichPragma(it)
     of wEmit: genAsmOrEmitStmt(p, it[1])
+    of wPush:
+      processPushBackendOption(p.optionsStack, p.options, n, i+1)
+    of wPop:
+      processPopBackendOption(p.optionsStack, p.options)
     else: discard
 
 proc genCast(p: PProc, n: PNode, r: var TCompRes) =
@@ -2573,24 +2752,35 @@ proc genCast(p: PProc, n: PNode, r: var TCompRes) =
   let fromInt = (src.kind in tyInt..tyInt32)
   let fromUint = (src.kind in tyUInt..tyUInt32)
 
-  if toUint and (fromInt or fromUint):
-    let trimmer = unsignedTrimmer(dest.size)
-    r.res = "($1 $2)" % [r.res, trimmer]
+  if toUint:
+    if fromInt or fromUint:
+      r.res = "Number(BigInt.asUintN($1, BigInt($2)))" % [$(dest.size * 8), r.res]
+    elif src.kind in {tyInt64, tyUInt64} and optJsBigInt64 in p.config.globalOptions:
+      r.res = "Number(BigInt.asUintN($1, $2))" % [$(dest.size * 8), r.res]
   elif toInt:
-    if fromInt:
-      return
-    elif fromUint:
-      if src.size == 4 and dest.size == 4:
-        # XXX prevent multi evaluations
-        r.res = "($1 | 0)" % [r.res]
-      else:
-        let trimmer = unsignedTrimmer(dest.size)
-        let minuend = case dest.size
-          of 1: "0xfe"
-          of 2: "0xfffe"
-          of 4: "0xfffffffe"
-          else: ""
-        r.res = "($1 - ($2 $3))" % [rope minuend, r.res, trimmer]
+    if fromInt or fromUint:
+      r.res = "Number(BigInt.asIntN($1, BigInt($2)))" % [$(dest.size * 8), r.res]
+    elif src.kind in {tyInt64, tyUInt64} and optJsBigInt64 in p.config.globalOptions:
+      r.res = "Number(BigInt.asIntN($1, $2))" % [$(dest.size * 8), r.res]
+  elif dest.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions:
+    if fromInt or fromUint or src.kind in {tyBool, tyChar, tyEnum}:
+      r.res = "BigInt($1)" % [r.res]
+    elif src.kind in {tyFloat..tyFloat64}:
+      r.res = "BigInt(Math.trunc($1))" % [r.res]
+    elif src.kind == tyUInt64:
+      r.res = "BigInt.asIntN(64, $1)" % [r.res]
+  elif dest.kind == tyUInt64 and optJsBigInt64 in p.config.globalOptions:
+    if fromUint or src.kind in {tyBool, tyChar, tyEnum}:
+      r.res = "BigInt($1)" % [r.res]
+    elif fromInt: # could be negative
+      r.res = "BigInt.asUintN(64, BigInt($1))" % [r.res]
+    elif src.kind in {tyFloat..tyFloat64}:
+      r.res = "BigInt.asUintN(64, BigInt(Math.trunc($1)))" % [r.res]
+    elif src.kind == tyInt64:
+      r.res = "BigInt.asUintN(64, $1)" % [r.res]
+  elif dest.kind in tyFloat..tyFloat64:
+    if src.kind in {tyInt64, tyUInt64} and optJsBigInt64 in p.config.globalOptions:
+      r.res = "Number($1)" % [r.res]
   elif (src.kind == tyPtr and mapType(p, src) == etyObject) and dest.kind == tyPointer:
     r.address = r.res
     r.res = "null"
@@ -2609,10 +2799,25 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkSym:
     genSym(p, n, r)
   of nkCharLit..nkUInt64Lit:
-    if n.typ.kind == tyBool:
+    case n.typ.skipTypes(abstractVarRange).kind
+    of tyBool:
       r.res = if n.intVal == 0: rope"false" else: rope"true"
+    of tyUInt64:
+      r.res = rope($cast[BiggestUInt](n.intVal))
+      if optJsBigInt64 in p.config.globalOptions:
+        r.res.add('n')
+    of tyInt64:
+      let wrap = n.intVal < 0 # wrap negative integers with parens
+      if wrap: r.res.add '('
+      r.res.addInt n.intVal
+      if optJsBigInt64 in p.config.globalOptions:
+        r.res.add('n')
+      if wrap: r.res.add ')'
     else:
-      r.res = rope(n.intVal)
+      let wrap = n.intVal < 0 # wrap negative integers with parens
+      if wrap: r.res.add '('
+      r.res.addInt n.intVal
+      if wrap: r.res.add ')'
     r.kind = resExpr
   of nkNilLit:
     if isEmptyType(n.typ):
@@ -2806,7 +3011,7 @@ proc genModule(p: PProc, n: PNode) =
   if p.config.hcrOn and n.kind == nkStmtList:
     let moduleSym = p.module.module
     var moduleLoadedVar = rope(moduleSym.name.s) & "_loaded" &
-                          idOrSig(moduleSym, moduleSym.name.s, p.module.sigConflicts)
+                          idOrSig(moduleSym, moduleSym.name.s, p.module.sigConflicts, p.config)
     lineF(p, "var $1;$n", [moduleLoadedVar])
     var inGuardedBlock = false
 
@@ -2823,11 +3028,11 @@ proc genModule(p: PProc, n: PNode) =
   if optStackTrace in p.options:
     p.body.add(frameDestroy(p))
 
-proc myProcess(b: PPassContext, n: PNode): PNode =
+proc processJSCodeGen*(b: PPassContext, n: PNode): PNode =
   ## Generate JS code for a node.
   result = n
   let m = BModule(b)
-  if passes.skipCodegen(m.config, n): return n
+  if pipelineutils.skipCodegen(m.config, n): return n
   if m.module == nil: internalError(m.config, n.info, "myProcess")
   let globals = PGlobals(m.graph.backend)
   var p = newInitProc(globals, m)
@@ -2844,7 +3049,7 @@ proc wholeCode(graph: ModuleGraph; m: BModule): Rope =
       var p = newInitProc(globals, m)
       attachProc(p, prc)
 
-  var disp = generateMethodDispatchers(graph)
+  var disp = generateMethodDispatchers(graph, m.idgen)
   for i in 0..<disp.len:
     let prc = disp[i].sym
     if not globals.generatedSyms.containsOrIncl(prc.id):
@@ -2862,7 +3067,7 @@ proc getClassName(t: PType): Rope =
   if s.loc.r != "": result = s.loc.r
   else: result = rope(s.name.s)
 
-proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
+proc finalJSCodeGen*(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
   ## Finalize JS code generation of a Nim module.
   ## Param `n` may contain nodes returned from the last module close call.
   var m = BModule(b)
@@ -2872,14 +3077,14 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
     for i in countdown(high(graph.globalDestructors), 0):
       n.add graph.globalDestructors[i]
   # Process any nodes left over from the last call to `myClose`.
-  result = myProcess(b, n)
+  result = processJSCodeGen(b, n)
   # Some codegen is different (such as no stacktraces; see `initProcOptions`)
   # when `std/system` is being processed.
   if sfSystemModule in m.module.flags:
     PGlobals(graph.backend).inSystem = false
   # Check if codegen should continue before any files are generated.
   # It may bail early is if too many errors have been raised.
-  if passes.skipCodegen(m.config, n): return n
+  if pipelineutils.skipCodegen(m.config, n): return n
   # Nim modules are compiled into a single JS file.
   # If this is the main module, then this is the final call to `myClose`.
   if sfMainModule in m.module.flags:
@@ -2897,9 +3102,6 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
       if not writeRope(code, outFile):
         rawMessage(m.config, errCannotOpenFile, outFile.string)
 
-proc myOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext =
-  ## Create the JS backend pass context `BModule` for a Nim module.
+proc setupJSgen*(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext =
   result = newModule(graph, s)
   result.idgen = idgen
-
-const JSgenPass* = makePass(myOpen, myProcess, myClose)

compiler/jstypes.nim

@@ -133,7 +133,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
       [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-  of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet:
+  of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet, tyOpenArray:
     var s =
       "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" %
               [result, rope(ord(t.kind))]

compiler/lambdalifting.nim

@@ -140,7 +140,7 @@ proc createClosureIterStateType*(g: ModuleGraph; iter: PSym; idgen: IdGenerator)
   rawAddSon(result, intType)
 
 proc createStateField(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym =
-  result = newSym(skField, getIdent(g.cache, ":state"), nextSymId(idgen), iter, iter.info)
+  result = newSym(skField, getIdent(g.cache, ":state"), idgen, iter, iter.info)
   result.typ = createClosureIterStateType(g, iter, idgen)
 
 proc createEnvObj(g: ModuleGraph; idgen: IdGenerator; owner: PSym; info: TLineInfo): PType =
@@ -154,7 +154,7 @@ proc getClosureIterResult*(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym
     result = iter.ast[resultPos].sym
   else:
     # XXX a bit hacky:
-    result = newSym(skResult, getIdent(g.cache, ":result"), nextSymId(idgen), iter, iter.info, {})
+    result = newSym(skResult, getIdent(g.cache, ":result"), idgen, iter, iter.info, {})
     result.typ = iter.typ[0]
     incl(result.flags, sfUsed)
     iter.ast.add newSymNode(result)
@@ -267,7 +267,7 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PN
     addUniqueField(it.typ.skipTypes({tyOwned})[0], hp, g.cache, idgen)
     env = indirectAccess(newSymNode(it), hp, hp.info)
   else:
-    let e = newSym(skLet, iter.name, nextSymId(idgen), owner, n.info)
+    let e = newSym(skLet, iter.name, idgen, owner, n.info)
     e.typ = hp.typ
     e.flags = hp.flags
     env = newSymNode(e)
@@ -380,7 +380,7 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
       if c.graph.config.selectedGC == gcDestructors and sfCursor notin upField.flags:
         localError(c.graph.config, dep.info, "internal error: up reference is not a .cursor")
   else:
-    let result = newSym(skField, upIdent, nextSymId(c.idgen), obj.owner, obj.owner.info)
+    let result = newSym(skField, upIdent, c.idgen, obj.owner, obj.owner.info)
     result.typ = fieldType
     when false:
       if c.graph.config.selectedGC == gcDestructors:
@@ -418,7 +418,7 @@ proc addClosureParam(c: var DetectionPass; fn: PSym; info: TLineInfo) =
   let owner = if fn.kind == skIterator: fn else: fn.skipGenericOwner
   let t = c.getEnvTypeForOwner(owner, info)
   if cp == nil:
-    cp = newSym(skParam, getIdent(c.graph.cache, paramName), nextSymId(c.idgen), fn, fn.info)
+    cp = newSym(skParam, getIdent(c.graph.cache, paramName), c.idgen, fn, fn.info)
     incl(cp.flags, sfFromGeneric)
     cp.typ = t
     addHiddenParam(fn, cp)
@@ -545,7 +545,7 @@ proc accessViaEnvParam(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   result = n
 
 proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo; idgen: IdGenerator): PNode =
-  var v = newSym(skVar, getIdent(cache, envName), nextSymId(idgen), owner, info)
+  var v = newSym(skVar, getIdent(cache, envName), idgen, owner, info)
   v.flags = {sfShadowed, sfGeneratedOp}
   v.typ = typ
   result = newSymNode(v)
@@ -569,7 +569,7 @@ proc setupEnvVar(owner: PSym; d: var DetectionPass;
     result = newEnvVar(d.graph.cache, owner, asOwnedRef(d, envVarType), info, d.idgen)
     c.envVars[owner.id] = result
     if optOwnedRefs in d.graph.config.globalOptions:
-      var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), nextSymId d.idgen, owner, info)
+      var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), d.idgen, owner, info)
       v.flags = {sfShadowed, sfGeneratedOp}
       v.typ = envVarType
       c.unownedEnvVars[owner.id] = newSymNode(v)
@@ -653,7 +653,7 @@ proc closureCreationForIter(iter: PNode;
                             d: var DetectionPass; c: var LiftingPass): PNode =
   result = newNodeIT(nkStmtListExpr, iter.info, iter.sym.typ)
   let owner = iter.sym.skipGenericOwner
-  var v = newSym(skVar, getIdent(d.graph.cache, envName), nextSymId(d.idgen), owner, iter.info)
+  var v = newSym(skVar, getIdent(d.graph.cache, envName), d.idgen, owner, iter.info)
   incl(v.flags, sfShadowed)
   v.typ = asOwnedRef(d, getHiddenParam(d.graph, iter.sym).typ)
   var vnode: PNode
@@ -859,7 +859,7 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType;
   fn.typ.callConv = oldCC
 
 proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
-                  idgen: IdGenerator): PNode =
+                  idgen: IdGenerator, force: bool): PNode =
   # XXX backend == backendJs does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
 
@@ -868,7 +868,7 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
 
   if body.kind == nkEmpty or (
       g.config.backend == backendJs and not isCompileTime) or
-      fn.skipGenericOwner.kind != skModule:
+      (fn.skipGenericOwner.kind != skModule and not force):
 
     # ignore forward declaration:
     result = body
@@ -943,7 +943,7 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; idgen: IdGenerator; owner: PSym):
     let iter = op.sym
 
     let hp = getHiddenParam(g, iter)
-    env = newSym(skLet, iter.name, nextSymId(idgen), owner, body.info)
+    env = newSym(skLet, iter.name, idgen, owner, body.info)
     env.typ = hp.typ
     env.flags = hp.flags
 

compiler/layouter.nim

@@ -510,7 +510,7 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
     rememberSplit(splitComma)
     wrSpace em
   of openPars:
-    if tok.strongSpaceA and not em.endsInWhite and
+    if tsLeading in tok.spacing and not em.endsInWhite and
         (not em.wasExportMarker or tok.tokType == tkCurlyDotLe):
       wrSpace em
     wr(em, $tok.tokType, ltSomeParLe)
@@ -528,7 +528,7 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
     wr(em, $tok.tokType, ltOther)
     if not em.inquote: wrSpace(em)
   of tkOpr, tkDotDot:
-    if em.inquote or (((not tok.strongSpaceA) and tok.strongSpaceB == tsNone) and
+    if em.inquote or (tok.spacing == {} and
         tok.ident.s notin ["<", ">", "<=", ">=", "==", "!="]):
       # bug #9504: remember to not spacify a keyword:
       lastTokWasTerse = true
@@ -538,7 +538,7 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
       if not em.endsInWhite: wrSpace(em)
       wr(em, tok.ident.s, ltOpr)
       template isUnary(tok): bool =
-        tok.strongSpaceB == tsNone and tok.strongSpaceA
+        tok.spacing == {tsLeading}
 
       if not isUnary(tok):
         rememberSplit(splitBinary)

compiler/lexer.nim

@@ -23,7 +23,6 @@ when defined(nimPreviewSlimSystem):
   import std/[assertions, formatfloat]
 
 const
-  MaxLineLength* = 80         # lines longer than this lead to a warning
   numChars*: set[char] = {'0'..'9', 'a'..'z', 'A'..'Z'}
   SymChars*: set[char] = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF'}
   SymStartChars*: set[char] = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'}
@@ -94,19 +93,18 @@ type
     base2, base8, base16
 
   TokenSpacing* = enum
-    tsNone, tsTrailing, tsEof
+    tsLeading, tsTrailing, tsEof
 
   Token* = object                # a Nim token
     tokType*: TokType            # the type of the token
+    base*: NumericalBase         # the numerical base; only valid for int
+                                 # or float literals
+    spacing*: set[TokenSpacing]  # spaces around token
     indent*: int                 # the indentation; != -1 if the token has been
                                  # preceded with indentation
     ident*: PIdent               # the parsed identifier
     iNumber*: BiggestInt         # the parsed integer literal
     fNumber*: BiggestFloat       # the parsed floating point literal
-    base*: NumericalBase         # the numerical base; only valid for int
-                                 # or float literals
-    strongSpaceA*: bool          # leading spaces of an operator
-    strongSpaceB*: TokenSpacing  # trailing spaces of an operator
     literal*: string             # the parsed (string) literal; and
                                  # documentation comments are here too
     line*, col*: int
@@ -127,6 +125,8 @@ type
     cache*: IdentCache
     when defined(nimsuggest):
       previousToken: TLineInfo
+      tokenEnd*: TLineInfo
+      previousTokenEnd*: TLineInfo
     config*: ConfigRef
 
 proc getLineInfo*(L: Lexer, tok: Token): TLineInfo {.inline.} =
@@ -176,7 +176,7 @@ proc initToken*(L: var Token) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
-  L.strongSpaceA = false
+  L.spacing = {}
   L.literal = ""
   L.fNumber = 0.0
   L.base = base10
@@ -189,7 +189,7 @@ proc fillToken(L: var Token) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
-  L.strongSpaceA = false
+  L.spacing = {}
   setLen(L.literal, 0)
   L.fNumber = 0.0
   L.base = base10
@@ -735,10 +735,6 @@ proc handleCRLF(L: var Lexer, pos: int): int =
   template registerLine =
     let col = L.getColNumber(pos)
 
-    when not defined(nimpretty):
-      if col > MaxLineLength:
-        lexMessagePos(L, hintLineTooLong, pos)
-
   case L.buf[pos]
   of CR:
     registerLine()
@@ -958,13 +954,15 @@ proc getOperator(L: var Lexer, tok: var Token) =
   tokenEnd(tok, pos-1)
   # advance pos but don't store it in L.bufpos so the next token (which might
   # be an operator too) gets the preceding spaces:
-  tok.strongSpaceB = tsNone
+  tok.spacing = tok.spacing - {tsTrailing, tsEof}
+  var trailing = false
   while L.buf[pos] == ' ':
     inc pos
-    if tok.strongSpaceB != tsTrailing:
-      tok.strongSpaceB = tsTrailing
+    trailing = true
   if L.buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
-    tok.strongSpaceB = tsEof
+    tok.spacing.incl(tsEof)
+  elif trailing:
+    tok.spacing.incl(tsTrailing)
 
 proc getPrecedence*(tok: Token): int =
   ## Calculates the precedence of the given token.
@@ -1075,7 +1073,6 @@ proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
       when defined(nimpretty): tok.literal.add "\L"
       if isDoc:
         when not defined(nimpretty): tok.literal.add "\n"
-        inc tok.iNumber
         var c = toStrip
         while L.buf[pos] == ' ' and c > 0:
           inc pos
@@ -1094,8 +1091,6 @@ proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
 proc scanComment(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tok.tokType = tkComment
-  # iNumber contains the number of '\n' in the token
-  tok.iNumber = 0
   assert L.buf[pos+1] == '#'
   when defined(nimpretty):
     tok.commentOffsetA = L.offsetBase + pos
@@ -1138,7 +1133,6 @@ proc scanComment(L: var Lexer, tok: var Token) =
         while L.buf[pos] == ' ' and c > 0:
           inc pos
           dec c
-        inc tok.iNumber
     else:
       if L.buf[pos] > ' ':
         L.indentAhead = indent
@@ -1151,7 +1145,7 @@ proc scanComment(L: var Lexer, tok: var Token) =
 proc skip(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tokenBegin(tok, pos)
-  tok.strongSpaceA = false
+  tok.spacing.excl(tsLeading)
   when defined(nimpretty):
     var hasComment = false
     var commentIndent = L.currLineIndent
@@ -1162,8 +1156,7 @@ proc skip(L: var Lexer, tok: var Token) =
     case L.buf[pos]
     of ' ':
       inc(pos)
-      if not tok.strongSpaceA:
-        tok.strongSpaceA = true
+      tok.spacing.incl(tsLeading)
     of '\t':
       if not L.allowTabs: lexMessagePos(L, errGenerated, pos, "tabs are not allowed, use spaces instead")
       inc(pos)
@@ -1185,7 +1178,7 @@ proc skip(L: var Lexer, tok: var Token) =
           pos = L.bufpos
         else:
           break
-      tok.strongSpaceA = false
+      tok.spacing.excl(tsLeading)
       when defined(nimpretty):
         if L.buf[pos] == '#' and tok.line < 0: commentIndent = indent
       if L.buf[pos] > ' ' and (L.buf[pos] != '#' or L.buf[pos+1] == '#'):
@@ -1224,6 +1217,10 @@ proc skip(L: var Lexer, tok: var Token) =
 proc rawGetTok*(L: var Lexer, tok: var Token) =
   template atTokenEnd() {.dirty.} =
     when defined(nimsuggest):
+      L.previousTokenEnd.line = L.tokenEnd.line
+      L.previousTokenEnd.col = L.tokenEnd.col
+      L.tokenEnd.line = tok.line.uint16
+      L.tokenEnd.col = getColNumber(L, L.bufpos).int16
       # we attach the cursor to the last *strong* token
       if tok.tokType notin weakTokens:
         L.previousToken.line = tok.line.uint16

compiler/liftdestructors.nim

@@ -8,7 +8,7 @@
 #
 
 ## This module implements lifting for type-bound operations
-## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``).
+## (`=sink`, `=copy`, `=destroy`, `=deepCopy`, `=wasMoved`, `=dup`).
 
 import modulegraphs, lineinfos, idents, ast, renderer, semdata,
   sighashes, lowerings, options, types, msgs, magicsys, tables, ccgutils
@@ -34,6 +34,7 @@ type
 
 template destructor*(t: PType): PSym = getAttachedOp(c.g, t, attachedDestructor)
 template assignment*(t: PType): PSym = getAttachedOp(c.g, t, attachedAsgn)
+template dup*(t: PType): PSym = getAttachedOp(c.g, t, attachedDup)
 template asink*(t: PType): PSym = getAttachedOp(c.g, t, attachedSink)
 
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode)
@@ -49,9 +50,9 @@ proc at(a, i: PNode, elemType: PType): PNode =
   result[1] = i
   result.typ = elemType
 
-proc destructorOverriden(g: ModuleGraph; t: PType): bool =
+proc destructorOverridden(g: ModuleGraph; t: PType): bool =
   let op = getAttachedOp(g, t, attachedDestructor)
-  op != nil and sfOverriden in op.flags
+  op != nil and sfOverridden in op.flags
 
 proc fillBodyTup(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   for i in 0..<t.len:
@@ -82,12 +83,14 @@ proc genBuiltin(c: var TLiftCtx; magic: TMagic; name: string; i: PNode): PNode =
   result = genBuiltin(c.g, c.idgen, magic, name, i)
 
 proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  if c.kind in {attachedAsgn, attachedDeepCopy, attachedSink}:
+  if c.kind in {attachedAsgn, attachedDeepCopy, attachedSink, attachedDup}:
     body.add newAsgnStmt(x, y)
   elif c.kind == attachedDestructor and c.addMemReset:
     let call = genBuiltin(c, mDefault, "default", x)
     call.typ = t
     body.add newAsgnStmt(x, call)
+  elif c.kind == attachedWasMoved:
+    body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc genAddr(c: var TLiftCtx; x: PNode): PNode =
   if x.kind == nkHiddenDeref:
@@ -137,14 +140,22 @@ proc genContainerOf(c: var TLiftCtx; objType: PType, field, x: PSym): PNode =
 proc destructorCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
   var destroy = newNodeIT(nkCall, x.info, op.typ[0])
   destroy.add(newSymNode(op))
-  destroy.add genAddr(c, x)
+  if op.typ[1].kind != tyVar:
+    destroy.add x
+  else:
+    destroy.add genAddr(c, x)
   if sfNeverRaises notin op.flags:
     c.canRaise = true
   if c.addMemReset:
-    result = newTree(nkStmtList, destroy, genBuiltin(c, mWasMoved,  "wasMoved", x))
+    result = newTree(nkStmtList, destroy, genBuiltin(c, mWasMoved,  "`=wasMoved`", x))
   else:
     result = destroy
 
+proc genWasMovedCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
+  result = newNodeIT(nkCall, x.info, op.typ[0])
+  result.add(newSymNode(op))
+  result.add genAddr(c, x)
+
 proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool) =
   case n.kind
   of nkSym:
@@ -152,7 +163,7 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
     let f = n.sym
     let b = if c.kind == attachedTrace: y else: y.dotField(f)
     if (sfCursor in f.flags and f.typ.skipTypes(abstractInst).kind in {tyRef, tyProc} and
-        c.g.config.selectedGC in {gcArc, gcOrc, gcHooks}) or
+        c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcHooks}) or
         enforceDefaultOp:
       defaultOp(c, f.typ, body, x.dotField(f), b)
     else:
@@ -236,7 +247,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
     # for every field (dependent on dest.kind):
     #   `=` dest.field, src.field
     # =destroy(blob)
-    var dummy = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId c.idgen, c.fn, c.info)
+    var dummy = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.idgen, c.fn, c.info)
     dummy.typ = y.typ
     if ccgIntroducedPtr(c.g.config, dummy, y.typ):
       # Because of potential aliasing when the src param is passed by ref, we need to check for equality here,
@@ -245,7 +256,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
         newTreeIT(nkAddr, c.info, makePtrType(c.fn, x.typ, c.idgen), x), newTreeIT(nkAddr, c.info, makePtrType(c.fn, y.typ, c.idgen), y))
       cond.typ = getSysType(c.g, x.info, tyBool)
       body.add genIf(c, cond, newTreeI(nkReturnStmt, c.info, newNodeI(nkEmpty, c.info)))
-    var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId c.idgen, c.fn, c.info)
+    var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.idgen, c.fn, c.info)
     temp.typ = x.typ
     incl(temp.flags, sfFromGeneric)
     var v = newNodeI(nkVarSection, c.info)
@@ -255,7 +266,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
     #body.add newAsgnStmt(blob, x)
 
     var wasMovedCall = newNodeI(nkCall, c.info)
-    wasMovedCall.add(newSymNode(createMagic(c.g, c.idgen, "wasMoved", mWasMoved)))
+    wasMovedCall.add(newSymNode(createMagic(c.g, c.idgen, "`=wasMoved`", mWasMoved)))
     wasMovedCall.add x # mWasMoved does not take the address
     body.add wasMovedCall
 
@@ -276,7 +287,7 @@ proc boolLit*(g: ModuleGraph; info: TLineInfo; value: bool): PNode =
   result.typ = getSysType(g, info, tyBool)
 
 proc getCycleParam(c: TLiftCtx): PNode =
-  assert c.kind == attachedAsgn
+  assert c.kind in {attachedAsgn, attachedDup}
   if c.fn.typ.len == 4:
     result = c.fn.typ.n.lastSon
     assert result.kind == nkSym
@@ -315,6 +326,9 @@ proc newOpCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
 proc newDeepCopyCall(c: var TLiftCtx; op: PSym; x, y: PNode): PNode =
   result = newAsgnStmt(x, newOpCall(c, op, y))
 
+proc newDupCall(c: var TLiftCtx; op: PSym; x, y: PNode): PNode =
+  result = newAsgnStmt(x, newOpCall(c, op, y))
+
 proc usesBuiltinArc(t: PType): bool =
   proc wrap(t: PType): bool {.nimcall.} = ast.isGCedMem(t)
   result = types.searchTypeFor(t, wrap)
@@ -339,9 +353,9 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
                         field: var PSym): bool =
   if optSeqDestructors in c.g.config.globalOptions:
     var op = field
-    let destructorOverriden = destructorOverriden(c.g, t)
+    let destructorOverridden = destructorOverridden(c.g, t)
     if op != nil and op != c.fn and
-        (sfOverriden in op.flags or destructorOverriden):
+        (sfOverridden in op.flags or destructorOverridden):
       if sfError in op.flags:
         incl c.fn.flags, sfError
       #else:
@@ -349,7 +363,7 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
       onUse(c.info, op)
       body.add newHookCall(c, op, x, y)
       result = true
-    elif op == nil and destructorOverriden:
+    elif op == nil and destructorOverridden:
       op = produceSym(c.g, c.c, t, c.kind, c.info, c.idgen)
       body.add newHookCall(c, op, x, y)
       result = true
@@ -387,7 +401,7 @@ proc addDestructorCall(c: var TLiftCtx; orig: PType; body, x: PNode) =
   let t = orig.skipTypes(abstractInst - {tyDistinct})
   var op = t.destructor
 
-  if op != nil and sfOverriden in op.flags:
+  if op != nil and sfOverridden in op.flags:
     if op.ast.isGenericRoutine:
       # patch generic destructor:
       op = instantiateGeneric(c, op, t, t.typeInst)
@@ -410,7 +424,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
   case c.kind
   of attachedDestructor:
     var op = t.destructor
-    if op != nil and sfOverriden in op.flags:
+    if op != nil and sfOverridden in op.flags:
 
       if op.ast.isGenericRoutine:
         # patch generic destructor:
@@ -424,7 +438,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
     #result = addDestructorCall(c, t, body, x)
   of attachedAsgn, attachedSink, attachedTrace:
     var op = getAttachedOp(c.g, t, c.kind)
-    if op != nil and sfOverriden in op.flags:
+    if op != nil and sfOverridden in op.flags:
       if op.ast.isGenericRoutine:
         # patch generic =trace:
         op = instantiateGeneric(c, op, t, t.typeInst)
@@ -442,8 +456,36 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       body.add newDeepCopyCall(c, op, x, y)
       result = true
 
+  of attachedWasMoved:
+    var op = getAttachedOp(c.g, t, attachedWasMoved)
+    if op != nil and sfOverridden in op.flags:
+
+      if op.ast.isGenericRoutine:
+        # patch generic destructor:
+        op = instantiateGeneric(c, op, t, t.typeInst)
+        setAttachedOp(c.g, c.idgen.module, t, attachedWasMoved, op)
+
+      #markUsed(c.g.config, c.info, op, c.g.usageSym)
+      onUse(c.info, op)
+      body.add genWasMovedCall(c, op, x)
+      result = true
+
+  of attachedDup:
+    var op = getAttachedOp(c.g, t, attachedDup)
+    if op != nil and sfOverridden in op.flags:
+
+      if op.ast.isGenericRoutine:
+        # patch generic destructor:
+        op = instantiateGeneric(c, op, t, t.typeInst)
+        setAttachedOp(c.g, c.idgen.module, t, attachedDup, op)
+
+      #markUsed(c.g.config, c.info, op, c.g.usageSym)
+      onUse(c.info, op)
+      body.add newDupCall(c, op, x, y)
+      result = true
+
 proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
-  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info)
+  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.idgen, c.fn, c.info)
   temp.typ = getSysType(c.g, body.info, tyInt)
   incl(temp.flags, sfFromGeneric)
 
@@ -453,7 +495,7 @@ proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
   body.add v
 
 proc declareTempOf(c: var TLiftCtx; body: PNode; value: PNode): PNode =
-  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info)
+  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.idgen, c.fn, c.info)
   temp.typ = value.typ
   incl(temp.flags, sfFromGeneric)
 
@@ -502,6 +544,9 @@ proc forallElements(c: var TLiftCtx; t: PType; body, x, y: PNode) =
 
 proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
+  of attachedDup:
+    body.add setLenSeqCall(c, t, x, y)
+    forallElements(c, t, body, x, y)
   of attachedAsgn, attachedDeepCopy:
     # we generate:
     # setLen(dest, y.len)
@@ -521,17 +566,18 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     forallElements(c, t, body, x, y)
     body.add genBuiltin(c, mDestroy, "destroy", x)
   of attachedTrace:
-    if canFormAcycle(t.elemType):
+    if canFormAcycle(c.g, t.elemType):
       # follow all elements:
       forallElements(c, t, body, x, y)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   createTypeBoundOps(c.g, c.c, t, body.info, c.idgen)
   # recursions are tricky, so we might need to forward the generated
   # operation here:
   var t = t
-  if t.assignment == nil or t.destructor == nil:
-    let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
+  if t.assignment == nil or t.destructor == nil or t.dup == nil:
+    let h = sighashes.hashType(t,c.g.config, {CoType, CoConsiderOwned, CoDistinct})
     let canon = c.g.canonTypes.getOrDefault(h)
     if canon != nil: t = canon
 
@@ -556,15 +602,22 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     doAssert t.destructor != nil
     body.add destructorCall(c, t.destructor, x)
   of attachedTrace:
-    if t.kind != tyString and canFormAcycle(t.elemType):
+    if t.kind != tyString and canFormAcycle(c.g, t.elemType):
       let op = getAttachedOp(c.g, t, c.kind)
       if op == nil:
         return # protect from recursion
       body.add newHookCall(c, op, x, y)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
+  of attachedDup:
+    # XXX: replace these with assertions.
+    let op = getAttachedOp(c.g, t, c.kind)
+    if op == nil:
+      return # protect from recursion
+    body.add newDupCall(c, op, x, y)
 
 proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
-  of attachedAsgn, attachedDeepCopy:
+  of attachedAsgn, attachedDeepCopy, attachedDup:
     body.add callCodegenProc(c.g, "nimAsgnStrV2", c.info, genAddr(c, x), y)
   of attachedSink:
     let moveCall = genBuiltin(c, mMove, "move", x)
@@ -576,10 +629,11 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genBuiltin(c, mDestroy, "destroy", x)
   of attachedTrace:
     discard "strings are atomic and have no inner elements that are to trace"
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
-proc cyclicType*(t: PType): bool =
+proc cyclicType*(g: ModuleGraph, t: PType): bool =
   case t.kind
-  of tyRef: result = types.canFormAcycle(t.lastSon)
+  of tyRef: result = types.canFormAcycle(g, t.lastSon)
   of tyProc: result = t.callConv == ccClosure
   else: result = false
 
@@ -607,10 +661,15 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   let elemType = t.lastSon
 
   createTypeBoundOps(c.g, c.c, elemType, c.info, c.idgen)
-  let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(elemType)
+  let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(c.g, elemType)
+
+  let isInheritableAcyclicRef = c.g.config.selectedGC == gcOrc and
+                      (not isPureObject(elemType)) and
+                      tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags
+  # dynamic Acyclic refs need to use dyn decRef
 
   let tmp =
-    if isCyclic and c.kind in {attachedAsgn, attachedSink}:
+    if isCyclic and c.kind in {attachedAsgn, attachedSink, attachedDup}:
       declareTempOf(c, body, x)
     else:
       x
@@ -632,6 +691,8 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo)
     else:
       cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicDyn", c.info, tmp)
+  elif isInheritableAcyclicRef:
+    cond = callCodegenProc(c.g, "nimDecRefIsLastDyn", c.info, x)
   else:
     cond = callCodegenProc(c.g, "nimDecRefIsLast", c.info, x)
   cond.typ = getSysType(c.g, x.info, tyBool)
@@ -667,6 +728,16 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
         # If the ref is polymorphic we have to account for this
         body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(x, c.idgen), y)
       #echo "can follow ", elemType, " static ", isFinal(elemType)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
+  of attachedDup:
+    if isCyclic:
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, y, callCodegenProc(c.g,
+          "nimIncRefCyclic", c.info, y, getCycleParam(c)))
+    else:
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, y, callCodegenProc(c.g,
+          "nimIncRef", c.info, y))
 
 proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   ## Closures are really like refs except they always use a virtual destructor
@@ -676,7 +747,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
 
   let isCyclic = c.g.config.selectedGC == gcOrc
   let tmp =
-    if isCyclic and c.kind in {attachedAsgn, attachedSink}:
+    if isCyclic and c.kind in {attachedAsgn, attachedSink, attachedDup}:
       declareTempOf(c, body, xenv)
     else:
       xenv
@@ -710,11 +781,21 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
 
       body.add genIf(c, cond, actions)
       body.add newAsgnStmt(x, y)
+  of attachedDup:
+    let yenv = genBuiltin(c, mAccessEnv, "accessEnv", y)
+    yenv.typ = getSysType(c.g, c.info, tyPointer)
+    if isCyclic:
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, yenv, callCodegenProc(c.g, "nimIncRefCyclic", c.info, yenv, getCycleParam(c)))
+    else:
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, yenv, callCodegenProc(c.g, "nimIncRef", c.info, yenv))
   of attachedDestructor:
     body.add genIf(c, cond, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace:
     body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv, c.idgen), y)
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
@@ -727,6 +808,9 @@ proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, y, callCodegenProc(c.g, "nimIncRef", c.info, y))
     body.add genIf(c, x, callCodegenProc(c.g, "nimDecWeakRef", c.info, x))
     body.add newAsgnStmt(x, y)
+  of attachedDup:
+    body.add newAsgnStmt(x, y)
+    body.add genIf(c, y, callCodegenProc(c.g, "nimIncRef", c.info, y))
   of attachedDestructor:
     # it's better to prepend the destruction of weak refs in order to
     # prevent wrong "dangling refs exist" problems:
@@ -739,6 +823,7 @@ proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.sons.insert(des, 0)
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace: discard
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   var actions = newNodeI(nkStmtList, c.info)
@@ -760,10 +845,13 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of attachedSink, attachedAsgn:
     body.add genIf(c, x, actions)
     body.add newAsgnStmt(x, y)
+  of attachedDup:
+    body.add newAsgnStmt(x, y)
   of attachedDestructor:
     body.add genIf(c, x, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace: discard
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   if c.kind == attachedDeepCopy:
@@ -775,7 +863,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     call[1] = y
     body.add newAsgnStmt(x, call)
   elif (optOwnedRefs in c.g.config.globalOptions and
-      optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcOrc}:
+      optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcAtomicArc, gcOrc}:
     let xx = genBuiltin(c, mAccessEnv, "accessEnv", x)
     xx.typ = getSysType(c.g, c.info, tyPointer)
     case c.kind
@@ -790,6 +878,11 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.add genIf(c, yy, callCodegenProc(c.g, "nimIncRef", c.info, yy))
       body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
       body.add newAsgnStmt(x, y)
+    of attachedDup:
+      let yy = genBuiltin(c, mAccessEnv, "accessEnv", y)
+      yy.typ = getSysType(c.g, c.info, tyPointer)
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, yy, callCodegenProc(c.g, "nimIncRef", c.info, yy))
     of attachedDestructor:
       let des = genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
       if body.len == 0:
@@ -798,6 +891,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
         body.sons.insert(des, 0)
     of attachedDeepCopy: assert(false, "cannot happen")
     of attachedTrace: discard
+    of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   let xx = genBuiltin(c, mAccessEnv, "accessEnv", x)
@@ -809,10 +903,13 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of attachedSink, attachedAsgn:
     body.add genIf(c, xx, actions)
     body.add newAsgnStmt(x, y)
+  of attachedDup:
+    body.add newAsgnStmt(x, y)
   of attachedDestructor:
     body.add genIf(c, xx, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace: discard
+  of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
 
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case t.kind
@@ -821,7 +918,7 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       tyPtr, tyUncheckedArray, tyVar, tyLent:
     defaultOp(c, t, body, x, y)
   of tyRef:
-    if c.g.config.selectedGC in {gcArc, gcOrc}:
+    if c.g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
       atomicRefOp(c, t, body, x, y)
     elif (optOwnedRefs in c.g.config.globalOptions and
         optRefCheck in c.g.config.options):
@@ -830,7 +927,7 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       defaultOp(c, t, body, x, y)
   of tyProc:
     if t.callConv == ccClosure:
-      if c.g.config.selectedGC in {gcArc, gcOrc}:
+      if c.g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
         atomicClosureOp(c, t, body, x, y)
       else:
         closureOp(c, t, body, x, y)
@@ -876,10 +973,25 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       defaultOp(c, t, body, x, y)
   of tyObject:
     if not considerUserDefinedOp(c, t, body, x, y):
-      if c.kind in {attachedAsgn, attachedSink} and t.sym != nil and sfImportc in t.sym.flags:
-        body.add newAsgnStmt(x, y)
+      if t.sym != nil and sfImportc in t.sym.flags:
+        case c.kind
+        of {attachedAsgn, attachedSink, attachedDup}:
+          body.add newAsgnStmt(x, y)
+        of attachedWasMoved:
+          body.add genBuiltin(c, mWasMoved, "`=wasMoved`", x)
+        else:
+          fillBodyObjT(c, t, body, x, y)
       else:
-        fillBodyObjT(c, t, body, x, y)
+        if c.kind == attachedDup:
+          var op2 = getAttachedOp(c.g, t, attachedAsgn)
+          if op2 != nil and sfOverridden in op2.flags:
+            #markUsed(c.g.config, c.info, op, c.g.usageSym)
+            onUse(c.info, op2)
+            body.add newHookCall(c, t.assignment, x, y)
+          else:
+            fillBodyObjT(c, t, body, x, y)
+        else:
+          fillBodyObjT(c, t, body, x, y)
   of tyDistinct:
     if not considerUserDefinedOp(c, t, body, x, y):
       fillBody(c, t[0], body, x, y)
@@ -912,15 +1024,56 @@ proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType;
   result = getAttachedOp(g, baseType, kind)
   setAttachedOp(g, idgen.module, typ, kind, result)
 
+proc symDupPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp;
+              info: TLineInfo; idgen: IdGenerator): PSym =
+  let procname = getIdent(g.cache, AttachedOpToStr[kind])
+  result = newSym(skProc, procname, idgen, owner, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), idgen, result, info)
+  let src = newSym(skParam, getIdent(g.cache, "src"),
+                   idgen, result, info)
+  res.typ = typ
+  src.typ = typ
+
+  result.typ = newType(tyProc, nextTypeId idgen, owner)
+  result.typ.n = newNodeI(nkFormalParams, info)
+  rawAddSon(result.typ, res.typ)
+  result.typ.n.add newNodeI(nkEffectList, info)
+
+  result.typ.addParam src
+
+  if g.config.selectedGC == gcOrc and
+    cyclicType(g, typ.skipTypes(abstractInst)):
+    let cycleParam = newSym(skParam, getIdent(g.cache, "cyclic"),
+                            idgen, result, info)
+    cycleParam.typ = getSysType(g, info, tyBool)
+    result.typ.addParam cycleParam
+
+  var n = newNodeI(nkProcDef, info, bodyPos+2)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = newNodeI(nkStmtList, info)
+  n[resultPos] = newSymNode(res)
+  result.ast = n
+  incl result.flags, sfFromGeneric
+  incl result.flags, sfGeneratedOp
+
 proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp;
               info: TLineInfo; idgen: IdGenerator): PSym =
+  if kind == attachedDup:
+    return symDupPrototype(g, typ, owner, kind, info, idgen)
 
   let procname = getIdent(g.cache, AttachedOpToStr[kind])
-  result = newSym(skProc, procname, nextSymId(idgen), owner, info)
-  let dest = newSym(skParam, getIdent(g.cache, "dest"), nextSymId(idgen), result, info)
+  result = newSym(skProc, procname, idgen, owner, info)
+  let dest = newSym(skParam, getIdent(g.cache, "dest"), idgen, result, info)
   let src = newSym(skParam, getIdent(g.cache, if kind == attachedTrace: "env" else: "src"),
-                   nextSymId(idgen), result, info)
-  dest.typ = makeVarType(typ.owner, typ, idgen)
+                   idgen, result, info)
+
+  if kind == attachedDestructor and typ.kind in {tyRef, tyString, tySequence} and g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
+    dest.typ = typ
+  else:
+    dest.typ = makeVarType(typ.owner, typ, idgen)
+
   if kind == attachedTrace:
     src.typ = getSysType(g, info, tyPointer)
   else:
@@ -928,13 +1081,13 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
 
   result.typ = newProcType(info, nextTypeId(idgen), owner)
   result.typ.addParam dest
-  if kind != attachedDestructor:
+  if kind notin {attachedDestructor, attachedWasMoved}:
     result.typ.addParam src
 
   if kind == attachedAsgn and g.config.selectedGC == gcOrc and
-      cyclicType(typ.skipTypes(abstractInst)):
+      cyclicType(g, typ.skipTypes(abstractInst)):
     let cycleParam = newSym(skParam, getIdent(g.cache, "cyclic"),
-                            nextSymId(idgen), result, info)
+                            idgen, result, info)
     cycleParam.typ = getSysType(g, info, tyBool)
     result.typ.addParam cycleParam
 
@@ -946,6 +1099,9 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
   result.ast = n
   incl result.flags, sfFromGeneric
   incl result.flags, sfGeneratedOp
+  if kind == attachedWasMoved:
+    incl result.flags, sfNoSideEffect
+    incl result.typ.flags, tfNoSideEffect
 
 proc genTypeFieldCopy(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   let xx = genBuiltin(c, mAccessTypeField, "accessTypeField", x)
@@ -966,22 +1122,25 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
   var a = TLiftCtx(info: info, g: g, kind: kind, c: c, asgnForType: typ, idgen: idgen,
                    fn: result)
 
-  let dest = result.typ.n[1].sym
-  let d = newDeref(newSymNode(dest))
-  let src = if kind == attachedDestructor: newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
+  let dest = if kind == attachedDup: result.ast[resultPos].sym else: result.typ.n[1].sym
+  let d = if result.typ[1].kind == tyVar: newDeref(newSymNode(dest)) else: newSymNode(dest)
+  let src = case kind
+            of {attachedDestructor, attachedWasMoved}: newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
+            of attachedDup: newSymNode(result.typ.n[1].sym)
             else: newSymNode(result.typ.n[2].sym)
 
   # register this operation already:
   setAttachedOpPartial(g, idgen.module, typ, kind, result)
 
-  if kind == attachedSink and destructorOverriden(g, typ):
+  if kind == attachedSink and destructorOverridden(g, typ):
     ## compiler can use a combination of `=destroy` and memCopy for sink op
     dest.flags.incl sfCursor
-    result.ast[bodyPos].add newOpCall(a, getAttachedOp(g, typ, attachedDestructor), d[0])
+    let op = getAttachedOp(g, typ, attachedDestructor)
+    result.ast[bodyPos].add newOpCall(a, op, if op.typ[1].kind == tyVar: d[0] else: d)
     result.ast[bodyPos].add newAsgnStmt(d, src)
   else:
     var tk: TTypeKind
-    if g.config.selectedGC in {gcArc, gcOrc, gcHooks}:
+    if g.config.selectedGC in {gcArc, gcOrc, gcHooks, gcAtomicArc}:
       tk = skipTypes(typ, {tyOrdinal, tyRange, tyInferred, tyGenericInst, tyStatic, tyAlias, tySink}).kind
     else:
       tk = tyNone # no special casing for strings and seqs
@@ -992,7 +1151,7 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
       fillStrOp(a, typ, result.ast[bodyPos], d, src)
     else:
       fillBody(a, typ, result.ast[bodyPos], d, src)
-      if tk == tyObject and a.kind in {attachedAsgn, attachedSink, attachedDeepCopy} and not lacksMTypeField(typ):
+      if tk == tyObject and a.kind in {attachedAsgn, attachedSink, attachedDeepCopy, attachedDup} and not lacksMTypeField(typ):
         # bug #19205: Do not forget to also copy the hidden type field:
         genTypeFieldCopy(a, typ, result.ast[bodyPos], d, src)
 
@@ -1011,7 +1170,7 @@ proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym,
   a.addMemReset = true
   let discrimantDest = result.typ.n[1].sym
 
-  let dst = newSym(skVar, getIdent(g.cache, "dest"), nextSymId(idgen), result, info)
+  let dst = newSym(skVar, getIdent(g.cache, "dest"), idgen, result, info)
   dst.typ = makePtrType(typ.owner, typ, idgen)
   let dstSym = newSymNode(dst)
   let d = newDeref(dstSym)
@@ -1037,7 +1196,7 @@ proc patchBody(g: ModuleGraph; c: PContext; n: PNode; info: TLineInfo; idgen: Id
       if op != nil:
         if op.ast.isGenericRoutine:
           internalError(g.config, info, "resolved destructor is generic")
-        if op.magic == mDestroy:
+        if op.magic == mDestroy and t.kind != tyString:
           internalError(g.config, info, "patching mDestroy with mDestroy?")
         n[0] = newSymNode(op)
   for x in n: patchBody(g, c, x, info, idgen)
@@ -1047,13 +1206,7 @@ proc inst(g: ModuleGraph; c: PContext; t: PType; kind: TTypeAttachedOp; idgen: I
   let op = getAttachedOp(g, t, kind)
   if op != nil and op.ast != nil and op.ast.isGenericRoutine:
     if t.typeInst != nil:
-      var a: TLiftCtx
-      a.info = info
-      a.g = g
-      a.kind = kind
-      a.c = c
-      a.idgen = idgen
-
+      var a = TLiftCtx(info: info, g: g, kind: kind, c: c, idgen: idgen)
       let opInst = instantiateGeneric(a, op, t, t.typeInst)
       if opInst.ast != nil:
         patchBody(g, c, opInst.ast, info, a.idgen)
@@ -1075,7 +1228,7 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
   let skipped = orig.skipTypes({tyGenericInst, tyAlias, tySink})
   if isEmptyContainer(skipped) or skipped.kind == tyStatic: return
 
-  let h = sighashes.hashType(skipped, {CoType, CoConsiderOwned, CoDistinct})
+  let h = sighashes.hashType(skipped, g.config, {CoType, CoConsiderOwned, CoDistinct})
   var canon = g.canonTypes.getOrDefault(h)
   if canon == nil:
     g.canonTypes[h] = skipped
@@ -1096,15 +1249,15 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
   # bug #15122: We need to produce all prototypes before entering the
   # mind boggling recursion. Hacks like these imply we should rewrite
   # this module.
-  var generics: array[attachedDestructor..attachedTrace, bool]
-  for k in attachedDestructor..lastAttached:
+  var generics: array[attachedWasMoved..attachedTrace, bool]
+  for k in attachedWasMoved..lastAttached:
     generics[k] = getAttachedOp(g, canon, k) != nil
     if not generics[k]:
       setAttachedOp(g, idgen.module, canon, k,
           symPrototype(g, canon, canon.owner, k, info, idgen))
 
   # we generate the destructor first so that other operators can depend on it:
-  for k in attachedDestructor..lastAttached:
+  for k in attachedWasMoved..lastAttached:
     if not generics[k]:
       discard produceSym(g, c, canon, k, info, idgen)
     else:

compiler/lineinfos.nim

@@ -44,6 +44,7 @@ type
     errRstSandboxedDirective,
     errProveInit, # deadcode
     errGenerated,
+    errFailedMove,
     errUser,
     # warnings
     warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape",
@@ -57,6 +58,7 @@ type
     warnRstBrokenLink = "BrokenLink",
     warnRstLanguageXNotSupported = "LanguageXNotSupported",
     warnRstFieldXNotSupported = "FieldXNotSupported",
+    warnRstUnusedImportdoc = "UnusedImportdoc",
     warnRstStyle = "warnRstStyle",
     warnCommentXIgnored = "CommentXIgnored",
     warnTypelessParam = "TypelessParam",
@@ -82,15 +84,17 @@ type
     warnCstringConv = "CStringConv",
     warnPtrToCstringConv = "PtrToCstringConv",
     warnEffect = "Effect",
-    warnCastSizes = "CastSizes"
+    warnCastSizes = "CastSizes", # deadcode
+    warnAboveMaxSizeSet = "AboveMaxSizeSet",
     warnImplicitTemplateRedefinition = "ImplicitTemplateRedefinition",
     warnUnnamedBreak = "UnnamedBreak",
     warnStmtListLambda = "StmtListLambda",
+    warnBareExcept = "BareExcept",
+    warnImplicitDefaultValue = "ImplicitDefaultValue",
     warnUser = "User",
     # hints
     hintSuccess = "Success", hintSuccessX = "SuccessX",
     hintCC = "CC",
-    hintLineTooLong = "LineTooLong",
     hintXDeclaredButNotUsed = "XDeclaredButNotUsed", hintDuplicateModuleImport = "DuplicateModuleImport",
     hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded",
     hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX",
@@ -125,6 +129,7 @@ const
     errRstSandboxedDirective: "disabled directive: '$1'",
     errProveInit: "Cannot prove that '$1' is initialized.",  # deadcode
     errGenerated: "$1",
+    errFailedMove: "$1",
     errUser: "$1",
     warnCannotOpenFile: "cannot open '$1'",
     warnOctalEscape: "octal escape sequences do not exist; leading zero is ignored",
@@ -141,6 +146,7 @@ const
     warnRstBrokenLink: "broken link '$1'",
     warnRstLanguageXNotSupported: "language '$1' not supported",
     warnRstFieldXNotSupported: "field '$1' not supported",
+    warnRstUnusedImportdoc: "importdoc for '$1' is not used",
     warnRstStyle: "RST style: $1",
     warnCommentXIgnored: "comment '$1' ignored",
     warnTypelessParam: "", # deadcode
@@ -181,16 +187,18 @@ const
     warnCstringConv: "$1",
     warnPtrToCstringConv: "unsafe conversion to 'cstring' from '$1'; this will become a compile time error in the future",
     warnEffect: "$1",
-    warnCastSizes: "$1",
+    warnCastSizes: "$1", # deadcode
+    warnAboveMaxSizeSet: "$1",
     warnImplicitTemplateRedefinition: "template '$1' is implicitly redefined; this is deprecated, add an explicit .redefine pragma",
     warnUnnamedBreak: "Using an unnamed break in a block is deprecated; Use a named block with a named break instead",
     warnStmtListLambda: "statement list expression assumed to be anonymous proc; this is deprecated, use `do (): ...` or `proc () = ...` instead",
+    warnBareExcept: "$1",
+    warnImplicitDefaultValue: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
     hintSuccessX: "$build\n$loc lines; ${sec}s; $mem; proj: $project; out: $output",
     hintCC: "CC: $1",
-    hintLineTooLong: "line too long",
     hintXDeclaredButNotUsed: "'$1' is declared but not used",
     hintDuplicateModuleImport: "$1",
     hintXCannotRaiseY: "$1",
@@ -240,7 +248,7 @@ type
   TNoteKinds* = set[TNoteKind]
 
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
-  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed, warnAnyEnumConv}
+  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed, warnAnyEnumConv, warnBareExcept}
   result[2] = result[3] - {hintStackTrace, hintExtendedContext, hintDeclaredLoc, hintProcessingStmt}
   result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
@@ -271,7 +279,7 @@ type
                                # and parsed; usually "" but is used
                                # for 'nimsuggest'
     hash*: string              # the checksum of the file
-    dirty*: bool               # for 'nimfix' / 'nimpretty' like tooling
+    dirty*: bool               # for 'nimpretty' like tooling
     when defined(nimpretty):
       fullContent*: string
   FileIndex* = distinct int32

compiler/linter.nim

@@ -95,7 +95,7 @@ template styleCheckDef*(ctx: PContext; info: TLineInfo; sym: PSym; k: TSymKind)
   if optStyleCheck in ctx.config.options and # ignore if styleChecks are off
      {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # check only if hint/error is enabled
      hintName in ctx.config.notes and # ignore if name checks are not requested
-     ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages
+     ctx.config.belongsToProjectPackage(sym) and # ignore foreign packages
      optStyleUsages notin ctx.config.globalOptions and # ignore if requested to only check name usage
      sym.kind != skResult and # ignore `result`
      sym.kind != skTemp and # ignore temporary variables created by the compiler
@@ -136,7 +136,7 @@ template styleCheckUse*(ctx: PContext; info: TLineInfo; sym: PSym) =
   ## Check symbol uses match their definition's style.
   if {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # ignore if styleChecks are off
      hintName in ctx.config.notes and # ignore if name checks are not requested
-     ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages
+     ctx.config.belongsToProjectPackage(sym) and # ignore foreign packages
      sym.kind != skTemp and # ignore temporary variables created by the compiler
      sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols???
      sfAnon notin sym.flags: # ignore temporary variables created by the compiler
@@ -147,6 +147,10 @@ proc checkPragmaUseImpl(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragm
   if pragmaName != wanted:
     lintReport(conf, info, wanted, pragmaName)
 
-template checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
-  if {optStyleHint, optStyleError} * conf.globalOptions != {}:
-    checkPragmaUseImpl(conf, info, w, pragmaName)
+template checkPragmaUse*(ctx: PContext; info: TLineInfo; w: TSpecialWord; pragmaName: string, sym: PSym) =
+  ## Check builtin pragma uses match their definition's style.
+  ## Note: This only applies to builtin pragmas, not user pragmas.
+  if {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # ignore if styleChecks are off
+     hintName in ctx.config.notes and # ignore if name checks are not requested
+     (sym != nil and ctx.config.belongsToProjectPackage(sym)): # ignore foreign packages
+    checkPragmaUseImpl(ctx.config, info, w, pragmaName)

compiler/lookups.nim

@@ -8,14 +8,14 @@
 #
 
 # This module implements lookup helpers.
-import std/[algorithm, strutils]
+import std/[algorithm, strutils, tables]
 
 when defined(nimPreviewSlimSystem):
   import std/assertions
 
 import
   intsets, ast, astalgo, idents, semdata, types, msgs, options,
-  renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs
+  renderer, lineinfos, modulegraphs, astmsgs, sets, wordrecg
 
 proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope)
 
@@ -94,17 +94,6 @@ iterator localScopesFrom*(c: PContext; scope: PScope): PScope =
     if s == c.topLevelScope: break
     yield s
 
-proc skipAlias*(s: PSym; n: PNode; conf: ConfigRef): PSym =
-  if s == nil or s.kind != skAlias:
-    result = s
-  else:
-    result = s.owner
-    if conf.cmd == cmdNimfix:
-      prettybase.replaceDeprecated(conf, n.info, s, result)
-    else:
-      message(conf, n.info, warnDeprecated, "use " & result.name.s & " instead; " &
-              s.name.s & " is deprecated")
-
 proc isShadowScope*(s: PScope): bool {.inline.} =
   s.parent != nil and s.parent.depthLevel == s.depthLevel
 
@@ -170,6 +159,7 @@ iterator allSyms*(c: PContext): (PSym, int, bool) =
   # really iterate over all symbols in all the scopes. This is expensive
   # and only used by suggest.nim.
   var isLocal = true
+
   var scopeN = 0
   for scope in allScopes(c.currentScope):
     if scope == c.topLevelScope: isLocal = false
@@ -184,6 +174,17 @@ iterator allSyms*(c: PContext): (PSym, int, bool) =
       assert s != nil
       yield (s, scopeN, isLocal)
 
+iterator uniqueSyms*(c: PContext): (PSym, int, bool) =
+  ## Like [allSyms] except only returns unique symbols (Uniqueness determined by line + name)
+  # Track seen symbols so we don't duplicate them.
+  # The int is for the symbols name, and line info is
+  # to be able to tell apart symbols with same name but on different lines
+  var seen = initHashSet[(TLineInfo, int)]()
+  for res in allSyms(c):
+    if not seen.containsOrIncl((res[0].info, res[0].name.id)):
+      yield res
+
+
 proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PSym =
   var marked = initIntSet()
   var symSet = OverloadableSyms
@@ -216,6 +217,23 @@ proc debugScopes*(c: PContext; limit=0, max = int.high) {.deprecated.} =
     if i == limit: return
     inc i
 
+proc searchInScopesAllCandidatesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] =
+  result = @[]
+  for scope in allScopes(c.currentScope):
+    var ti: TIdentIter
+    var candidate = initIdentIter(ti, scope.symbols, s)
+    while candidate != nil:
+      if candidate.kind in filter:
+        result.add candidate
+      candidate = nextIdentIter(ti, scope.symbols)
+
+  if result.len == 0:
+    var marked = initIntSet()
+    for im in c.imports.mitems:
+      for s in symbols(im, marked, s, c.graph):
+        if s.kind in filter:
+          result.add s
+
 proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] =
   result = @[]
   block outer:
@@ -245,7 +263,7 @@ proc errorSym*(c: PContext, n: PNode): PSym =
       considerQuotedIdent(c, m)
     else:
       getIdent(c.cache, "err:" & renderTree(m))
-  result = newSym(skError, ident, nextSymId(c.idgen), getCurrOwner(c), n.info, {})
+  result = newSym(skError, ident, c.idgen, getCurrOwner(c), n.info, {})
   result.typ = errorType(c)
   incl(result.flags, sfDiscardable)
   # pretend it's from the top level scope to prevent cascading errors:
@@ -311,15 +329,18 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string;
 # xxx pending bootstrap >= 1.4, replace all those overloads with a single one:
 # proc addDecl*(c: PContext, sym: PSym, info = sym.info, scope = c.currentScope) {.inline.} =
 proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) =
+  if sym.name.id == ord(wUnderscore): return
   let conflict = scope.addUniqueSym(sym)
   if conflict != nil:
-    if sym.kind == skModule and conflict.kind == skModule and sym.owner == conflict.owner:
+    if sym.kind == skModule and conflict.kind == skModule:
       # e.g.: import foo; import foo
       # xxx we could refine this by issuing a different hint for the case
       # where a duplicate import happens inside an include.
-      localError(c.config, info, hintDuplicateModuleImport,
-        "duplicate import of '$1'; previous import here: $2" %
-        [sym.name.s, c.config $ conflict.info])
+      if c.importModuleMap[sym.id] == c.importModuleMap[conflict.id]:
+        #only hints if the conflict is the actual module not just a shared name
+        localError(c.config, info, hintDuplicateModuleImport,
+          "duplicate import of '$1'; previous import here: $2" %
+          [sym.name.s, c.config $ conflict.info])
     else:
       wrongRedefinition(c, info, sym.name.s, conflict.info, errGenerated)
 
@@ -365,11 +386,12 @@ proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) =
   if fn.kind notin OverloadableSyms:
     internalError(c.config, fn.info, "addOverloadableSymAt")
     return
-  let check = strTableGet(scope.symbols, fn.name)
-  if check != nil and check.kind notin OverloadableSyms:
-    wrongRedefinition(c, fn.info, fn.name.s, check.info)
-  else:
-    scope.addSym(fn)
+  if fn.name.id != ord(wUnderscore):
+    let check = strTableGet(scope.symbols, fn.name)
+    if check != nil and check.kind notin OverloadableSyms:
+      wrongRedefinition(c, fn.info, fn.name.s, check.info)
+    else:
+      scope.addSym(fn)
 
 proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) =
   ## adds an overloadable symbol on the scope and the interface if appropriate
@@ -406,13 +428,6 @@ proc mergeShadowScope*(c: PContext) =
     else:
       c.addInterfaceDecl(sym)
 
-when false:
-  # `nimfix` used to call `altSpelling` and prettybase.replaceDeprecated(n.info, ident, alt)
-  proc altSpelling(c: PContext, x: PIdent): PIdent =
-    case x.s[0]
-    of 'A'..'Z': result = getIdent(c.cache, toLowerAscii(x.s[0]) & x.s.substr(1))
-    of 'a'..'z': result = getIdent(c.cache, toLowerAscii(x.s[0]) & x.s.substr(1))
-    else: result = x
 
 import std/editdistance, heapqueue
 
@@ -445,29 +460,29 @@ proc fixSpelling(c: PContext, n: PNode, ident: PIdent, result: var string) =
     let dist = editDistance(name0, sym.name.s.nimIdentNormalize)
     var msg: string
     msg.add "\n ($1, $2): '$3'" % [$dist, $depth, sym.name.s]
-    addDeclaredLoc(msg, c.config, sym) # `msg` needed for deterministic ordering.
     list.push SpellCandidate(dist: dist, depth: depth, msg: msg, sym: sym)
 
   if list.len == 0: return
   let e0 = list[0]
-  var count = 0
+  var
+    count = 0
+    last: PIdent = nil
   while true:
     # pending https://github.com/timotheecour/Nim/issues/373 use more efficient `itemsSorted`.
     if list.len == 0: break
     let e = list.pop()
     if c.config.spellSuggestMax == spellSuggestSecretSauce:
       const
-        smallThres = 2
-        maxCountForSmall = 4
-        # avoids ton of operator matches when mis-matching short symbols such as `i`
-        # other heuristics could be devised, such as only suggesting operators if `name0`
-        # is an operator (likewise with non-operators).
-      if e.dist > e0.dist or (name0.len <= smallThres and count >= maxCountForSmall): break
+        minLengthForSuggestion = 4
+        maxCount = 3 # avoids ton of matches; three counts for equal distances
+      if e.dist > e0.dist or count >= maxCount or name0.len < minLengthForSuggestion: break
     elif count >= c.config.spellSuggestMax: break
     if count == 0:
       result.add "\ncandidates (edit distance, scope distance); see '--spellSuggest': "
-    result.add e.msg
-    count.inc
+    if e.sym.name != last:
+      result.add e.msg
+      count.inc
+      last = e.sym.name
 
 proc errorUseQualifier(c: PContext; info: TLineInfo; s: PSym; amb: var bool): PSym =
   var err = "ambiguous identifier: '" & s.name.s & "'"
@@ -512,12 +527,16 @@ proc errorUseQualifier*(c: PContext; info:TLineInfo; choices: PNode) =
   errorUseQualifier(c, info, candidates, prefix)
 
 proc errorUndeclaredIdentifier*(c: PContext; info: TLineInfo; name: string, extra = "") =
-  var err = "undeclared identifier: '" & name & "'" & extra
-  if c.recursiveDep.len > 0:
-    err.add "\nThis might be caused by a recursive module dependency:\n"
-    err.add c.recursiveDep
-    # prevent excessive errors for 'nim check'
-    c.recursiveDep = ""
+  var err: string
+  if name == "_":
+    err = "the special identifier '_' is ignored in declarations and cannot be used"
+  else:
+    err = "undeclared identifier: '" & name & "'" & extra
+    if c.recursiveDep.len > 0:
+      err.add "\nThis might be caused by a recursive module dependency:\n"
+      err.add c.recursiveDep
+      # prevent excessive errors for 'nim check'
+      c.recursiveDep = ""
   localError(c.config, info, errGenerated, err)
 
 proc errorUndeclaredIdentifierHint*(c: PContext; n: PNode, ident: PIdent): PSym =
@@ -531,13 +550,13 @@ proc lookUp*(c: PContext, n: PNode): PSym =
   var amb = false
   case n.kind
   of nkIdent:
-    result = searchInScopes(c, n.ident, amb).skipAlias(n, c.config)
+    result = searchInScopes(c, n.ident, amb)
     if result == nil: result = errorUndeclaredIdentifierHint(c, n, n.ident)
   of nkSym:
     result = n.sym
   of nkAccQuoted:
     var ident = considerQuotedIdent(c, n)
-    result = searchInScopes(c, ident, amb).skipAlias(n, c.config)
+    result = searchInScopes(c, ident, amb)
     if result == nil: result = errorUndeclaredIdentifierHint(c, n, ident)
   else:
     internalError(c.config, n.info, "lookUp")
@@ -559,9 +578,9 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
     var amb = false
     var ident = considerQuotedIdent(c, n)
     if checkModule in flags:
-      result = searchInScopes(c, ident, amb).skipAlias(n, c.config)
+      result = searchInScopes(c, ident, amb)
     else:
-      let candidates = searchInScopesFilterBy(c, ident, allExceptModule) #.skipAlias(n, c.config)
+      let candidates = searchInScopesFilterBy(c, ident, allExceptModule)
       if candidates.len > 0:
         result = candidates[0]
         amb = candidates.len > 1
@@ -593,13 +612,20 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
         ident = considerQuotedIdent(c, n[1])
       if ident != nil:
         if m == c.module:
-          result = strTableGet(c.topLevelScope.symbols, ident).skipAlias(n, c.config)
+          result = strTableGet(c.topLevelScope.symbols, ident)
         else:
-          result = someSym(c.graph, m, ident).skipAlias(n, c.config)
+          if c.importModuleLookup.getOrDefault(m.name.id).len > 1:
+            var amb: bool
+            result = errorUseQualifier(c, n.info, m, amb)
+          else:
+            result = someSym(c.graph, m, ident)
         if result == nil and checkUndeclared in flags:
           result = errorUndeclaredIdentifierHint(c, n[1], ident)
       elif n[1].kind == nkSym:
         result = n[1].sym
+        if result.owner != nil and result.owner != m and checkUndeclared in flags:
+          # dotExpr in templates can end up here
+          result = errorUndeclaredIdentifierHint(c, n[1], considerQuotedIdent(c, n[1]))
       elif checkUndeclared in flags and
            n[1].kind notin {nkOpenSymChoice, nkClosedSymChoice}:
         localError(c.config, n[1].info, "identifier expected, but got: " &
@@ -619,7 +645,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
     var scope = c.currentScope
     o.mode = oimNoQualifier
     while true:
-      result = initIdentIter(o.it, scope.symbols, ident).skipAlias(n, c.config)
+      result = initIdentIter(o.it, scope.symbols, ident)
       if result != nil:
         o.currentScope = scope
         break
@@ -627,7 +653,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         scope = scope.parent
         if scope == nil:
           for i in 0..c.imports.high:
-            result = initIdentIter(o.mit, o.marked, c.imports[i], ident, c.graph).skipAlias(n, c.config)
+            result = initIdentIter(o.mit, o.marked, c.imports[i], ident, c.graph)
             if result != nil:
               o.currentScope = nil
               o.importIdx = i
@@ -650,10 +676,10 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         if o.m == c.module:
           # a module may access its private members:
           result = initIdentIter(o.it, c.topLevelScope.symbols,
-                                 ident).skipAlias(n, c.config)
+                                 ident)
           o.mode = oimSelfModule
         else:
-          result = initModuleIter(o.mit, c.graph, o.m, ident).skipAlias(n, c.config)
+          result = initModuleIter(o.mit, c.graph, o.m, ident)
       else:
         noidentError(c.config, n[1], n)
         result = errorSym(c, n[1])
@@ -686,7 +712,7 @@ proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym
   var idx = o.importIdx+1
   o.importIdx = c.imports.len # assume the other imported modules lack this symbol too
   while idx < c.imports.len:
-    result = initIdentIter(o.mit, o.marked, c.imports[idx], o.it.name, c.graph).skipAlias(n, c.config)
+    result = initIdentIter(o.mit, o.marked, c.imports[idx], o.it.name, c.graph)
     if result != nil:
       # oh, we were wrong, some other module had the symbol, so remember that:
       o.importIdx = idx
@@ -696,7 +722,7 @@ proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym
 proc symChoiceExtension(o: var TOverloadIter; c: PContext; n: PNode): PSym =
   assert o.currentScope == nil
   while o.importIdx < c.imports.len:
-    result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph).skipAlias(n, c.config)
+    result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph)
     #while result != nil and result.id in o.marked:
     #  result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx])
     if result != nil:
@@ -711,29 +737,29 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
   of oimNoQualifier:
     if o.currentScope != nil:
       assert o.importIdx < 0
-      result = nextIdentIter(o.it, o.currentScope.symbols).skipAlias(n, c.config)
+      result = nextIdentIter(o.it, o.currentScope.symbols)
       while result == nil:
         o.currentScope = o.currentScope.parent
         if o.currentScope != nil:
-          result = initIdentIter(o.it, o.currentScope.symbols, o.it.name).skipAlias(n, c.config)
+          result = initIdentIter(o.it, o.currentScope.symbols, o.it.name)
           # BUGFIX: o.it.name <-> n.ident
         else:
           o.importIdx = 0
           if c.imports.len > 0:
-            result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph).skipAlias(n, c.config)
+            result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph)
             if result == nil:
               result = nextOverloadIterImports(o, c, n)
           break
     elif o.importIdx < c.imports.len:
-      result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph).skipAlias(n, c.config)
+      result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph)
       if result == nil:
         result = nextOverloadIterImports(o, c, n)
     else:
       result = nil
   of oimSelfModule:
-    result = nextIdentIter(o.it, c.topLevelScope.symbols).skipAlias(n, c.config)
+    result = nextIdentIter(o.it, c.topLevelScope.symbols)
   of oimOtherModule:
-    result = nextModuleIter(o.mit, c.graph).skipAlias(n, c.config)
+    result = nextModuleIter(o.mit, c.graph)
   of oimSymChoice:
     if o.symChoiceIndex < n.len:
       result = n[o.symChoiceIndex].sym
@@ -744,12 +770,12 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
       o.mode = oimSymChoiceLocalLookup
       o.currentScope = c.currentScope
       result = firstIdentExcluding(o.it, o.currentScope.symbols,
-                                   n[0].sym.name, o.marked).skipAlias(n, c.config)
+                                   n[0].sym.name, o.marked)
       while result == nil:
         o.currentScope = o.currentScope.parent
         if o.currentScope != nil:
           result = firstIdentExcluding(o.it, o.currentScope.symbols,
-                                      n[0].sym.name, o.marked).skipAlias(n, c.config)
+                                      n[0].sym.name, o.marked)
         else:
           o.importIdx = 0
           result = symChoiceExtension(o, c, n)
@@ -758,12 +784,12 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         incl o.marked, result.id
   of oimSymChoiceLocalLookup:
     if o.currentScope != nil:
-      result = nextIdentExcluding(o.it, o.currentScope.symbols, o.marked).skipAlias(n, c.config)
+      result = nextIdentExcluding(o.it, o.currentScope.symbols, o.marked)
       while result == nil:
         o.currentScope = o.currentScope.parent
         if o.currentScope != nil:
           result = firstIdentExcluding(o.it, o.currentScope.symbols,
-                                      n[0].sym.name, o.marked).skipAlias(n, c.config)
+                                      n[0].sym.name, o.marked)
         else:
           o.importIdx = 0
           result = symChoiceExtension(o, c, n)
@@ -772,10 +798,10 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         incl o.marked, result.id
 
     elif o.importIdx < c.imports.len:
-      result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph).skipAlias(n, c.config)
+      result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph)
       #assert result.id notin o.marked
       #while result != nil and result.id in o.marked:
-      #  result = nextIdentIter(o.it, c.imports[o.importIdx]).skipAlias(n, c.config)
+      #  result = nextIdentIter(o.it, c.imports[o.importIdx])
       if result == nil:
         inc o.importIdx
         result = symChoiceExtension(o, c, n)

compiler/lowerings.nim

@@ -79,7 +79,7 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: P
   if avoidTemp:
     tempAsNode = value
   else:
-    var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen),
+    var temp = newSym(skTemp, getIdent(g.cache, genPrefix), idgen,
                   owner, value.info, g.config.options)
     temp.typ = skipTypes(value.typ, abstractInst)
     incl(temp.flags, sfFromGeneric)
@@ -100,7 +100,7 @@ proc evalOnce*(g: ModuleGraph; value: PNode; idgen: IdGenerator; owner: PSym): P
   ## freely, multiple times. This is frequently required and such a builtin would also be
   ## handy to have in macros.nim. The value that can be reused is 'result.lastSon'!
   result = newNodeIT(nkStmtListExpr, value.info, value.typ)
-  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen),
+  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), idgen,
                     owner, value.info, g.config.options)
   temp.typ = skipTypes(value.typ, abstractInst)
   incl(temp.flags, sfFromGeneric)
@@ -126,13 +126,13 @@ proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; idgen: IdGenerator; o
   let value = n.lastSon
   result = newNodeI(nkStmtList, n.info)
 
-  var temp = newSym(skTemp, getIdent(g.cache, "_"), nextSymId(idgen), owner, value.info, owner.options)
+  var temp = newSym(skTemp, getIdent(g.cache, "_"), idgen, owner, value.info, owner.options)
   var v = newNodeI(nkLetSection, value.info)
   let tempAsNode = newSymNode(temp) #newIdentNode(getIdent(genPrefix & $temp.id), value.info)
 
   var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
   vpart[0] = tempAsNode
-  vpart[1] = newNodeI(nkEmpty, value.info)
+  vpart[1] = newNodeI(nkTupleClassTy, value.info)
   vpart[2] = value
   v.add vpart
   result.add(v)
@@ -144,7 +144,7 @@ proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; idgen: IdGenerator; o
 proc lowerSwap*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNode =
   result = newNodeI(nkStmtList, n.info)
   # note: cannot use 'skTemp' here cause we really need the copy for the VM :-(
-  var temp = newSym(skVar, getIdent(g.cache, genPrefix), nextSymId(idgen), owner, n.info, owner.options)
+  var temp = newSym(skVar, getIdent(g.cache, genPrefix), idgen, owner, n.info, owner.options)
   temp.typ = n[1].typ
   incl(temp.flags, sfFromGeneric)
   incl(temp.flags, sfGenSym)
@@ -171,8 +171,7 @@ proc createObj*(g: ModuleGraph; idgen: IdGenerator; owner: PSym, info: TLineInfo
     rawAddSon(result, getCompilerProc(g, "RootObj").typ)
   result.n = newNodeI(nkRecList, info)
   let s = newSym(skType, getIdent(g.cache, "Env_" & toFilename(g.config, info) & "_" & $owner.name.s),
-                  nextSymId(idgen),
-                  owner, info, owner.options)
+                  idgen, owner, info, owner.options)
   incl s.flags, sfAnon
   s.typ = result
   result.sym = s
@@ -234,7 +233,7 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym
   # because of 'gensym' support, we have to mangle the name with its ID.
   # This is hacky but the clean solution is much more complex than it looks.
   var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len),
-                     nextSymId(idgen), s.owner, s.info, s.options)
+                     idgen, s.owner, s.info, s.options)
   field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
   let t = skipIntLit(s.typ, idgen)
   field.typ = t
@@ -250,7 +249,7 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym
 proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym {.discardable.} =
   result = lookupInRecord(obj.n, s.itemId)
   if result == nil:
-    var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), nextSymId(idgen),
+    var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), idgen,
                        s.owner, s.info, s.options)
     field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
     let t = skipIntLit(s.typ, idgen)
@@ -347,7 +346,8 @@ proc genDeref*(n: PNode; k = nkHiddenDeref): PNode =
 
 proc callCodegenProc*(g: ModuleGraph; name: string;
                       info: TLineInfo = unknownLineInfo;
-                      arg1, arg2, arg3, optionalArgs: PNode = nil): PNode =
+                      arg1: PNode = nil, arg2: PNode = nil,
+                      arg3: PNode = nil, optionalArgs: PNode = nil): PNode =
   result = newNodeI(nkCall, info)
   let sym = magicsys.getCompilerProc(g, name)
   if sym == nil:

compiler/magicsys.nim

@@ -26,9 +26,8 @@ proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym =
   result = systemModuleSym(g, getIdent(g.cache, name))
   if result == nil:
     localError(g.config, info, "system module needs: " & name)
-    result = newSym(skError, getIdent(g.cache, name), nextSymId(g.idgen), g.systemModule, g.systemModule.info, {})
+    result = newSym(skError, getIdent(g.cache, name), g.idgen, g.systemModule, g.systemModule.info, {})
     result.typ = newType(tyError, nextTypeId(g.idgen), g.systemModule)
-  if result.kind == skAlias: result = result.owner
 
 proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSym =
   let id = getIdent(g.cache, name)
@@ -39,7 +38,7 @@ proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSy
       result = r
   if result != nil: return result
   localError(g.config, info, "system module needs: " & name)
-  result = newSym(skError, id, nextSymId(g.idgen), g.systemModule, g.systemModule.info, {})
+  result = newSym(skError, id, g.idgen, g.systemModule, g.systemModule.info, {})
   result.typ = newType(tyError, nextTypeId(g.idgen), g.systemModule)
 
 proc sysTypeFromName*(g: ModuleGraph; info: TLineInfo; name: string): PType =

compiler/main.nim

@@ -13,12 +13,12 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  std/[strutils, os, times, tables, sha1, with, json],
+  std/[strutils, os, times, tables, with, json],
   llstream, ast, lexer, syntaxes, options, msgs,
   condsyms,
-  sem, idents, passes, extccomp,
+  idents, extccomp,
   cgen, nversion,
-  platform, nimconf, passaux, depends, vm,
+  platform, nimconf, depends,
   modules,
   modulegraphs, lineinfos, pathutils, vmprofiler
 
@@ -29,12 +29,12 @@ when defined(nimPreviewSlimSystem):
 import ic / [cbackend, integrity, navigator]
 from ic / ic import rodViewer
 
-when not defined(leanCompiler):
-  import jsgen, docgen, docgen2
+import ../dist/checksums/src/checksums/sha1
 
-proc semanticPasses(g: ModuleGraph) =
-  registerPass g, verbosePass
-  registerPass g, semPass
+import pipelines
+
+when not defined(leanCompiler):
+  import docgen
 
 proc writeDepsFile(g: ModuleGraph) =
   let fname = g.config.nimcacheDir / RelativeFile(g.config.projectName & ".deps")
@@ -68,12 +68,18 @@ proc writeCMakeDepsFile(conf: ConfigRef) =
     fl.close()
 
 proc commandGenDepend(graph: ModuleGraph) =
-  semanticPasses(graph)
-  registerPass(graph, gendependPass)
-  compileProject(graph)
+  setPipeLinePass(graph, GenDependPass)
+  compilePipelineProject(graph)
   let project = graph.config.projectFull
   writeDepsFile(graph)
   generateDot(graph, project)
+
+  # dot in graphivz tool kit is required
+  let graphvizDotPath = findExe("dot")
+  if graphvizDotPath.len == 0:
+    quit("gendepend: Graphviz's tool dot is required," &
+    "see https://graphviz.org/download for downloading")
+
   execExternalProgram(graph.config, "dot -Tpng -o" &
       changeFileExt(project, "png").string &
       ' ' & changeFileExt(project, "dot").string)
@@ -87,8 +93,8 @@ proc commandCheck(graph: ModuleGraph) =
     defineSymbol(conf.symbols, "nimconfig")
   elif conf.backend == backendJs:
     setTarget(conf.target, osJS, cpuJS)
-  semanticPasses(graph)  # use an empty backend for semantic checking only
-  compileProject(graph)
+  setPipeLinePass(graph, SemPass)
+  compilePipelineProject(graph)
 
   if conf.symbolFiles != disabledSf:
     case conf.ideCmd
@@ -102,22 +108,20 @@ when not defined(leanCompiler):
   proc commandDoc2(graph: ModuleGraph; ext: string) =
     handleDocOutputOptions graph.config
     graph.config.setErrorMaxHighMaybe
-    semanticPasses(graph)
     case ext:
-    of TexExt:  registerPass(graph, docgen2TexPass)
-    of JsonExt: registerPass(graph, docgen2JsonPass)
-    of HtmlExt: registerPass(graph, docgen2Pass)
+    of TexExt:
+      setPipeLinePass(graph, Docgen2TexPass)
+    of JsonExt:
+      setPipeLinePass(graph, Docgen2JsonPass)
+    of HtmlExt:
+      setPipeLinePass(graph, Docgen2Pass)
     else: doAssert false, $ext
-    compileProject(graph)
-    finishDoc2Pass(graph.config.projectName)
+    compilePipelineProject(graph)
 
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
   extccomp.initVars(conf)
-  semanticPasses(graph)
   if conf.symbolFiles == disabledSf:
-    registerPass(graph, cgenPass)
-
     if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"):
       if not changeDetectedViaJsonBuildInstructions(conf, conf.jsonBuildInstructionsFile):
         # nothing changed
@@ -127,7 +131,11 @@ proc commandCompileToC(graph: ModuleGraph) =
   if not extccomp.ccHasSaneOverflow(conf):
     conf.symbols.defineSymbol("nimEmulateOverflowChecks")
 
-  compileProject(graph)
+  if conf.symbolFiles == disabledSf:
+    setPipeLinePass(graph, CgenPass)
+  else:
+    setPipeLinePass(graph, SemPass)
+  compilePipelineProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933
   if conf.symbolFiles == disabledSf:
@@ -160,33 +168,27 @@ proc commandCompileToJS(graph: ModuleGraph) =
     conf.exc = excCpp
     setTarget(conf.target, osJS, cpuJS)
     defineSymbol(conf.symbols, "ecmascript") # For backward compatibility
-    semanticPasses(graph)
-    registerPass(graph, JSgenPass)
-    compileProject(graph)
+    setPipeLinePass(graph, JSgenPass)
+    compilePipelineProject(graph)
     if optGenScript in conf.globalOptions:
       writeDepsFile(graph)
 
-proc interactivePasses(graph: ModuleGraph) =
+proc commandInteractive(graph: ModuleGraph) =
+  graph.config.setErrorMaxHighMaybe
   initDefines(graph.config.symbols)
   defineSymbol(graph.config.symbols, "nimscript")
   # note: seems redundant with -d:nimHasLibFFI
   when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
-  registerPass(graph, verbosePass)
-  registerPass(graph, semPass)
-  registerPass(graph, evalPass)
-
-proc commandInteractive(graph: ModuleGraph) =
-  graph.config.setErrorMaxHighMaybe
-  interactivePasses(graph)
-  compileSystemModule(graph)
+  setPipeLinePass(graph, InterpreterPass)
+  compilePipelineSystemModule(graph)
   if graph.config.commandArgs.len > 0:
-    discard graph.compileModule(fileInfoIdx(graph.config, graph.config.projectFull), {})
+    discard graph.compilePipelineModule(fileInfoIdx(graph.config, graph.config.projectFull), {})
   else:
     var m = graph.makeStdinModule()
     incl(m.flags, sfMainModule)
     var idgen = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0)
     let s = llStreamOpenStdIn(onPrompt = proc() = flushDot(graph.config))
-    processModule(graph, m, idgen, s)
+    discard processPipelineModule(graph, m, idgen, s)
 
 proc commandScan(cache: IdentCache, config: ConfigRef) =
   var f = addFileExt(AbsoluteFile mainCommandArg(config), NimExt)
@@ -241,8 +243,6 @@ proc mainCommand*(graph: ModuleGraph) =
   let conf = graph.config
   let cache = graph.cache
 
-  # In "nim serve" scenario, each command must reset the registered passes
-  clearPasses(graph)
   conf.lastCmdTime = epochTime()
   conf.searchPaths.add(conf.libpath)
 
@@ -417,7 +417,7 @@ proc mainCommand*(graph: ModuleGraph) =
   of cmdJsonscript:
     setOutFile(graph.config)
     commandJsonScript(graph)
-  of cmdUnknown, cmdNone, cmdIdeTools, cmdNimfix:
+  of cmdUnknown, cmdNone, cmdIdeTools:
     rawMessage(conf, errGenerated, "invalid command: " & conf.command)
 
   if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop}:

compiler/md5_old.nim

@@ -1,297 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2010 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-# `std/md5` without VM and JavaScript support, to circumvent a bug with
-# openarrays on Nim < 1.4.
-
-when defined(nimHasStyleChecks):
-  {.push styleChecks: off.}
-
-type
-  MD5State = array[0..3, uint32]
-  MD5Block = array[0..15, uint32]
-  MD5CBits = array[0..7, uint8]
-  MD5Digest* = array[0..15, uint8]
-    ## MD5 checksum of a string, obtained with the `toMD5 proc <#toMD5,string>`_.
-  MD5Buffer = array[0..63, uint8]
-  MD5Context* {.final.} = object
-    state: MD5State
-    count: array[0..1, uint32]
-    buffer: MD5Buffer
-
-const
-  padding: array[0..63, uint8] = [
-    0x80'u8, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0
-  ]
-
-proc F(x, y, z: uint32): uint32 {.inline.} =
-  result = (x and y) or ((not x) and z)
-
-proc G(x, y, z: uint32): uint32 {.inline.} =
-  result = (x and z) or (y and (not z))
-
-proc H(x, y, z: uint32): uint32 {.inline.} =
-  result = x xor y xor z
-
-proc I(x, y, z: uint32): uint32 {.inline.} =
-  result = y xor (x or (not z))
-
-proc rot(x: var uint32, n: uint8) {.inline.} =
-  x = (x shl n) or (x shr (32'u32 - n))
-
-proc FF(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) =
-  a = a + F(b, c, d) + x + ac
-  rot(a, s)
-  a = a + b
-
-proc GG(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) =
-  a = a + G(b, c, d) + x + ac
-  rot(a, s)
-  a = a + b
-
-proc HH(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) =
-  a = a + H(b, c, d) + x + ac
-  rot(a, s)
-  a = a + b
-
-proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) =
-  a = a + I(b, c, d) + x + ac
-  rot(a, s)
-  a = a + b
-
-proc encode(dest: var MD5Block, src: openArray[uint8]) =
-  var j = 0
-  for i in 0..high(dest):
-    dest[i] = uint32(ord(src[j])) or
-              uint32(ord(src[j+1])) shl 8 or
-              uint32(ord(src[j+2])) shl 16 or
-              uint32(ord(src[j+3])) shl 24
-    inc(j, 4)
-
-proc decode(dest: var openArray[uint8], src: openArray[uint32]) =
-  var i = 0
-  for j in 0..high(src):
-    dest[i] = uint8(src[j] and 0xff'u32)
-    dest[i+1] = uint8(src[j] shr 8 and 0xff'u32)
-    dest[i+2] = uint8(src[j] shr 16 and 0xff'u32)
-    dest[i+3] = uint8(src[j] shr 24 and 0xff'u32)
-    inc(i, 4)
-
-template slice(s: string | cstring, a, b): openArray[uint8] =
-  s.toOpenArrayByte(a, b)
-
-template slice(s: openArray[uint8], a, b): openArray[uint8] =
-  s.toOpenArray(a, b)
-
-proc transform(buffer: openArray[uint8], state: var MD5State) =
-  var
-    myBlock: MD5Block
-  encode(myBlock, buffer)
-  var a = state[0]
-  var b = state[1]
-  var c = state[2]
-  var d = state[3]
-  FF(a, b, c, d, myBlock[0], 7'u8, 0xD76AA478'u32)
-  FF(d, a, b, c, myBlock[1], 12'u8, 0xE8C7B756'u32)
-  FF(c, d, a, b, myBlock[2], 17'u8, 0x242070DB'u32)
-  FF(b, c, d, a, myBlock[3], 22'u8, 0xC1BDCEEE'u32)
-  FF(a, b, c, d, myBlock[4], 7'u8, 0xF57C0FAF'u32)
-  FF(d, a, b, c, myBlock[5], 12'u8, 0x4787C62A'u32)
-  FF(c, d, a, b, myBlock[6], 17'u8, 0xA8304613'u32)
-  FF(b, c, d, a, myBlock[7], 22'u8, 0xFD469501'u32)
-  FF(a, b, c, d, myBlock[8], 7'u8, 0x698098D8'u32)
-  FF(d, a, b, c, myBlock[9], 12'u8, 0x8B44F7AF'u32)
-  FF(c, d, a, b, myBlock[10], 17'u8, 0xFFFF5BB1'u32)
-  FF(b, c, d, a, myBlock[11], 22'u8, 0x895CD7BE'u32)
-  FF(a, b, c, d, myBlock[12], 7'u8, 0x6B901122'u32)
-  FF(d, a, b, c, myBlock[13], 12'u8, 0xFD987193'u32)
-  FF(c, d, a, b, myBlock[14], 17'u8, 0xA679438E'u32)
-  FF(b, c, d, a, myBlock[15], 22'u8, 0x49B40821'u32)
-  GG(a, b, c, d, myBlock[1], 5'u8, 0xF61E2562'u32)
-  GG(d, a, b, c, myBlock[6], 9'u8, 0xC040B340'u32)
-  GG(c, d, a, b, myBlock[11], 14'u8, 0x265E5A51'u32)
-  GG(b, c, d, a, myBlock[0], 20'u8, 0xE9B6C7AA'u32)
-  GG(a, b, c, d, myBlock[5], 5'u8, 0xD62F105D'u32)
-  GG(d, a, b, c, myBlock[10], 9'u8, 0x02441453'u32)
-  GG(c, d, a, b, myBlock[15], 14'u8, 0xD8A1E681'u32)
-  GG(b, c, d, a, myBlock[4], 20'u8, 0xE7D3FBC8'u32)
-  GG(a, b, c, d, myBlock[9], 5'u8, 0x21E1CDE6'u32)
-  GG(d, a, b, c, myBlock[14], 9'u8, 0xC33707D6'u32)
-  GG(c, d, a, b, myBlock[3], 14'u8, 0xF4D50D87'u32)
-  GG(b, c, d, a, myBlock[8], 20'u8, 0x455A14ED'u32)
-  GG(a, b, c, d, myBlock[13], 5'u8, 0xA9E3E905'u32)
-  GG(d, a, b, c, myBlock[2], 9'u8, 0xFCEFA3F8'u32)
-  GG(c, d, a, b, myBlock[7], 14'u8, 0x676F02D9'u32)
-  GG(b, c, d, a, myBlock[12], 20'u8, 0x8D2A4C8A'u32)
-  HH(a, b, c, d, myBlock[5], 4'u8, 0xFFFA3942'u32)
-  HH(d, a, b, c, myBlock[8], 11'u8, 0x8771F681'u32)
-  HH(c, d, a, b, myBlock[11], 16'u8, 0x6D9D6122'u32)
-  HH(b, c, d, a, myBlock[14], 23'u8, 0xFDE5380C'u32)
-  HH(a, b, c, d, myBlock[1], 4'u8, 0xA4BEEA44'u32)
-  HH(d, a, b, c, myBlock[4], 11'u8, 0x4BDECFA9'u32)
-  HH(c, d, a, b, myBlock[7], 16'u8, 0xF6BB4B60'u32)
-  HH(b, c, d, a, myBlock[10], 23'u8, 0xBEBFBC70'u32)
-  HH(a, b, c, d, myBlock[13], 4'u8, 0x289B7EC6'u32)
-  HH(d, a, b, c, myBlock[0], 11'u8, 0xEAA127FA'u32)
-  HH(c, d, a, b, myBlock[3], 16'u8, 0xD4EF3085'u32)
-  HH(b, c, d, a, myBlock[6], 23'u8, 0x04881D05'u32)
-  HH(a, b, c, d, myBlock[9], 4'u8, 0xD9D4D039'u32)
-  HH(d, a, b, c, myBlock[12], 11'u8, 0xE6DB99E5'u32)
-  HH(c, d, a, b, myBlock[15], 16'u8, 0x1FA27CF8'u32)
-  HH(b, c, d, a, myBlock[2], 23'u8, 0xC4AC5665'u32)
-  II(a, b, c, d, myBlock[0], 6'u8, 0xF4292244'u32)
-  II(d, a, b, c, myBlock[7], 10'u8, 0x432AFF97'u32)
-  II(c, d, a, b, myBlock[14], 15'u8, 0xAB9423A7'u32)
-  II(b, c, d, a, myBlock[5], 21'u8, 0xFC93A039'u32)
-  II(a, b, c, d, myBlock[12], 6'u8, 0x655B59C3'u32)
-  II(d, a, b, c, myBlock[3], 10'u8, 0x8F0CCC92'u32)
-  II(c, d, a, b, myBlock[10], 15'u8, 0xFFEFF47D'u32)
-  II(b, c, d, a, myBlock[1], 21'u8, 0x85845DD1'u32)
-  II(a, b, c, d, myBlock[8], 6'u8, 0x6FA87E4F'u32)
-  II(d, a, b, c, myBlock[15], 10'u8, 0xFE2CE6E0'u32)
-  II(c, d, a, b, myBlock[6], 15'u8, 0xA3014314'u32)
-  II(b, c, d, a, myBlock[13], 21'u8, 0x4E0811A1'u32)
-  II(a, b, c, d, myBlock[4], 6'u8, 0xF7537E82'u32)
-  II(d, a, b, c, myBlock[11], 10'u8, 0xBD3AF235'u32)
-  II(c, d, a, b, myBlock[2], 15'u8, 0x2AD7D2BB'u32)
-  II(b, c, d, a, myBlock[9], 21'u8, 0xEB86D391'u32)
-  state[0] = state[0] + a
-  state[1] = state[1] + b
-  state[2] = state[2] + c
-  state[3] = state[3] + d
-
-proc md5Init*(c: var MD5Context) {.raises: [], tags: [], gcsafe.}
-proc md5Update*(c: var MD5Context, input: openArray[uint8]) {.raises: [],
-    tags: [], gcsafe.}
-proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], gcsafe.}
-
-proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [],
-    tags: [], gcsafe.} =
-  ## Updates the `MD5Context` with the `input` data of length `len`.
-  ##
-  ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this
-  ## function explicitly.
-  md5Update(c, input.slice(0, len - 1))
-
-
-proc toMD5*(s: string): MD5Digest =
-  ## Computes the `MD5Digest` value for a string `s`.
-  ##
-  ## **See also:**
-  ## * `getMD5 proc <#getMD5,string>`_ which returns a string representation
-  ##   of the `MD5Digest`
-  ## * `$ proc <#$,MD5Digest>`_ for converting MD5Digest to string
-  runnableExamples:
-    assert $toMD5("abc") == "900150983cd24fb0d6963f7d28e17f72"
-
-  var c: MD5Context
-  md5Init(c)
-  md5Update(c, s.slice(0, s.len - 1))
-  md5Final(c, result)
-
-proc `$`*(d: MD5Digest): string =
-  ## Converts a `MD5Digest` value into its string representation.
-  const digits = "0123456789abcdef"
-  result = ""
-  for i in 0..15:
-    add(result, digits[(d[i].int shr 4) and 0xF])
-    add(result, digits[d[i].int and 0xF])
-
-proc getMD5*(s: string): string =
-  ## Computes an MD5 value of `s` and returns its string representation.
-  ##
-  ## **See also:**
-  ## * `toMD5 proc <#toMD5,string>`_ which returns the `MD5Digest` of a string
-  runnableExamples:
-    assert getMD5("abc") == "900150983cd24fb0d6963f7d28e17f72"
-
-  var
-    c: MD5Context
-    d: MD5Digest
-  md5Init(c)
-  md5Update(c, s.slice(0, s.len - 1))
-  md5Final(c, d)
-  result = $d
-
-proc `==`*(D1, D2: MD5Digest): bool =
-  ## Checks if two `MD5Digest` values are identical.
-  for i in 0..15:
-    if D1[i] != D2[i]: return false
-  return true
-
-
-proc clearBuffer(c: var MD5Context) {.inline.} =
-  zeroMem(addr(c.buffer), sizeof(MD5Buffer))
-
-proc md5Init*(c: var MD5Context) =
-  ## Initializes an `MD5Context`.
-  ##
-  ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this
-  ## function explicitly.
-  c.state[0] = 0x67452301'u32
-  c.state[1] = 0xEFCDAB89'u32
-  c.state[2] = 0x98BADCFE'u32
-  c.state[3] = 0x10325476'u32
-  c.count[0] = 0'u32
-  c.count[1] = 0'u32
-  clearBuffer(c)
-
-proc writeBuffer(c: var MD5Context, index: int,
-                 input: openArray[uint8], inputIndex, len: int) {.inline.} =
-  copyMem(addr(c.buffer[index]), unsafeAddr(input[inputIndex]), len)
-
-proc md5Update*(c: var MD5Context, input: openArray[uint8]) =
-  ## Updates the `MD5Context` with the `input` data.
-  ##
-  ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this
-  ## function explicitly.
-  var Index = int((c.count[0] shr 3) and 0x3F)
-  c.count[0] = c.count[0] + (uint32(input.len) shl 3)
-  if c.count[0] < (uint32(input.len) shl 3): c.count[1] = c.count[1] + 1'u32
-  c.count[1] = c.count[1] + (uint32(input.len) shr 29)
-  var PartLen = 64 - Index
-  if input.len >= PartLen:
-    writeBuffer(c, Index, input, 0, PartLen)
-    transform(c.buffer, c.state)
-    var i = PartLen
-    while i + 63 < input.len:
-      transform(input.slice(i, i + 63), c.state)
-      inc(i, 64)
-    if i < input.len:
-      writeBuffer(c, 0, input, i, input.len - i)
-  elif input.len > 0:
-    writeBuffer(c, Index, input, 0, input.len)
-
-proc md5Final*(c: var MD5Context, digest: var MD5Digest) =
-  ## Finishes the `MD5Context` and stores the result in `digest`.
-  ##
-  ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this
-  ## function explicitly.
-  var
-    Bits: MD5CBits
-    PadLen: int
-  decode(Bits, c.count)
-  var Index = int((c.count[0] shr 3) and 0x3F)
-  if Index < 56: PadLen = 56 - Index
-  else: PadLen = 120 - Index
-  md5Update(c, padding.slice(0, PadLen - 1))
-  md5Update(c, Bits)
-  decode(digest, c.state)
-  clearBuffer(c)
-
-
-when defined(nimHasStyleChecks):
-  {.pop.} #{.push styleChecks: off.}

compiler/modulegraphs.nim

@@ -11,7 +11,8 @@
 ## represents a complete Nim project. Single modules can either be kept in RAM
 ## or stored in a rod-file.
 
-import intsets, tables, hashes, md5_old
+import intsets, tables, hashes
+import ../dist/checksums/src/checksums/md5
 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages
 import ic / [packed_ast, ic]
 
@@ -57,6 +58,18 @@ type
     sym*: PSym
     info*: TLineInfo
 
+  PipelinePass* = enum
+    NonePass
+    SemPass
+    JSgenPass
+    CgenPass
+    EvalPass
+    InterpreterPass
+    GenDependPass
+    Docgen2TexPass
+    Docgen2JsonPass
+    Docgen2Pass
+
   ModuleGraph* {.acyclic.} = ref object
     ifaces*: seq[Iface]  ## indexed by int32 fileIdx
     packed*: PackedModuleGraph
@@ -66,6 +79,7 @@ type
     procInstCache*: Table[ItemId, seq[LazyInstantiation]] # A symbol's ItemId.
     attachedOps*: array[TTypeAttachedOp, Table[ItemId, LazySym]] # Type ID, destructors, etc.
     methodsPerType*: Table[ItemId, seq[(int, LazySym)]] # Type ID, attached methods
+    memberProcsPerType*: Table[ItemId, seq[PSym]] # Type ID, attached member procs (only c++, virtual and ctor so far)
     enumToStringProcs*: Table[ItemId, LazySym]
     emittedTypeInfo*: Table[string, FileIndex]
 
@@ -104,6 +118,7 @@ type
     cacheCounters*: Table[string, BiggestInt] # IC: implemented
     cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented
     passes*: seq[TPass]
+    pipelinePass*: PipelinePass
     onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
@@ -398,7 +413,7 @@ proc stopCompile*(g: ModuleGraph): bool {.inline.} =
   result = g.doStopCompile != nil and g.doStopCompile()
 
 proc createMagic*(g: ModuleGraph; idgen: IdGenerator; name: string, m: TMagic): PSym =
-  result = newSym(skProc, getIdent(g.cache, name), nextSymId(idgen), nil, unknownLineInfo, {})
+  result = newSym(skProc, getIdent(g.cache, name), idgen, nil, unknownLineInfo, {})
   result.magic = m
   result.flags = {sfNeverRaises}
 

compiler/modules.nim

@@ -11,13 +11,8 @@
 
 import
   ast, magicsys, msgs, options,
-  idents, lexer, passes, syntaxes, llstream, modulegraphs,
-  lineinfos, pathutils, tables, packages
-
-when defined(nimPreviewSlimSystem):
-  import std/[syncio, assertions]
-
-import ic / replayer
+  idents, lexer, syntaxes, modulegraphs,
+  lineinfos, pathutils
 
 proc resetSystemArtifacts*(g: ModuleGraph) =
   magicsys.resetSysTypes(g)
@@ -25,12 +20,12 @@ proc resetSystemArtifacts*(g: ModuleGraph) =
 template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent =
   getIdent(graph.cache, splitFile(filename).name)
 
-proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
+proc partialInitModule*(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
   let packSym = getPackage(graph, fileIdx)
   result.owner = packSym
   result.position = int fileIdx
 
-proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+proc newModule*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
   let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID.
@@ -38,103 +33,21 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
                 name: getModuleIdent(graph, filename),
                 info: newLineInfo(fileIdx, 1, 1))
   if not isNimIdentifier(result.name.s):
-    rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
+    rawMessage(graph.config, errGenerated, "invalid module name: '" & result.name.s &
+              "'; a module name must be a valid Nim identifier.")
   partialInitModule(result, graph, fileIdx, filename)
   graph.registerModule(result)
 
-proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym =
-  var flags = flags
-  if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
-  result = graph.getModule(fileIdx)
-
-  template processModuleAux(moduleStatus) =
-    onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule)
-    var s: PLLStream
-    if sfMainModule in flags:
-      if graph.config.projectIsStdin: s = stdin.llStreamOpen
-      elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput)
-    discard processModule(graph, result, idGeneratorFromModule(result), s)
-  if result == nil:
-    var cachedModules: seq[FileIndex]
-    result = moduleFromRodFile(graph, fileIdx, cachedModules)
-    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-    if result == nil:
-      result = newModule(graph, fileIdx)
-      result.flags.incl flags
-      registerModule(graph, result)
-      processModuleAux("import")
-    else:
-      if sfSystemModule in flags:
-        graph.systemModule = result
-      partialInitModule(result, graph, fileIdx, filename)
-    for m in cachedModules:
-      registerModuleById(graph, m)
-      replayStateChanges(graph.packed[m.int].module, graph)
-      replayGenericCacheInformation(graph, m.int)
-  elif graph.isDirty(result):
-    result.flags.excl sfDirty
-    # reset module fields:
-    initStrTables(graph, result)
-    result.ast = nil
-    processModuleAux("import(dirty)")
-    graph.markClientsDirty(fileIdx)
-
-proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
-  # this is called by the semantic checking phase
-  assert graph.config != nil
-  result = compileModule(graph, fileIdx, {}, s)
-  graph.addDep(s, fileIdx)
-  # keep track of import relationships
-  if graph.config.hcrOn:
-    graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx)
-  #if sfSystemModule in result.flags:
-  #  localError(result.info, errAttemptToRedefine, result.name.s)
-  # restore the notes for outer module:
-  graph.config.notes =
-    if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
-    else: graph.config.foreignPackageNotes
-
 proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode =
   result = syntaxes.parseFile(fileIdx, graph.cache, graph.config)
   graph.addDep(s, fileIdx)
   graph.addIncludeDep(s.position.FileIndex, fileIdx)
 
-proc connectCallbacks*(graph: ModuleGraph) =
-  graph.includeFileCallback = includeModule
-  graph.importModuleCallback = importModule
-
-proc compileSystemModule*(graph: ModuleGraph) =
-  if graph.systemModule == nil:
-    connectCallbacks(graph)
-    graph.config.m.systemFileIdx = fileInfoIdx(graph.config,
-        graph.config.libpath / RelativeFile"system.nim")
-    discard graph.compileModule(graph.config.m.systemFileIdx, {sfSystemModule})
-
 proc wantMainModule*(conf: ConfigRef) =
   if conf.projectFull.isEmpty:
     fatal(conf, gCmdLineInfo, "command expects a filename")
   conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt))
 
-proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
-  connectCallbacks(graph)
-  let conf = graph.config
-  wantMainModule(conf)
-  configComplete(graph)
-
-  let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
-  let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
-  conf.projectMainIdx2 = projectFile
-
-  let packSym = getPackage(graph, projectFile)
-  graph.config.mainPackageId = packSym.getPackageId
-  graph.importStack.add projectFile
-
-  if projectFile == systemFileIdx:
-    discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})
-  else:
-    graph.compileSystemModule()
-    discard graph.compileModule(projectFile, {sfMainModule})
-
 proc makeModule*(graph: ModuleGraph; filename: AbsoluteFile): PSym =
   result = graph.newModule(fileInfoIdx(graph.config, filename))
   registerModule(graph, result)

compiler/msgs.nim

@@ -128,6 +128,13 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile): FileIndex =
   var dummy: bool
   result = fileInfoIdx(conf, filename, dummy)
 
+proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile; isKnownFile: var bool): FileIndex =
+  fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), isKnownFile)
+
+proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile): FileIndex =
+  var dummy: bool
+  fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), dummy)
+
 proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo =
   result.fileIndex = fileInfoIdx
   if line < int high(uint16):
@@ -219,7 +226,7 @@ proc setDirtyFile*(conf: ConfigRef; fileIdx: FileIndex; filename: AbsoluteFile)
 
 proc setHash*(conf: ConfigRef; fileIdx: FileIndex; hash: string) =
   assert fileIdx.int32 >= 0
-  when defined(gcArc) or defined(gcOrc):
+  when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc):
     conf.m.fileInfos[fileIdx.int32].hash = hash
   else:
     shallowCopy(conf.m.fileInfos[fileIdx.int32].hash, hash)
@@ -227,7 +234,7 @@ proc setHash*(conf: ConfigRef; fileIdx: FileIndex; hash: string) =
 
 proc getHash*(conf: ConfigRef; fileIdx: FileIndex): string =
   assert fileIdx.int32 >= 0
-  when defined(gcArc) or defined(gcOrc):
+  when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc):
     result = conf.m.fileInfos[fileIdx.int32].hash
   else:
     shallowCopy(result, conf.m.fileInfos[fileIdx.int32].hash)
@@ -504,7 +511,8 @@ proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): s
   conf.toFileLineCol(info) & " " & title & getMessageStr(msg, arg)
 
 proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
-               eh: TErrorHandling, info2: InstantiationInfo, isRaw = false) {.gcsafe, noinline.} =
+               eh: TErrorHandling, info2: InstantiationInfo, isRaw = false,
+               ignoreError = false) {.gcsafe, noinline.} =
   var
     title: string
     color: ForegroundColor
@@ -534,10 +542,11 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
     ignoreMsg = not conf.hasWarn(msg)
     if not ignoreMsg and msg in conf.warningAsErrors:
       title = ErrorTitle
+      color = ErrorColor
     else:
       title = WarningTitle
+      color = WarningColor
     if not ignoreMsg: writeContext(conf, info)
-    color = WarningColor
     inc(conf.warnCounter)
   of hintMin..hintMax:
     sev = Severity.Hint
@@ -568,7 +577,8 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
             " compiler msg initiated here", KindColor,
             KindFormat % $hintMsgOrigin,
             resetStyle, conf.unitSep)
-  handleError(conf, msg, eh, s, ignoreMsg)
+  if not ignoreError:
+    handleError(conf, msg, eh, s, ignoreMsg)
   if msg in fatalMsgs:
     # most likely would have died here but just in case, we restore state
     conf.m.errorOutputs = errorOutputsOld

compiler/nilcheck.nim

@@ -909,7 +909,7 @@ proc infix(ctx: NilCheckerContext, l: PNode, r: PNode, magic: TMagic): PNode =
     else: ""
 
   var cache = newIdentCache()
-  var op = newSym(skVar, cache.getIdent(name), nextSymId ctx.idgen, nil, r.info)
+  var op = newSym(skVar, cache.getIdent(name), ctx.idgen, nil, r.info)
 
   op.magic = magic
   result = nkInfix.newTree(
@@ -920,7 +920,7 @@ proc infix(ctx: NilCheckerContext, l: PNode, r: PNode, magic: TMagic): PNode =
 
 proc prefixNot(ctx: NilCheckerContext, node: PNode): PNode =
   var cache = newIdentCache()
-  var op = newSym(skVar, cache.getIdent("not"), nextSymId ctx.idgen, nil, node.info)
+  var op = newSym(skVar, cache.getIdent("not"), ctx.idgen, nil, node.info)
 
   op.magic = mNot
   result = nkPrefix.newTree(

compiler/nim.cfg

@@ -7,6 +7,7 @@ define:nimcore
 define:nimPreviewFloatRoundtrip
 define:nimPreviewSlimSystem
 define:nimPreviewCstringConversion
+define:nimPreviewProcConversion
 define:nimPreviewRangeDefault
 threads:off
 
@@ -14,7 +15,6 @@ threads:off
 
 @if windows:
   cincludes: "$lib/wrappers/libffi/common"
-  tlsEmulation:off
 @end
 
 define:useStdoutAsStdmsg
@@ -31,6 +31,7 @@ define:useStdoutAsStdmsg
   warning[ObservableStores]:off
 @end
 
+
 @if nimHasWarningAsError:
   warningAsError[GcUnsafe2]:on
 @end
@@ -38,3 +39,7 @@ define:useStdoutAsStdmsg
 @if nimHasWarnUnnamedBreak:
   warningAserror[UnnamedBreak]:on
 @end
+
+@if nimHasWarnBareExcept:
+  warningAserror[BareExcept]:on
+@end

compiler/nim.nim

@@ -12,10 +12,7 @@ import std/[os, strutils, parseopt]
 when defined(nimPreviewSlimSystem):
   import std/assertions
 
-when defined(windows) and not defined(nimKochBootstrap):
-  # remove workaround pending bootstrap >= 1.5.1
-  # refs https://github.com/nim-lang/Nim/issues/18334#issuecomment-867114536
-  # alternative would be to prepend `currentSourcePath.parentDir.quoteShell`
+when defined(windows):
   when defined(gcc):
     when defined(x86):
       {.link: "../icons/nim.res".}
@@ -41,6 +38,15 @@ when defined(profiler) or defined(memProfiler):
   {.hint: "Profiling support is turned on!".}
   import nimprof
 
+proc nimbleLockExists(config: ConfigRef): bool =
+  const nimbleLock = "nimble.lock"
+  let pd = if not config.projectPath.isEmpty: config.projectPath else: AbsoluteDir(getCurrentDir())
+  if optSkipParentConfigFiles notin config.globalOptions:
+    for dir in parentDirs(pd.string, fromRoot=true, inclusive=false):
+      if fileExists(dir / nimbleLock):
+        return true
+  return fileExists(pd.string / nimbleLock)
+
 proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
   var p = parseopt.initOptParser(cmd)
   var argsCount = 0
@@ -74,6 +80,11 @@ proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
         config.arguments.len > 0 and config.cmd notin {cmdTcc, cmdNimscript, cmdCrun}:
       rawMessage(config, errGenerated, errArgsNeedRunOption)
 
+  if config.nimbleLockExists:
+    # disable nimble path if nimble.lock is present.
+    # see https://github.com/nim-lang/nimble/issues/1004
+    disableNimblePath(config)
+
 proc getNimRunExe(conf: ConfigRef): string =
   # xxx consider defining `conf.getConfigVar("nimrun.exe")` to allow users to
   # customize the binary to run the command with, e.g. for custom `nodejs` or `wine`.

compiler/nimblecmd.nim

@@ -10,11 +10,13 @@
 ## Implements some helper procs for Nimble (Nim's package manager) support.
 
 import parseutils, strutils, os, options, msgs, sequtils, lineinfos, pathutils,
-  std/sha1, tables
+  tables
 
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions]
 
+import ../dist/checksums/src/checksums/sha1
+
 proc addPath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
   if not conf.searchPaths.contains(path):
     conf.searchPaths.insert(path, 0)

compiler/nimeval.nim

@@ -9,10 +9,16 @@
 
 ## exposes the Nim VM to clients.
 import
-  ast, astalgo, modules, passes, condsyms,
-  options, sem, llstream, lineinfos, vm,
+  ast, modules, condsyms,
+  options, llstream, lineinfos, vm,
   vmdef, modulegraphs, idents, os, pathutils,
-  passaux, scriptconfig, std/compilesettings
+  scriptconfig, std/[compilesettings, tables]
+
+import pipelines
+
+
+when defined(nimPreviewSlimSystem):
+  import std/[assertions, syncio]
 
 type
   Interpreter* = ref object ## Use Nim as an interpreter with this object
@@ -72,11 +78,14 @@ proc evalScript*(i: Interpreter; scriptStream: PLLStream = nil) =
   assert i != nil
   assert i.mainModule != nil, "no main module selected"
   initStrTables(i.graph, i.mainModule)
+  i.graph.cacheSeqs.clear()
+  i.graph.cacheCounters.clear()
+  i.graph.cacheTables.clear()
   i.mainModule.ast = nil
 
   let s = if scriptStream != nil: scriptStream
           else: llStreamOpen(findFile(i.graph.config, i.scriptName), fmRead)
-  processModule(i.graph, i.mainModule, i.idgen, s)
+  discard processPipelineModule(i.graph, i.mainModule, i.idgen, s)
 
 proc findNimStdLib*(): string =
   ## Tries to find a path to a valid "system.nim" file.
@@ -109,12 +118,10 @@ proc createInterpreter*(scriptName: string;
   var conf = newConfigRef()
   var cache = newIdentCache()
   var graph = newModuleGraph(cache, conf)
-  connectCallbacks(graph)
+  connectPipelineCallbacks(graph)
   initDefines(conf.symbols)
   for define in defines:
     defineSymbol(conf.symbols, define[0], define[1])
-  registerPass(graph, semPass)
-  registerPass(graph, evalPass)
 
   for p in searchPaths:
     conf.searchPaths.add(AbsoluteDir p)
@@ -129,7 +136,8 @@ proc createInterpreter*(scriptName: string;
   if registerOps:
     vm.registerAdditionalOps() # Required to register parts of stdlib modules
   graph.vm = vm
-  graph.compileSystemModule()
+  setPipeLinePass(graph, EvalPass)
+  graph.compilePipelineSystemModule()
   result = Interpreter(mainModule: m, graph: graph, scriptName: scriptName, idgen: idgen)
 
 proc destroyInterpreter*(i: Interpreter) =
@@ -159,13 +167,11 @@ proc runRepl*(r: TLLRepl;
   defineSymbol(conf.symbols, "nimscript")
   if supportNimscript: defineSymbol(conf.symbols, "nimconfig")
   when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
-  registerPass(graph, verbosePass)
-  registerPass(graph, semPass)
-  registerPass(graph, evalPass)
   var m = graph.makeStdinModule()
   incl(m.flags, sfMainModule)
   var idgen = idGeneratorFromModule(m)
 
   if supportNimscript: graph.vm = setupVM(m, cache, "stdin", graph, idgen)
-  graph.compileSystemModule()
-  processModule(graph, m, idgen, llStreamOpenStdIn(r))
+  setPipeLinePass(graph, InterpreterPass)
+  graph.compilePipelineSystemModule()
+  discard processPipelineModule(graph, m, idgen, llStreamOpenStdIn(r))

compiler/nimfix/nimfix.nim

@@ -1,111 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Nimfix is a tool that helps to convert old-style Nimrod code to Nim code.
-
-import strutils, os, parseopt
-import compiler/[options, commands, modules, sem,
-  passes, passaux, linter,
-  msgs, nimconf,
-  extccomp, condsyms,
-  modulegraphs, idents]
-
-const Usage = """
-Nimfix - Tool to patch Nim code
-Usage:
-  nimfix [options] projectfile.nim
-
-Options:
-  --overwriteFiles:on|off          overwrite the original nim files.
-                                   DEFAULT is ON!
-  --wholeProject                   overwrite every processed file.
-  --checkExtern:on|off             style check also extern names
-  --styleCheck:on|off|auto         performs style checking for identifiers
-                                   and suggests an alternative spelling;
-                                   'auto' corrects the spelling.
-  --bestEffort                     try to fix the code even when there
-                                   are errors.
-
-In addition, all command line options of Nim are supported.
-"""
-
-proc mainCommand =
-  registerPass verbosePass
-  registerPass semPass
-  conf.setCmd cmdNimfix
-  searchPaths.add options.libpath
-  if gProjectFull.len != 0:
-    # current path is always looked first for modules
-    searchPaths.insert(gProjectPath, 0)
-
-  compileProject(newModuleGraph(), newIdentCache())
-  pretty.overwriteFiles()
-
-proc processCmdLine*(pass: TCmdLinePass, cmd: string, config: ConfigRef) =
-  var p = parseopt.initOptParser(cmd)
-  var argsCount = 0
-  gOnlyMainfile = true
-  while true:
-    parseopt.next(p)
-    case p.kind
-    of cmdEnd: break
-    of cmdLongoption, cmdShortOption:
-      case p.key.normalize
-      of "overwritefiles":
-        case p.val.normalize
-        of "on": gOverWrite = true
-        of "off": gOverWrite = false
-        else: localError(gCmdLineInfo, errOnOrOffExpected)
-      of "checkextern":
-        case p.val.normalize
-        of "on": gCheckExtern = true
-        of "off": gCheckExtern = false
-        else: localError(gCmdLineInfo, errOnOrOffExpected)
-      of "stylecheck":
-        case p.val.normalize
-        of "off": gStyleCheck = StyleCheck.None
-        of "on": gStyleCheck = StyleCheck.Warn
-        of "auto": gStyleCheck = StyleCheck.Auto
-        else: localError(gCmdLineInfo, errOnOrOffExpected)
-      of "wholeproject": gOnlyMainfile = false
-      of "besteffort": msgs.gErrorMax = high(int) # don't stop after first error
-      else:
-        processSwitch(pass, p, config)
-    of cmdArgument:
-      options.gProjectName = unixToNativePath(p.key)
-      # if processArgument(pass, p, argsCount): break
-
-proc handleCmdLine(config: ConfigRef) =
-  if paramCount() == 0:
-    stdout.writeLine(Usage)
-  else:
-    processCmdLine(passCmd1, "", config)
-    if gProjectName != "":
-      try:
-        gProjectFull = canonicalizePath(gProjectName)
-      except OSError:
-        gProjectFull = gProjectName
-      var p = splitFile(gProjectFull)
-      gProjectPath = p.dir
-      gProjectName = p.name
-    else:
-      gProjectPath = getCurrentDir()
-    loadConfigs(DefaultConfig, config) # load all config files
-    # now process command line arguments again, because some options in the
-    # command line can overwrite the config file's settings
-    extccomp.initVars()
-    processCmdLine(passCmd2, "", config)
-    mainCommand()
-
-when compileOption("gc", "refc"):
-  GC_disableMarkAndSweep()
-
-condsyms.initDefines()
-defineSymbol "nimfix"
-handleCmdline newConfigRef()

compiler/nimfix/nimfix.nim.cfg

@@ -1,17 +0,0 @@
-# Special configuration file for the Nim project
-# gc:markAndSweep
-
-hint[XDeclaredButNotUsed]:off
-path:"$projectPath/.."
-
-path:"$lib/packages/docutils"
-path:"$nim"
-
-define:useStdoutAsStdmsg
-symbol:nimfix
-define:nimfix
-
-cs:partial
-#define:useNodeIds
-define:booting
-define:noDocgen

compiler/nimfix/prettybase.nim

@@ -1,45 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-import strutils except Letters
-import ".." / [ast, msgs, lineinfos, idents, options, linter]
-
-proc replaceDeprecated*(conf: ConfigRef; info: TLineInfo; oldSym, newSym: PIdent) =
-  let line = sourceLine(conf, info)
-  var first = min(info.col.int, line.len)
-  if first < 0: return
-  #inc first, skipIgnoreCase(line, "proc ", first)
-  while first > 0 and line[first-1] in Letters: dec first
-  if first < 0: return
-  if line[first] == '`': inc first
-
-  let last = first+identLen(line, first)-1
-  if cmpIgnoreStyle(line[first..last], oldSym.s) == 0:
-    var x = line.substr(0, first-1) & newSym.s & line.substr(last+1)
-    when defined(gcArc) or defined(gcOrc):
-      conf.m.fileInfos[info.fileIndex.int32].lines[info.line.int-1] = move x
-    else:
-      system.shallowCopy(conf.m.fileInfos[info.fileIndex.int32].lines[info.line.int-1], x)
-    conf.m.fileInfos[info.fileIndex.int32].dirty = true
-    #if newSym.s == "File": writeStackTrace()
-
-proc replaceDeprecated*(conf: ConfigRef; info: TLineInfo; oldSym, newSym: PSym) =
-  replaceDeprecated(conf, info, oldSym.name, newSym.name)
-
-proc replaceComment*(conf: ConfigRef; info: TLineInfo) =
-  let line = sourceLine(conf, info)
-  var first = info.col.int
-  if line[first] != '#': inc first
-
-  var x = line.substr(0, first-1) & "discard " & line.substr(first+1).escape
-  when defined(gcArc) or defined(gcOrc):
-    conf.m.fileInfos[info.fileIndex.int32].lines[info.line.int-1] = move x
-  else:
-    system.shallowCopy(conf.m.fileInfos[info.fileIndex.int32].lines[info.line.int-1], x)
-  conf.m.fileInfos[info.fileIndex.int32].dirty = true

compiler/nodejs.nim

@@ -7,4 +7,4 @@ proc findNodeJs*(): string {.inline.} =
     result = findExe("node")
   if result.len == 0:
     echo "Please install NodeJS first, see https://nodejs.org/en/download"
-    raise newException(IOError, "NodeJS not found in PATH: " & result)
+    raise newException(IOError, "NodeJS not found in PATH")

compiler/nversion.nim

@@ -12,6 +12,8 @@
 
 const
   MaxSetElements* = 1 shl 16  # (2^16) to support unicode character sets?
+  DefaultSetElements* = 1 shl 8
+    ## assumed set element count when using int literals
   VersionAsString* = system.NimVersion
   RodFileVersion* = "1223"       # modify this if the rod-format changes!
 

compiler/optimizer.nim

@@ -16,6 +16,8 @@ import
 
 from trees import exprStructuralEquivalent
 
+import std/strutils
+
 const
   nfMarkForDeletion = nfNone # faster than a lookup table
 
@@ -110,16 +112,17 @@ proc analyse(c: var Con; b: var BasicBlock; n: PNode) =
     var reverse = false
     if n[0].kind == nkSym:
       let s = n[0].sym
-      if s.magic == mWasMoved:
+      let name = s.name.s.normalize
+      if name == "=wasmoved":
         b.wasMovedLocs.add n
         special = true
-      elif s.name.s == "=destroy":
+      elif name == "=destroy":
         if c.inFinally > 0 and (b.hasReturn or b.hasBreak):
           discard "cannot optimize away the destructor"
         else:
           c.wasMovedDestroyPair b, n
         special = true
-      elif s.name.s == "=sink":
+      elif name == "=sink":
         reverse = true
 
     if not special:

compiler/options.nim

@@ -24,7 +24,7 @@ const
   useEffectSystem* = true
   useWriteTracking* = false
   hasFFI* = defined(nimHasLibFFI)
-  copyrightYear* = "2022"
+  copyrightYear* = "2023"
 
   nimEnableCovariance* = defined(nimEnableCovariance)
 
@@ -49,6 +49,7 @@ type                          # please make sure we have under 32 options
     optSinkInference          # 'sink T' inference
     optCursorInference
     optImportHidden
+    optQuirky
 
   TOptions* = set[TOption]
   TGlobalOption* = enum
@@ -78,6 +79,8 @@ type                          # please make sure we have under 32 options
     optThreadAnalysis,        # thread analysis pass
     optTlsEmulation,          # thread var emulation turned on
     optGenIndex               # generate index file for documentation;
+    optGenIndexOnly           # generate only index file for documentation
+    optNoImportdoc            # disable loading external documentation files
     optEmbedOrigSrc           # embed the original source in the generated code
                               # also: generate header file
     optIdeDebug               # idetools: debug mode
@@ -100,12 +103,11 @@ type                          # please make sure we have under 32 options
     optBenchmarkVM            # Enables cpuTime() in the VM
     optProduceAsm             # produce assembler code
     optPanics                 # turn panics (sysFatal) into a process termination
-    optNimV1Emulation         # emulate Nim v1.0
-    optNimV12Emulation        # emulate Nim v1.2
-    optNimV16Emulation        # emulate Nim v1.6
     optSourcemap
     optProfileVM              # enable VM profiler
     optEnableDeepCopy         # ORC specific: enable 'deepcopy' for all types.
+    optShowNonExportedFields  # for documentation: show fields that are not exported
+    optJsBigInt64             # use bigints for 64-bit integers in JS
 
   TGlobalOptions* = set[TGlobalOption]
 
@@ -165,7 +167,6 @@ type
     cmdInteractive # start interactive session
     cmdNop
     cmdJsonscript # compile a .json build file
-    cmdNimfix
     # old unused: cmdInterpret, cmdDef: def feature (find definition for IDEs)
 
 const
@@ -183,6 +184,7 @@ type
     gcRegions = "regions"
     gcArc = "arc"
     gcOrc = "orc"
+    gcAtomicArc = "atomicArc"
     gcMarkAndSweep = "markAndSweep"
     gcHooks = "hooks"
     gcRefc = "refc"
@@ -193,7 +195,7 @@ type
   IdeCmd* = enum
     ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideChkFile, ideMod,
     ideHighlight, ideOutline, ideKnown, ideMsg, ideProject, ideGlobalSymbols,
-    ideRecompile, ideChanged, ideType, ideDeclaration
+    ideRecompile, ideChanged, ideType, ideDeclaration, ideExpand
 
   Feature* = enum  ## experimental features; DO NOT RENAME THESE!
     dotOperators,
@@ -231,6 +233,7 @@ type
       ## are not anymore.
     laxEffects
       ## Lax effects system prior to Nim 2.0.
+    verboseTypeMismatch
 
   SymbolFilesOption* = enum
     disabledSf, writeOnlySf, readOnlySf, v2Sf, stressTest
@@ -275,6 +278,9 @@ type
     scope*, localUsages*, globalUsages*: int # more usages is better
     tokenLen*: int
     version*: int
+    endLine*: uint16
+    endCol*: int
+
   Suggestions* = seq[Suggest]
 
   ProfileInfo* = object
@@ -328,6 +334,7 @@ type
 
     cppDefines*: HashSet[string] # (*)
     headerFile*: string
+    nimbasePattern*: string # pattern to find nimbase.h
     features*: set[Feature]
     legacyFeatures*: set[LegacyFeature]
     arguments*: string ## the arguments to be passed to the program that
@@ -405,6 +412,12 @@ type
     nimMainPrefix*: string
     vmProfileData*: ProfileData
 
+    expandProgress*: bool
+    expandLevels*: int
+    expandNodeResult*: string
+    expandPosition*: TLineInfo
+
+
 proc parseNimVersion*(a: string): NimVer =
   # could be moved somewhere reusable
   if a.len > 0:
@@ -467,7 +480,8 @@ const
     optBoundsCheck, optOverflowCheck, optAssert, optWarns, optRefCheck,
     optHints, optStackTrace, optLineTrace, # consider adding `optStackTraceMsgs`
     optTrMacros, optStyleCheck, optCursorInference}
-  DefaultGlobalOptions* = {optThreadAnalysis, optExcessiveStackTrace}
+  DefaultGlobalOptions* = {optThreadAnalysis, optExcessiveStackTrace,
+    optJsBigInt64}
 
 proc getSrcTimestamp(): DateTime =
   try:
@@ -609,7 +623,7 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool =
                             osQnx, osAtari, osAix,
                             osHaiku, osVxWorks, osSolaris, osNetbsd,
                             osFreebsd, osOpenbsd, osDragonfly, osMacosx, osIos,
-                            osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos, osZephyr}
+                            osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos, osZephyr, osNuttX}
     of "linux":
       result = conf.target.targetOS in {osLinux, osAndroid}
     of "bsd":
@@ -631,6 +645,8 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool =
       result = conf.target.targetOS == osFreeRTOS
     of "zephyr":
       result = conf.target.targetOS == osZephyr
+    of "nuttx":
+      result = conf.target.targetOS == osNuttX
     of "littleendian": result = CPU[conf.target.targetCPU].endian == littleEndian
     of "bigendian": result = CPU[conf.target.targetCPU].endian == bigEndian
     of "cpu8": result = CPU[conf.target.targetCPU].bit == 8
@@ -705,22 +721,24 @@ proc getPrefixDir*(conf: ConfigRef): AbsoluteDir =
   ##  clone or using installed nim, so that these exist: `result/doc/advopt.txt`
   ## and `result/lib/system.nim`
   if not conf.prefixDir.isEmpty: result = conf.prefixDir
-  else: result = AbsoluteDir splitPath(getAppDir()).head
+  else:
+    let binParent = AbsoluteDir splitPath(getAppDir()).head
+    when defined(posix):
+      if binParent == AbsoluteDir"/usr":
+        result = AbsoluteDir"/usr/lib/nim"
+      elif binParent == AbsoluteDir"/usr/local":
+        result = AbsoluteDir"/usr/local/lib/nim"
+      else:
+        result = binParent
+    else:
+      result = binParent
 
 proc setDefaultLibpath*(conf: ConfigRef) =
   # set default value (can be overwritten):
   if conf.libpath.isEmpty:
     # choose default libpath:
     var prefix = getPrefixDir(conf)
-    when defined(posix):
-      if prefix == AbsoluteDir"/usr":
-        conf.libpath = AbsoluteDir"/usr/lib/nim"
-      elif prefix == AbsoluteDir"/usr/local":
-        conf.libpath = AbsoluteDir"/usr/local/lib/nim"
-      else:
-        conf.libpath = prefix / RelativeDir"lib"
-    else:
-      conf.libpath = prefix / RelativeDir"lib"
+    conf.libpath = prefix / RelativeDir"lib"
 
     # Special rule to support other tools (nimble) which import the compiler
     # modules and make use of them.
@@ -850,14 +868,6 @@ template patchModule(conf: ConfigRef) {.dirty.} =
       let ov = conf.moduleOverrides[key]
       if ov.len > 0: result = AbsoluteFile(ov)
 
-when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo):
-  proc isRelativeTo(path, base: string): bool =
-    # pending #13212 use os.isRelativeTo
-    let path = path.normalizedPath
-    let base = base.normalizedPath
-    let ret = relativePath(path, base)
-    result = path.len > 0 and not ret.startsWith ".."
-
 const stdlibDirs* = [
   "pure", "core", "arch",
   "pure/collections",
@@ -902,6 +912,7 @@ proc findFile*(conf: ConfigRef; f: string; suppressStdlib = false): AbsoluteFile
 proc findModule*(conf: ConfigRef; modulename, currentModule: string): AbsoluteFile =
   # returns path to module
   var m = addFileExt(modulename, NimExt)
+  var hasRelativeDot = false
   if m.startsWith(pkgPrefix):
     result = findFile(conf, m.substr(pkgPrefix.len), suppressStdlib = true)
   else:
@@ -915,7 +926,11 @@ proc findModule*(conf: ConfigRef; modulename, currentModule: string): AbsoluteFi
     else: # If prefixed with std/ why would we add the current module path!
       let currentPath = currentModule.splitFile.dir
       result = AbsoluteFile currentPath / m
-    if not fileExists(result):
+      if m.startsWith('.') and not fileExists(result):
+        result = AbsoluteFile ""
+        hasRelativeDot = true
+
+    if not fileExists(result) and not hasRelativeDot:
       result = findFile(conf, m)
   patchModule(conf)
 
@@ -993,6 +1008,12 @@ proc isDynlibOverride*(conf: ConfigRef; lib: string): bool =
   result = optDynlibOverrideAll in conf.globalOptions or
      conf.dllOverrides.hasKey(lib.canonDynlibName)
 
+proc showNonExportedFields*(conf: ConfigRef) =
+  incl(conf.globalOptions, optShowNonExportedFields)
+
+proc expandDone*(conf: ConfigRef): bool =
+  result = conf.ideCmd == ideExpand and conf.expandLevels == 0 and conf.expandProgress
+
 proc parseIdeCmd*(s: string): IdeCmd =
   case s:
   of "sug": ideSug
@@ -1032,6 +1053,7 @@ proc `$`*(c: IdeCmd): string =
   of ideProject: "project"
   of ideGlobalSymbols: "globalSymbols"
   of ideDeclaration: "declaration"
+  of ideExpand: "expand"
   of ideRecompile: "recompile"
   of ideChanged: "changed"
   of ideType: "type"

compiler/packages.nim

@@ -8,7 +8,7 @@
 #
 
 ## Package related procs.
-## 
+##
 ## See Also:
 ## * `packagehandling` for package path handling
 ## * `modulegraphs.getPackage`
@@ -22,7 +22,7 @@ when defined(nimPreviewSlimSystem):
 
 proc getPackage*(conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym =
   ## Return a new package symbol.
-  ## 
+  ##
   ## See Also:
   ## * `modulegraphs.getPackage`
   let
@@ -31,7 +31,7 @@ proc getPackage*(conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym =
     info = newLineInfo(fileIdx, 1, 1)
     pkgName = getPackageName(conf, filename.string)
     pkgIdent = getIdent(cache, pkgName)
-  newSym(skPackage, pkgIdent, ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info)
+  newSym(skPackage, pkgIdent, idGeneratorForPackage(int32(fileIdx)), nil, info)
 
 func getPackageSymbol*(sym: PSym): PSym =
   ## Return the owning package symbol.
@@ -47,7 +47,7 @@ func getPackageId*(sym: PSym): int =
 
 func belongsToProjectPackage*(conf: ConfigRef, sym: PSym): bool =
   ## Return whether the symbol belongs to the project's package.
-  ## 
+  ##
   ## See Also:
   ## * `modulegraphs.belongsToStdlib`
   conf.mainPackageId == sym.getPackageId

compiler/parser.nim

@@ -42,7 +42,7 @@ when isMainModule or defined(nimTestGrammar):
 
     proc checkSameGrammar*() =
       doAssert sameFileContent(newGrammarText, "doc/grammar.txt"),
-              "execute 'nim r compiler.nim' to keep grammar.txt up-to-date"
+              "execute 'nim r compiler/parser.nim' to keep grammar.txt up-to-date"
   else:
     writeGrammarFile("doc/grammar.txt")
     import ".." / tools / grammar_nanny
@@ -301,17 +301,16 @@ proc isRightAssociative(tok: Token): bool {.inline.} =
 proc isUnary(tok: Token): bool =
   ## Check if the given token is a unary operator
   tok.tokType in {tkOpr, tkDotDot} and
-  tok.strongSpaceB == tsNone and
-  tok.strongSpaceA
+  tok.spacing == {tsLeading}
 
 proc checkBinary(p: Parser) {.inline.} =
   ## Check if the current parser token is a binary operator.
   # we don't check '..' here as that's too annoying
   if p.tok.tokType == tkOpr:
-    if p.tok.strongSpaceB == tsTrailing and not p.tok.strongSpaceA:
+    if p.tok.spacing == {tsTrailing}:
       parMessage(p, warnInconsistentSpacing, prettyTok(p.tok))
 
-#| module = stmt ^* (';' / IND{=})
+#| module = complexOrSimpleStmt ^* (';' / IND{=})
 #|
 #| comma = ',' COMMENT?
 #| semicolon = ';' COMMENT?
@@ -321,7 +320,7 @@ proc checkBinary(p: Parser) {.inline.} =
 #| operator =  OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9
 #|          | 'or' | 'xor' | 'and'
 #|          | 'is' | 'isnot' | 'in' | 'notin' | 'of' | 'as' | 'from'
-#|          | 'div' | 'mod' | 'shl' | 'shr' | 'not' | 'static' | '..'
+#|          | 'div' | 'mod' | 'shl' | 'shr' | 'not' | '..'
 #|
 #| prefixOperator = operator
 #|
@@ -354,9 +353,16 @@ proc colcom(p: var Parser, n: PNode) =
 
 const tkBuiltInMagics = {tkType, tkStatic, tkAddr}
 
+template setEndInfo() =
+  when defined(nimsuggest):
+    result.endInfo = TLineInfo(fileIndex: p.lex.fileIdx,
+                     line: p.lex.previousTokenEnd.line,
+                     col: p.lex.previousTokenEnd.col)
+
 proc parseSymbol(p: var Parser, mode = smNormal): PNode =
   #| symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`'
-  #|        | IDENT | KEYW
+  #|        | IDENT | 'addr' | 'type' | 'static'
+  #| symbolOrKeyword = symbol | KEYW
   case p.tok.tokType
   of tkSymbol:
     result = newIdentNodeP(p.tok.ident, p)
@@ -406,6 +412,7 @@ proc parseSymbol(p: var Parser, mode = smNormal): PNode =
     # if it is a keyword:
     #if not isKeyword(p.tok.tokType): getTok(p)
     result = p.emptyNode
+  setEndInfo()
 
 proc equals(p: var Parser, a: PNode): PNode =
   if p.tok.tokType == tkEquals:
@@ -429,7 +436,8 @@ proc colonOrEquals(p: var Parser, a: PNode): PNode =
     result = equals(p, a)
 
 proc exprColonEqExpr(p: var Parser): PNode =
-  #| exprColonEqExpr = expr (':'|'=' expr)?
+  #| exprColonEqExpr = expr ((':'|'=') expr
+  #|                        / doBlock extraPostExprBlock*)?
   var a = parseExpr(p)
   if p.tok.tokType == tkDo:
     result = postExprBlocks(p, a)
@@ -437,7 +445,8 @@ proc exprColonEqExpr(p: var Parser): PNode =
     result = colonOrEquals(p, a)
 
 proc exprEqExpr(p: var Parser): PNode =
-  #| exprEqExpr = expr ('=' expr)?
+  #| exprEqExpr = expr ('=' expr
+  #|                   / doBlock extraPostExprBlock*)?
   var a = parseExpr(p)
   if p.tok.tokType == tkDo:
     result = postExprBlocks(p, a)
@@ -508,7 +517,7 @@ proc dotExpr(p: var Parser, a: PNode): PNode =
   optInd(p, result)
   result.add(a)
   result.add(parseSymbol(p, smAfterDot))
-  if p.tok.tokType == tkBracketLeColon and not p.tok.strongSpaceA:
+  if p.tok.tokType == tkBracketLeColon and tsLeading notin p.tok.spacing:
     var x = newNodeI(nkBracketExpr, p.parLineInfo)
     # rewrite 'x.y[:z]()' to 'y[z](x)'
     x.add result[1]
@@ -517,7 +526,7 @@ proc dotExpr(p: var Parser, a: PNode): PNode =
     var y = newNodeI(nkCall, p.parLineInfo)
     y.add x
     y.add result[0]
-    if p.tok.tokType == tkParLe and not p.tok.strongSpaceA:
+    if p.tok.tokType == tkParLe and tsLeading notin p.tok.spacing:
       exprColonEqExprListAux(p, tkParRi, y)
     result = y
 
@@ -532,7 +541,7 @@ proc dotLikeExpr(p: var Parser, a: PNode): PNode =
   result.add(parseSymbol(p, smAfterDot))
 
 proc qualifiedIdent(p: var Parser): PNode =
-  #| qualifiedIdent = symbol ('.' optInd symbol)?
+  #| qualifiedIdent = symbol ('.' optInd symbolOrKeyword)?
   result = parseSymbol(p)
   if p.tok.tokType == tkDot: result = dotExpr(p, result)
 
@@ -577,6 +586,7 @@ proc parseCast(p: var Parser): PNode =
     result.add(exprColonEqExpr(p))
   optPar(p)
   eat(p, tkParRi)
+  setEndInfo()
 
 proc setBaseFlags(n: PNode, base: NumericalBase) =
   case base
@@ -599,6 +609,7 @@ proc parseGStrLit(p: var Parser, a: PNode): PNode =
     getTok(p)
   else:
     result = a
+  setEndInfo()
 
 proc complexOrSimpleStmt(p: var Parser): PNode
 proc simpleExpr(p: var Parser, mode = pmNormal): PNode
@@ -638,7 +649,8 @@ proc parsePar(p: var Parser): PNode =
   #|           ( &parKeyw (ifExpr / complexOrSimpleStmt) ^+ ';'
   #|           | ';' (ifExpr / complexOrSimpleStmt) ^+ ';'
   #|           | pragmaStmt
-  #|           | simpleExpr ( ('=' expr (';' (ifExpr / complexOrSimpleStmt) ^+ ';' )? )
+  #|           | simpleExpr ( (doBlock extraPostExprBlock*)
+  #|                        | ('=' expr (';' (ifExpr / complexOrSimpleStmt) ^+ ';' )? )
   #|                        | (':' expr (',' exprColonEqExpr     ^+ ',' )? ) ) )
   #|           optPar ')'
   #
@@ -703,6 +715,7 @@ proc parsePar(p: var Parser): PNode =
           skipComment(p, a)
   optPar(p)
   eat(p, tkParRi)
+  setEndInfo()
 
 proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode =
   #| literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT
@@ -859,8 +872,8 @@ proc isDotLike(tok: Token): bool =
 proc primarySuffix(p: var Parser, r: PNode,
                    baseIndent: int, mode: PrimaryMode): PNode =
   #| primarySuffix = '(' (exprColonEqExpr comma?)* ')'
-  #|       | '.' optInd symbol ('[:' exprList ']' ( '(' exprColonEqExpr ')' )?)? generalizedLit?
-  #|       | DOTLIKEOP optInd symbol generalizedLit?
+  #|       | '.' optInd symbolOrKeyword ('[:' exprList ']' ( '(' exprColonEqExpr ')' )?)? generalizedLit?
+  #|       | DOTLIKEOP optInd symbolOrKeyword generalizedLit?
   #|       | '[' optInd exprColonEqExprList optPar ']'
   #|       | '{' optInd exprColonEqExprList optPar '}'
   # XXX strong spaces need to be reflected above
@@ -872,7 +885,7 @@ proc primarySuffix(p: var Parser, r: PNode,
     case p.tok.tokType
     of tkParLe:
       # progress guaranteed
-      if p.tok.strongSpaceA:
+      if tsLeading in p.tok.spacing:
         result = commandExpr(p, result, mode)
         break
       result = namedParams(p, result, nkCall, tkParRi)
@@ -884,13 +897,13 @@ proc primarySuffix(p: var Parser, r: PNode,
       result = parseGStrLit(p, result)
     of tkBracketLe:
       # progress guaranteed
-      if p.tok.strongSpaceA:
+      if tsLeading in p.tok.spacing:
         result = commandExpr(p, result, mode)
         break
       result = namedParams(p, result, nkBracketExpr, tkBracketRi)
     of tkCurlyLe:
       # progress guaranteed
-      if p.tok.strongSpaceA:
+      if tsLeading in p.tok.spacing:
         result = commandExpr(p, result, mode)
         break
       result = namedParams(p, result, nkCurlyExpr, tkCurlyRi)
@@ -941,6 +954,7 @@ proc parseOperators(p: var Parser, headNode: PNode,
     a.add(b)
     result = a
     opPrec = getPrecedence(p.tok)
+  setEndInfo()
 
 proc simpleExprAux(p: var Parser, limit: int, mode: PrimaryMode): PNode =
   var mode = mode
@@ -990,10 +1004,11 @@ proc parsePragma(p: var Parser): PNode =
   when defined(nimpretty):
     dec p.em.doIndentMore
     dec p.em.keepIndents
+  setEndInfo()
 
 proc identVis(p: var Parser; allowDot=false): PNode =
   #| identVis = symbol OPR?  # postfix position
-  #| identVisDot = symbol '.' optInd symbol OPR?
+  #| identVisDot = symbol '.' optInd symbolOrKeyword OPR?
   var a = parseSymbol(p)
   if p.tok.tokType == tkOpr:
     when defined(nimpretty):
@@ -1058,6 +1073,7 @@ proc parseIdentColonEquals(p: var Parser, flags: DeclaredIdentFlags): PNode =
     result.add(parseExpr(p))
   else:
     result.add(newNodeP(nkEmpty, p))
+  setEndInfo()
 
 proc parseTuple(p: var Parser, indentAllowed = false): PNode =
   #| tupleTypeBracket = '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']'
@@ -1102,6 +1118,7 @@ proc parseTuple(p: var Parser, indentAllowed = false): PNode =
     parMessage(p, errGenerated, "the syntax for tuple types is 'tuple[...]', not 'tuple(...)'")
   else:
     result = newNodeP(nkTupleClassTy, p)
+  setEndInfo()
 
 proc parseParamList(p: var Parser, retColon = true): PNode =
   #| paramList = '(' declColonEquals ^* (comma/semicolon) ')'
@@ -1150,6 +1167,7 @@ proc parseParamList(p: var Parser, retColon = true): PNode =
   when defined(nimpretty):
     dec p.em.doIndentMore
     dec p.em.keepIndents
+  setEndInfo()
 
 proc optPragmas(p: var Parser): PNode =
   if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)):
@@ -1170,6 +1188,7 @@ proc parseDoBlock(p: var Parser; info: TLineInfo): PNode =
     result = newProcNode(nkDo, info,
       body = result, params = params, name = p.emptyNode, pattern = p.emptyNode,
       genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode)
+  setEndInfo()
 
 proc parseProcExpr(p: var Parser; isExpr: bool; kind: TNodeKind): PNode =
   #| routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)?
@@ -1181,17 +1200,22 @@ proc parseProcExpr(p: var Parser; isExpr: bool; kind: TNodeKind): PNode =
   let pragmas = optPragmas(p)
   if p.tok.tokType == tkEquals and isExpr:
     getTok(p)
-    skipComment(p, result)
-    result = newProcNode(kind, info, body = parseStmt(p),
+    result = newProcNode(kind, info, body = p.emptyNode,
       params = params, name = p.emptyNode, pattern = p.emptyNode,
       genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode)
+    skipComment(p, result)
+    result[bodyPos] = parseStmt(p)
   else:
     result = newNodeI(if kind == nkIteratorDef: nkIteratorTy else: nkProcTy, info)
-    if hasSignature:
-      result.add(params)
+    if hasSignature or pragmas.kind != nkEmpty:
+      if hasSignature:
+        result.add(params)
+      else: # pragmas but no param list, implies typeclass with pragmas
+        result.add(p.emptyNode)
       if kind == nkFuncDef:
         parMessage(p, "func keyword is not allowed in type descriptions, use proc with {.noSideEffect.} pragma instead")
       result.add(pragmas)
+  setEndInfo()
 
 proc isExprStart(p: Parser): bool =
   case p.tok.tokType
@@ -1211,6 +1235,7 @@ proc parseSymbolList(p: var Parser, result: PNode) =
     if p.tok.tokType != tkComma: break
     getTok(p)
     optInd(p, s)
+  setEndInfo()
 
 proc parseTypeDescKAux(p: var Parser, kind: TNodeKind,
                        mode: PrimaryMode): PNode =
@@ -1239,6 +1264,7 @@ proc parseTypeDescKAux(p: var Parser, kind: TNodeKind,
     parseSymbolList(p, list)
   if mode == pmTypeDef and not isTypedef:
     result = parseOperators(p, result, -1, mode)
+  setEndInfo()
 
 proc parseVarTuple(p: var Parser): PNode
 
@@ -1264,6 +1290,7 @@ proc parseFor(p: var Parser): PNode =
   result.add(parseExpr(p))
   colcom(p, result)
   result.add(parseStmt(p))
+  setEndInfo()
 
 template nimprettyDontTouch(body) =
   when defined(nimpretty):
@@ -1302,6 +1329,7 @@ proc parseExpr(p: var Parser): PNode =
     nimprettyDontTouch:
       result = parseTry(p, isExpr=true)
   else: result = simpleExpr(p)
+  setEndInfo()
 
 proc parseEnum(p: var Parser): PNode
 proc parseObject(p: var Parser): PNode
@@ -1311,7 +1339,7 @@ proc primary(p: var Parser, mode: PrimaryMode): PNode =
   #| simplePrimary = SIGILLIKEOP? identOrLiteral primarySuffix*
   #| commandStart = &('`'|IDENT|literal|'cast'|'addr'|'type'|'var'|'out'|
   #|                  'static'|'enum'|'tuple'|'object'|'proc')
-  #| primary = simplePrimary (commandStart expr)
+  #| primary = simplePrimary (commandStart expr (doBlock extraPostExprBlock*)?)?
   #|         / operatorB primary
   #|         / routineExpr
   #|         / rawTypeDesc
@@ -1368,7 +1396,7 @@ proc binaryNot(p: var Parser; a: PNode): PNode =
     let notOpr = newIdentNodeP(p.tok.ident, p)
     getTok(p)
     optInd(p, notOpr)
-    let b = parseExpr(p)
+    let b = primary(p, pmTypeDesc)
     result = newNodeP(nkInfix, p)
     result.add notOpr
     result.add a
@@ -1379,8 +1407,8 @@ proc binaryNot(p: var Parser; a: PNode): PNode =
 proc parseTypeDesc(p: var Parser, fullExpr = false): PNode =
   #| rawTypeDesc = (tupleType | routineType | 'enum' | 'object' |
   #|                 ('var' | 'out' | 'ref' | 'ptr' | 'distinct') typeDesc?)
-  #|                 ('not' expr)?
-  #| typeDescExpr = (routineType / simpleExpr) ('not' expr)?
+  #|                 ('not' primary)?
+  #| typeDescExpr = (routineType / simpleExpr) ('not' primary)?
   #| typeDesc = rawTypeDesc / typeDescExpr
   newlineWasSplitting(p)
   if fullExpr:
@@ -1411,12 +1439,13 @@ proc parseTypeDesc(p: var Parser, fullExpr = false): PNode =
     else:
       result = simpleExpr(p, pmTypeDesc)
   result = binaryNot(p, result)
+  setEndInfo()
 
 proc parseTypeDefValue(p: var Parser): PNode =
   #| typeDefValue = ((tupleDecl | enumDecl | objectDecl | conceptDecl |
   #|                  ('ref' | 'ptr' | 'distinct') (tupleDecl | objectDecl))
-  #|                / (simpleExpr (exprEqExpr ^+ comma postExprBlocks)?))
-  #|                ('not' expr)?
+  #|                / (simpleExpr (exprEqExpr ^+ comma postExprBlocks?)?))
+  #|                ('not' primary)?
   case p.tok.tokType
   of tkTuple: result = parseTuple(p, true)
   of tkRef: result = parseTypeDescKAux(p, nkRefTy, pmTypeDef)
@@ -1441,6 +1470,7 @@ proc parseTypeDefValue(p: var Parser): PNode =
           result.add(commandParam(p, isFirstParam, pmTypeDef))
       result = postExprBlocks(p, result)
   result = binaryNot(p, result)
+  setEndInfo()
 
 proc makeCall(n: PNode): PNode =
   ## Creates a call if the given node isn't already a call.
@@ -1451,12 +1481,13 @@ proc makeCall(n: PNode): PNode =
     result.add n
 
 proc postExprBlocks(p: var Parser, x: PNode): PNode =
-  #| postExprBlocks = ':' stmt? ( IND{=} doBlock
-  #|                            | IND{=} 'of' exprList ':' stmt
-  #|                            | IND{=} 'elif' expr ':' stmt
-  #|                            | IND{=} 'except' optionalExprList ':' stmt
-  #|                            | IND{=} 'finally' ':' stmt
-  #|                            | IND{=} 'else' ':' stmt )*
+  #| extraPostExprBlock = ( IND{=} doBlock
+  #|                      | IND{=} 'of' exprList ':' stmt
+  #|                      | IND{=} 'elif' expr ':' stmt
+  #|                      | IND{=} 'except' optionalExprList ':' stmt
+  #|                      | IND{=} 'finally' ':' stmt
+  #|                      | IND{=} 'else' ':' stmt )
+  #| postExprBlocks = (doBlock / ':' (extraPostExprBlock / stmt)) extraPostExprBlock*
   result = x
   if p.tok.indent >= 0: return
 
@@ -1473,7 +1504,7 @@ proc postExprBlocks(p: var Parser, x: PNode): PNode =
     result = makeCall(result)
     getTok(p)
     skipComment(p, result)
-    if p.tok.tokType notin {tkOf, tkElif, tkElse, tkExcept}:
+    if not (p.tok.tokType in {tkOf, tkElif, tkElse, tkExcept, tkFinally} and sameInd(p)):
       var stmtList = newNodeP(nkStmtList, p)
       stmtList.add parseStmt(p)
       # to keep backwards compatibility (see tests/vm/tstringnil)
@@ -1561,6 +1592,7 @@ proc parseExprStmt(p: var Parser): PNode =
     else:
       result = a
     result = postExprBlocks(p, result)
+  setEndInfo()
 
 proc parseModuleName(p: var Parser, kind: TNodeKind): PNode =
   result = parseExpr(p)
@@ -1572,6 +1604,7 @@ proc parseModuleName(p: var Parser, kind: TNodeKind): PNode =
       getTok(p)
       result.add(a)
       result.add(parseExpr(p))
+  setEndInfo()
 
 proc parseImport(p: var Parser, kind: TNodeKind): PNode =
   #| importStmt = 'import' optInd expr
@@ -1600,6 +1633,7 @@ proc parseImport(p: var Parser, kind: TNodeKind): PNode =
       getTok(p)
       optInd(p, a)
   #expectNl(p)
+  setEndInfo()
 
 proc parseIncludeStmt(p: var Parser): PNode =
   #| includeStmt = 'include' optInd expr ^+ comma
@@ -1616,6 +1650,7 @@ proc parseIncludeStmt(p: var Parser): PNode =
     getTok(p)
     optInd(p, a)
   #expectNl(p)
+  setEndInfo()
 
 proc parseFromStmt(p: var Parser): PNode =
   #| fromStmt = 'from' expr 'import' optInd expr (comma expr)*
@@ -1636,6 +1671,7 @@ proc parseFromStmt(p: var Parser): PNode =
     getTok(p)
     optInd(p, a)
   #expectNl(p)
+  setEndInfo()
 
 proc parseReturnOrRaise(p: var Parser, kind: TNodeKind): PNode =
   #| returnStmt = 'return' optInd expr?
@@ -1657,6 +1693,7 @@ proc parseReturnOrRaise(p: var Parser, kind: TNodeKind): PNode =
     var e = parseExpr(p)
     e = postExprBlocks(p, e)
     result.add(e)
+  setEndInfo()
 
 proc parseIfOrWhen(p: var Parser, kind: TNodeKind): PNode =
   #| condStmt = expr colcom stmt COMMENT?
@@ -1681,11 +1718,12 @@ proc parseIfOrWhen(p: var Parser, kind: TNodeKind): PNode =
     colcom(p, branch)
     branch.add(parseStmt(p))
     result.add(branch)
+  setEndInfo()
 
 proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode =
-  #| condExpr = expr colcom expr optInd
-  #|         ('elif' expr colcom expr optInd)*
-  #|          'else' colcom expr
+  #| condExpr = expr colcom stmt optInd
+  #|         ('elif' expr colcom stmt optInd)*
+  #|          'else' colcom stmt
   #| ifExpr = 'if' condExpr
   #| whenExpr = 'when' condExpr
   result = newNodeP(kind, p)
@@ -1705,6 +1743,7 @@ proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode =
     colcom(p, branch)
     branch.add(parseStmt(p))
     result.add(branch)
+  setEndInfo()
 
 proc parseWhile(p: var Parser): PNode =
   #| whileStmt = 'while' expr colcom stmt
@@ -1714,6 +1753,7 @@ proc parseWhile(p: var Parser): PNode =
   result.add(parseExpr(p))
   colcom(p, result)
   result.add(parseStmt(p))
+  setEndInfo()
 
 proc parseCase(p: var Parser): PNode =
   #| ofBranch = 'of' exprList colcom stmt
@@ -1761,6 +1801,7 @@ proc parseCase(p: var Parser): PNode =
 
   if wasIndented:
     p.currInd = oldInd
+  setEndInfo()
 
 proc parseTry(p: var Parser; isExpr: bool): PNode =
   #| tryStmt = 'try' colcom stmt &(IND{=}? 'except'|'finally')
@@ -1770,11 +1811,13 @@ proc parseTry(p: var Parser; isExpr: bool): PNode =
   #|            (optInd 'except' optionalExprList colcom stmt)*
   #|            (optInd 'finally' colcom stmt)?
   result = newNodeP(nkTryStmt, p)
+  let parentIndent = p.currInd # isExpr
   getTok(p)
   colcom(p, result)
   result.add(parseStmt(p))
   var b: PNode = nil
-  while sameOrNoInd(p) or isExpr:
+
+  while sameOrNoInd(p) or (isExpr and parentIndent <= p.tok.indent):
     case p.tok.tokType
     of tkExcept:
       b = newNodeP(nkExceptBranch, p)
@@ -1787,12 +1830,14 @@ proc parseTry(p: var Parser; isExpr: bool): PNode =
     b.add(parseStmt(p))
     result.add(b)
   if b == nil: parMessage(p, "expected 'except'")
+  setEndInfo()
 
 proc parseExceptBlock(p: var Parser, kind: TNodeKind): PNode =
   result = newNodeP(kind, p)
   getTok(p)
   colcom(p, result)
   result.add(parseStmt(p))
+  setEndInfo()
 
 proc parseBlock(p: var Parser): PNode =
   #| blockStmt = 'block' symbol? colcom stmt
@@ -1803,6 +1848,7 @@ proc parseBlock(p: var Parser): PNode =
   else: result.add(parseSymbol(p))
   colcom(p, result)
   result.add(parseStmt(p))
+  setEndInfo()
 
 proc parseStaticOrDefer(p: var Parser; k: TNodeKind): PNode =
   #| staticStmt = 'static' colcom stmt
@@ -1811,6 +1857,7 @@ proc parseStaticOrDefer(p: var Parser; k: TNodeKind): PNode =
   getTok(p)
   colcom(p, result)
   result.add(parseStmt(p))
+  setEndInfo()
 
 proc parseAsm(p: var Parser): PNode =
   #| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLESTR_LIT)
@@ -1827,6 +1874,7 @@ proc parseAsm(p: var Parser): PNode =
     result.add(p.emptyNode)
     return
   getTok(p)
+  setEndInfo()
 
 proc parseGenericParam(p: var Parser): PNode =
   #| genericParam = symbol (comma symbol)* (colon expr)? ('=' optInd expr)?
@@ -1862,6 +1910,7 @@ proc parseGenericParam(p: var Parser): PNode =
     result.add(parseExpr(p))
   else:
     result.add(p.emptyNode)
+  setEndInfo()
 
 proc parseGenericParamList(p: var Parser): PNode =
   #| genericParamList = '[' optInd
@@ -1880,12 +1929,14 @@ proc parseGenericParamList(p: var Parser): PNode =
     skipComment(p, a)
   optPar(p)
   eat(p, tkBracketRi)
+  setEndInfo()
 
 proc parsePattern(p: var Parser): PNode =
   #| pattern = '{' stmt '}'
   eat(p, tkCurlyLe)
   result = parseStmt(p)
   eat(p, tkCurlyRi)
+  setEndInfo()
 
 proc parseRoutine(p: var Parser, kind: TNodeKind): PNode =
   #| indAndComment = (IND{>} COMMENT)? | COMMENT?
@@ -1930,6 +1981,7 @@ proc parseRoutine(p: var Parser, kind: TNodeKind): PNode =
     #else:
     #  assert false, p.lex.config$body.info # avoids hard to track bugs, fail early.
     # Yeah, that worked so well. There IS a bug in this logic, now what?
+  setEndInfo()
 
 proc newCommentStmt(p: var Parser): PNode =
   #| commentStmt = COMMENT
@@ -1965,6 +2017,7 @@ proc parseSection(p: var Parser, kind: TNodeKind,
     result.add(defparser(p))
   else:
     parMessage(p, errIdentifierExpected, p.tok)
+  setEndInfo()
 
 proc parseEnum(p: var Parser): PNode =
   #| enumDecl = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+
@@ -2011,6 +2064,7 @@ proc parseEnum(p: var Parser): PNode =
       break
   if result.len <= 1:
     parMessage(p, errIdentifierExpected, p.tok)
+  setEndInfo()
 
 proc parseObjectPart(p: var Parser): PNode
 proc parseObjectWhen(p: var Parser): PNode =
@@ -2036,6 +2090,7 @@ proc parseObjectWhen(p: var Parser): PNode =
     branch.add(parseObjectPart(p))
     flexComment(p, branch)
     result.add(branch)
+  setEndInfo()
 
 proc parseObjectCase(p: var Parser): PNode =
   #| objectBranch = 'of' exprList colcom objectPart
@@ -2077,6 +2132,7 @@ proc parseObjectCase(p: var Parser): PNode =
     if b.kind == nkElse: break
   if wasIndented:
     p.currInd = oldInd
+  setEndInfo()
 
 proc parseObjectPart(p: var Parser): PNode =
   #| objectPart = IND{>} objectPart^+IND{=} DED
@@ -2109,6 +2165,7 @@ proc parseObjectPart(p: var Parser): PNode =
       result = p.emptyNode
   else:
     result = p.emptyNode
+  setEndInfo()
 
 proc parseObject(p: var Parser): PNode =
   #| objectDecl = 'object' ('of' typeDesc)? COMMENT? objectPart
@@ -2129,6 +2186,7 @@ proc parseObject(p: var Parser): PNode =
     result.add(p.emptyNode)
   else:
     result.add(parseObjectPart(p))
+  setEndInfo()
 
 proc parseTypeClassParam(p: var Parser): PNode =
   let modifier =
@@ -2146,6 +2204,7 @@ proc parseTypeClassParam(p: var Parser): PNode =
     result.add(p.parseSymbol)
   else:
     result = p.parseSymbol
+  setEndInfo()
 
 proc parseTypeClass(p: var Parser): PNode =
   #| conceptParam = ('var' | 'out')? symbol
@@ -2189,6 +2248,7 @@ proc parseTypeClass(p: var Parser): PNode =
     result.add(p.emptyNode)
   else:
     result.add(parseStmt(p))
+  setEndInfo()
 
 proc parseTypeDef(p: var Parser): PNode =
   #|
@@ -2222,15 +2282,22 @@ proc parseTypeDef(p: var Parser): PNode =
   else:
     result.add(p.emptyNode)
   indAndComment(p, result)    # special extension!
+  setEndInfo()
 
 proc parseVarTuple(p: var Parser): PNode =
-  #| varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr
+  #| varTupleLhs = '(' optInd (identWithPragma / varTupleLhs) ^+ comma optPar ')'
+  #| varTuple = varTupleLhs '=' optInd expr
   result = newNodeP(nkVarTuple, p)
   getTok(p)                   # skip '('
   optInd(p, result)
   # progress guaranteed
-  while p.tok.tokType in {tkSymbol, tkAccent}:
-    var a = identWithPragma(p, allowDot=true)
+  while p.tok.tokType in {tkSymbol, tkAccent, tkParLe}:
+    var a: PNode
+    if p.tok.tokType == tkParLe:
+      a = parseVarTuple(p)
+      a.add(p.emptyNode)
+    else:
+      a = identWithPragma(p, allowDot=true)
     result.add(a)
     if p.tok.tokType != tkComma: break
     getTok(p)
@@ -2238,6 +2305,7 @@ proc parseVarTuple(p: var Parser): PNode =
   result.add(p.emptyNode)         # no type desc
   optPar(p)
   eat(p, tkParRi)
+  setEndInfo()
 
 proc parseVariable(p: var Parser): PNode =
   #| colonBody = colcom stmt postExprBlocks?
@@ -2250,6 +2318,7 @@ proc parseVariable(p: var Parser): PNode =
   else: result = parseIdentColonEquals(p, {withPragma, withDot})
   result[^1] = postExprBlocks(p, result[^1])
   indAndComment(p, result)
+  setEndInfo()
 
 proc parseConstant(p: var Parser): PNode =
   #| constant = (varTuple / identWithPragma) (colon typeDesc)? '=' optInd expr indAndComment
@@ -2269,6 +2338,7 @@ proc parseConstant(p: var Parser): PNode =
   result.add(parseExpr(p))
   result[^1] = postExprBlocks(p, result[^1])
   indAndComment(p, result)
+  setEndInfo()
 
 proc parseBind(p: var Parser, k: TNodeKind): PNode =
   #| bindStmt = 'bind' optInd qualifiedIdent ^+ comma
@@ -2284,6 +2354,7 @@ proc parseBind(p: var Parser, k: TNodeKind): PNode =
     getTok(p)
     optInd(p, a)
   #expectNl(p)
+  setEndInfo()
 
 proc parseStmtPragma(p: var Parser): PNode =
   #| pragmaStmt = pragma (':' COMMENT? stmt)?
@@ -2295,6 +2366,7 @@ proc parseStmtPragma(p: var Parser): PNode =
     skipComment(p, result)
     result.add a
     result.add parseStmt(p)
+  setEndInfo()
 
 proc simpleStmt(p: var Parser): PNode =
   #| simpleStmt = ((returnStmt | raiseStmt | yieldStmt | discardStmt | breakStmt
@@ -2437,6 +2509,7 @@ proc parseStmt(p: var Parser): PNode =
           if p.tok.tokType != tkSemiColon: break
           getTok(p)
           if err and p.tok.tokType == tkEof: break
+  setEndInfo()
 
 proc parseAll(p: var Parser): PNode =
   ## Parses the rest of the input stream held by the parser into a PNode.
@@ -2452,9 +2525,10 @@ proc parseAll(p: var Parser): PNode =
       getTok(p)
     if p.tok.indent != 0:
       parMessage(p, errInvalidIndentation)
+  setEndInfo()
 
 proc checkFirstLineIndentation*(p: var Parser) =
-  if p.tok.indent != 0 and p.tok.strongSpaceA:
+  if p.tok.indent != 0 and tsLeading in p.tok.spacing:
     parMessage(p, errInvalidIndentation)
 
 proc parseTopLevelStmt(p: var Parser): PNode =
@@ -2485,6 +2559,7 @@ proc parseTopLevelStmt(p: var Parser): PNode =
       result = complexOrSimpleStmt(p)
       if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok)
       break
+  setEndInfo()
 
 proc parseString*(s: string; cache: IdentCache; config: ConfigRef;
                   filename: string = ""; line: int = 0;
@@ -2496,9 +2571,10 @@ proc parseString*(s: string; cache: IdentCache; config: ConfigRef;
   var stream = llStreamOpen(s)
   stream.lineOffset = line
 
-  var parser: Parser
-  parser.lex.errorHandler = errorHandler
-  openParser(parser, AbsoluteFile filename, stream, cache, config)
+  var p: Parser
+  p.lex.errorHandler = errorHandler
+  openParser(p, AbsoluteFile filename, stream, cache, config)
 
-  result = parser.parseAll
-  closeParser(parser)
+  result = p.parseAll
+  closeParser(p)
+  setEndInfo()

compiler/passes.nim

@@ -14,13 +14,22 @@ import
   options, ast, llstream, msgs,
   idents,
   syntaxes, modulegraphs, reorder,
-  lineinfos, pathutils, packages
+  lineinfos,
+  pipelineutils,
+  modules, pathutils, packages,
+  sem, semdata
+
+import ic/replayer
+
+export skipCodegen, resolveMod, prepareConfigNotes
 
 when defined(nimsuggest):
-  import std/sha1
+  import ../dist/checksums/src/checksums/sha1
 
 when defined(nimPreviewSlimSystem):
-  import std/syncio
+  import std/[syncio, assertions]
+
+import std/tables
 
 type
   TPassData* = tuple[input: PNode, closeOutput: PNode]
@@ -38,12 +47,6 @@ proc makePass*(open: TPassOpen = nil,
   result.process = process
   result.isFrontend = isFrontend
 
-proc skipCodegen*(config: ConfigRef; n: PNode): bool {.inline.} =
-  # can be used by codegen passes to determine whether they should do
-  # something with `n`. Currently, this ignores `n` and uses the global
-  # error count instead.
-  result = config.errorCounter > 0
-
 const
   maxPasses = 10
 
@@ -80,13 +83,6 @@ proc processTopLevelStmt(graph: ModuleGraph, n: PNode, a: var TPassContextArray)
       if isNil(m): return false
   result = true
 
-proc resolveMod(conf: ConfigRef; module, relativeTo: string): FileIndex =
-  let fullPath = findModule(conf, module, relativeTo)
-  if fullPath.isEmpty:
-    result = InvalidFileIdx
-  else:
-    result = fileInfoIdx(conf, fullPath)
-
 proc processImplicits(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind,
                       a: var TPassContextArray; m: PSym) =
   # XXX fixme this should actually be relative to the config file!
@@ -100,23 +96,6 @@ proc processImplicits(graph: ModuleGraph; implicits: seq[string], nodeKind: TNod
       importStmt.add str
       if not processTopLevelStmt(graph, importStmt, a): break
 
-const
-  imperativeCode = {low(TNodeKind)..high(TNodeKind)} - {nkTemplateDef, nkProcDef, nkMethodDef,
-    nkMacroDef, nkConverterDef, nkIteratorDef, nkFuncDef, nkPragma,
-    nkExportStmt, nkExportExceptStmt, nkFromStmt, nkImportStmt, nkImportExceptStmt}
-
-proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
-  # don't be verbose unless the module belongs to the main package:
-  if graph.config.belongsToProjectPackage(module):
-    graph.config.notes = graph.config.mainPackageNotes
-  else:
-    if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes
-    graph.config.notes = graph.config.foreignPackageNotes
-
-proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
-  result = true
-  #module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")
-
 proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
                     stream: PLLStream): bool {.discardable.} =
   if graph.stopCompile(): return true
@@ -143,7 +122,7 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
   while true:
     openParser(p, fileIdx, s, graph.cache, graph.config)
 
-    if not belongsToStdlib(graph, module) or (belongsToStdlib(graph, module) and module.name.s == "distros"):
+    if (not belongsToStdlib(graph, module)) or module.name.s == "distros":
       # XXX what about caching? no processing then? what if I change the
       # modules to include between compilation runs? we'd need to track that
       # in ROD files. I think we should enable this feature only
@@ -153,43 +132,22 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
         processImplicits graph, graph.config.implicitIncludes, nkIncludeStmt, a, module
 
     checkFirstLineIndentation(p)
-    while true:
-      if graph.stopCompile(): break
+    block processCode:
+      if graph.stopCompile(): break processCode
       var n = parseTopLevelStmt(p)
-      if n.kind == nkEmpty: break
-      if (sfSystemModule notin module.flags and
-          ({sfNoForward, sfReorder} * module.flags != {} or
-          codeReordering in graph.config.features)):
-        # read everything, no streaming possible
-        var sl = newNodeI(nkStmtList, n.info)
+      if n.kind == nkEmpty: break processCode
+
+      # read everything, no streaming possible
+      var sl = newNodeI(nkStmtList, n.info)
+      sl.add n
+      while true:
+        var n = parseTopLevelStmt(p)
+        if n.kind == nkEmpty: break
         sl.add n
-        while true:
-          var n = parseTopLevelStmt(p)
-          if n.kind == nkEmpty: break
-          sl.add n
-        if sfReorder in module.flags or codeReordering in graph.config.features:
-          sl = reorder(graph, sl, module)
-        discard processTopLevelStmt(graph, sl, a)
-        break
-      elif n.kind in imperativeCode:
-        # read everything until the next proc declaration etc.
-        var sl = newNodeI(nkStmtList, n.info)
-        sl.add n
-        var rest: PNode = nil
-        while true:
-          var n = parseTopLevelStmt(p)
-          if n.kind == nkEmpty or n.kind notin imperativeCode:
-            rest = n
-            break
-          sl.add n
-        #echo "-----\n", sl
-        if not processTopLevelStmt(graph, sl, a): break
-        if rest != nil:
-          #echo "-----\n", rest
-          if not processTopLevelStmt(graph, rest, a): break
-      else:
-        #echo "----- single\n", n
-        if not processTopLevelStmt(graph, n, a): break
+      if sfReorder in module.flags or codeReordering in graph.config.features:
+        sl = reorder(graph, sl, module)
+      discard processTopLevelStmt(graph, sl, a)
+
     closeParser(p)
     if s.kind != llsStdIn: break
   closePasses(graph, a)
@@ -199,3 +157,99 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
     # They are responsible for closing the rod files. See `cbackend.nim`.
     closeRodFile(graph, module)
   result = true
+
+proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym =
+  var flags = flags
+  if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
+  result = graph.getModule(fileIdx)
+
+  template processModuleAux(moduleStatus) =
+    onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule)
+    var s: PLLStream
+    if sfMainModule in flags:
+      if graph.config.projectIsStdin: s = stdin.llStreamOpen
+      elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput)
+    discard processModule(graph, result, idGeneratorFromModule(result), s)
+  if result == nil:
+    var cachedModules: seq[FileIndex]
+    result = moduleFromRodFile(graph, fileIdx, cachedModules)
+    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+    if result == nil:
+      result = newModule(graph, fileIdx)
+      result.flags.incl flags
+      registerModule(graph, result)
+      processModuleAux("import")
+    else:
+      if sfSystemModule in flags:
+        graph.systemModule = result
+      partialInitModule(result, graph, fileIdx, filename)
+    for m in cachedModules:
+      registerModuleById(graph, m)
+      replayStateChanges(graph.packed[m.int].module, graph)
+      replayGenericCacheInformation(graph, m.int)
+  elif graph.isDirty(result):
+    result.flags.excl sfDirty
+    # reset module fields:
+    initStrTables(graph, result)
+    result.ast = nil
+    processModuleAux("import(dirty)")
+    graph.markClientsDirty(fileIdx)
+
+proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
+  # this is called by the semantic checking phase
+  assert graph.config != nil
+  result = compileModule(graph, fileIdx, {}, s)
+  graph.addDep(s, fileIdx)
+  # keep track of import relationships
+  if graph.config.hcrOn:
+    graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx)
+  #if sfSystemModule in result.flags:
+  #  localError(result.info, errAttemptToRedefine, result.name.s)
+  # restore the notes for outer module:
+  graph.config.notes =
+    if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
+    else: graph.config.foreignPackageNotes
+
+proc connectCallbacks*(graph: ModuleGraph) =
+  graph.includeFileCallback = modules.includeModule
+  graph.importModuleCallback = importModule
+
+proc compileSystemModule*(graph: ModuleGraph) =
+  if graph.systemModule == nil:
+    connectCallbacks(graph)
+    graph.config.m.systemFileIdx = fileInfoIdx(graph.config,
+        graph.config.libpath / RelativeFile"system.nim")
+    discard graph.compileModule(graph.config.m.systemFileIdx, {sfSystemModule})
+
+proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
+  connectCallbacks(graph)
+  let conf = graph.config
+  wantMainModule(conf)
+  configComplete(graph)
+
+  let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
+  let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
+  conf.projectMainIdx2 = projectFile
+
+  let packSym = getPackage(graph, projectFile)
+  graph.config.mainPackageId = packSym.getPackageId
+  graph.importStack.add projectFile
+
+  if projectFile == systemFileIdx:
+    discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})
+  else:
+    graph.compileSystemModule()
+    discard graph.compileModule(projectFile, {sfMainModule})
+
+proc mySemOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+  result = preparePContext(graph, module, idgen)
+
+proc mySemClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
+  var c = PContext(context)
+  closePContext(graph, c, n)
+
+proc mySemProcess(context: PPassContext, n: PNode): PNode =
+  result = semWithPContext(PContext(context), n)
+
+const semPass* = makePass(mySemOpen, mySemProcess, mySemClose,
+                          isFrontend = true)

compiler/pathutils.nim

@@ -10,7 +10,7 @@
 ## Path handling utilities for Nim. Strictly typed code in order
 ## to avoid the never ending time sink in getting path handling right.
 
-import os, pathnorm
+import os, pathnorm, strutils
 
 when defined(nimPreviewSlimSystem):
   import std/[syncio, assertions]
@@ -102,3 +102,52 @@ when true:
   proc addFileExt*(x: RelativeFile; ext: string): RelativeFile {.borrow.}
 
   proc writeFile*(x: AbsoluteFile; content: string) {.borrow.}
+
+proc skipHomeDir(x: string): int =
+  when defined(windows):
+    if x.continuesWith("Users/", len("C:/")):
+      result = 3
+    else:
+      result = 0
+  else:
+    if x.startsWith("/home/") or x.startsWith("/Users/"):
+      result = 3
+    elif x.startsWith("/mnt/") and x.continuesWith("/Users/", len("/mnt/c")):
+      result = 5
+    else:
+      result = 0
+
+proc relevantPart(s: string; afterSlashX: int): string =
+  result = newStringOfCap(s.len - 8)
+  var slashes = afterSlashX
+  for i in 0..<s.len:
+    if slashes == 0:
+      result.add s[i]
+    elif s[i] == '/':
+      dec slashes
+
+template canonSlashes(x: string): string =
+  when defined(windows):
+    x.replace('\\', '/')
+  else:
+    x
+
+proc customPathImpl(x: string): string =
+  # Idea: Encode a "protocol" via "//protocol/path" which is not ambiguous
+  # as path canonicalization would have removed the double slashes.
+  # /mnt/X/Users/Y
+  # X:\\Users\Y
+  # /home/Y
+  # -->
+  # //user/
+  if not isAbsolute(x):
+    result = customPathImpl(canonSlashes(getCurrentDir() / x))
+  else:
+    let slashes = skipHomeDir(x)
+    if slashes > 0:
+      result = "//user/" & relevantPart(x, slashes)
+    else:
+      result = x
+
+proc customPath*(x: string): string =
+  customPathImpl canonSlashes x

compiler/pipelines.nim

@@ -0,0 +1,294 @@
+import sem, cgen, modulegraphs, ast, llstream, parser, msgs,
+       lineinfos, reorder, options, semdata, cgendata, modules, pathutils,
+       packages, syntaxes, depends, vm, pragmas, idents, lookups, wordrecg,
+       liftdestructors
+
+import pipelineutils
+
+when not defined(leanCompiler):
+  import jsgen, docgen2
+
+import std/[syncio, objectdollar, assertions, tables, strutils]
+import renderer
+import ic/replayer
+
+
+proc setPipeLinePass*(graph: ModuleGraph; pass: PipelinePass) =
+  graph.pipelinePass = pass
+
+proc processPipeline(graph: ModuleGraph; semNode: PNode; bModule: PPassContext): PNode =
+  case graph.pipelinePass
+  of CgenPass:
+    result = semNode
+    if bModule != nil:
+      genTopLevelStmt(BModule(bModule), result)
+  of JSgenPass:
+    when not defined(leanCompiler):
+      result = processJSCodeGen(bModule, semNode)
+  of GenDependPass:
+    result = addDotDependency(bModule, semNode)
+  of SemPass:
+    result = graph.emptyNode
+  of Docgen2Pass, Docgen2TexPass:
+    when not defined(leanCompiler):
+      result = processNode(bModule, semNode)
+  of Docgen2JsonPass:
+    when not defined(leanCompiler):
+      result = processNodeJson(bModule, semNode)
+  of EvalPass, InterpreterPass:
+    result = interpreterCode(bModule, semNode)
+  of NonePass:
+    doAssert false, "use setPipeLinePass to set a proper PipelinePass"
+
+proc processImplicitImports(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind,
+                      m: PSym, ctx: PContext, bModule: PPassContext, idgen: IdGenerator,
+                      ) =
+  # XXX fixme this should actually be relative to the config file!
+  let relativeTo = toFullPath(graph.config, m.info)
+  for module in items(implicits):
+    # implicit imports should not lead to a module importing itself
+    if m.position != resolveMod(graph.config, module, relativeTo).int32:
+      var importStmt = newNodeI(nodeKind, m.info)
+      var str = newStrNode(nkStrLit, module)
+      str.info = m.info
+      importStmt.add str
+      message(graph.config, importStmt.info, hintProcessingStmt, $idgen[])
+      let semNode = semWithPContext(ctx, importStmt)
+      if semNode == nil or processPipeline(graph, semNode, bModule) == nil:
+        break
+
+proc prePass(c: PContext; n: PNode) =
+  for son in n:
+    if son.kind == nkPragma:
+      for s in son:
+        var key = if s.kind in nkPragmaCallKinds and s.len > 1: s[0] else: s
+        if key.kind in {nkBracketExpr, nkCast} or key.kind notin nkIdentKinds:
+          continue
+        let ident = whichKeyword(considerQuotedIdent(c, key))
+        case ident
+        of wReorder:
+          pragmaNoForward(c, s, flag = sfReorder)
+        of wExperimental:
+          if isTopLevel(c) and s.kind in nkPragmaCallKinds and s.len == 2:
+            let name = c.semConstExpr(c, s[1])
+            case name.kind
+            of nkStrLit, nkRStrLit, nkTripleStrLit:
+              try:
+                let feature = parseEnum[Feature](name.strVal)
+                if feature == codeReordering:
+                  c.features.incl feature
+                  c.module.flags.incl sfReorder
+              except ValueError:
+                discard
+            else:
+              discard
+        else:
+          discard
+
+proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
+                    stream: PLLStream): bool =
+  if graph.stopCompile(): return true
+  var
+    p: Parser
+    s: PLLStream
+    fileIdx = module.fileIdx
+
+  prepareConfigNotes(graph, module)
+  let ctx = preparePContext(graph, module, idgen)
+  let bModule: PPassContext =
+    case graph.pipelinePass
+    of CgenPass:
+      setupCgen(graph, module, idgen)
+    of JSgenPass:
+      when not defined(leanCompiler):
+        setupJSgen(graph, module, idgen)
+      else:
+        nil
+    of EvalPass, InterpreterPass:
+      setupEvalGen(graph, module, idgen)
+    of GenDependPass:
+      setupDependPass(graph, module, idgen)
+    of Docgen2Pass:
+      when not defined(leanCompiler):
+        openHtml(graph, module, idgen)
+      else:
+        nil
+    of Docgen2TexPass:
+      when not defined(leanCompiler):
+        openTex(graph, module, idgen)
+      else:
+        nil
+    of Docgen2JsonPass:
+      when not defined(leanCompiler):
+        openJson(graph, module, idgen)
+      else:
+        nil
+    of SemPass:
+      nil
+    of NonePass:
+      doAssert false, "use setPipeLinePass to set a proper PipelinePass"
+      nil
+
+  if stream == nil:
+    let filename = toFullPathConsiderDirty(graph.config, fileIdx)
+    s = llStreamOpen(filename, fmRead)
+    if s == nil:
+      rawMessage(graph.config, errCannotOpenFile, filename.string)
+      return false
+  else:
+    s = stream
+
+  while true:
+    syntaxes.openParser(p, fileIdx, s, graph.cache, graph.config)
+
+    if not belongsToStdlib(graph, module) or (belongsToStdlib(graph, module) and module.name.s == "distros"):
+      # XXX what about caching? no processing then? what if I change the
+      # modules to include between compilation runs? we'd need to track that
+      # in ROD files. I think we should enable this feature only
+      # for the interactive mode.
+      if module.name.s != "nimscriptapi":
+        processImplicitImports graph, graph.config.implicitImports, nkImportStmt, module, ctx, bModule, idgen
+        processImplicitImports graph, graph.config.implicitIncludes, nkIncludeStmt, module, ctx, bModule, idgen
+
+    checkFirstLineIndentation(p)
+    block processCode:
+      if graph.stopCompile(): break processCode
+      var n = parseTopLevelStmt(p)
+      if n.kind == nkEmpty: break processCode
+      # read everything, no streaming possible
+      var sl = newNodeI(nkStmtList, n.info)
+      sl.add n
+      while true:
+        var n = parseTopLevelStmt(p)
+        if n.kind == nkEmpty: break
+        sl.add n
+
+      prePass(ctx, sl)
+      if sfReorder in module.flags or codeReordering in graph.config.features:
+        sl = reorder(graph, sl, module)
+      if graph.pipelinePass != EvalPass:
+        message(graph.config, sl.info, hintProcessingStmt, $idgen[])
+      var semNode = semWithPContext(ctx, sl)
+      discard processPipeline(graph, semNode, bModule)
+
+    closeParser(p)
+    if s.kind != llsStdIn: break
+  let finalNode = closePContext(graph, ctx, nil)
+  case graph.pipelinePass
+  of CgenPass:
+    if bModule != nil:
+      let disps = finalCodegenActions(graph, BModule(bModule), finalNode)
+      if disps != nil:
+        let ctx = preparePContext(graph, module, idgen)
+        for disp in disps:
+          let retTyp = disp.sym.typ[0]
+          if retTyp != nil:
+            # todo properly semcheck the code of dispatcher?
+            createTypeBoundOps(graph, ctx, retTyp, disp.info, idgen)
+          genProcAux(BModule(bModule), disp.sym)
+        discard closePContext(graph, ctx, nil)
+  of JSgenPass:
+    when not defined(leanCompiler):
+      discard finalJSCodeGen(graph, bModule, finalNode)
+  of EvalPass, InterpreterPass:
+    discard interpreterCode(bModule, finalNode)
+  of SemPass, GenDependPass:
+    discard
+  of Docgen2Pass, Docgen2TexPass:
+    when not defined(leanCompiler):
+      discard closeDoc(graph, bModule, finalNode)
+  of Docgen2JsonPass:
+    when not defined(leanCompiler):
+      discard closeJson(graph, bModule, finalNode)
+  of NonePass:
+    doAssert false, "use setPipeLinePass to set a proper PipelinePass"
+
+  if graph.config.backend notin {backendC, backendCpp, backendObjc}:
+    # We only write rod files here if no C-like backend is active.
+    # The C-like backends have been patched to support the IC mechanism.
+    # They are responsible for closing the rod files. See `cbackend.nim`.
+    closeRodFile(graph, module)
+  result = true
+
+proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym =
+  var flags = flags
+  if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
+  result = graph.getModule(fileIdx)
+
+  template processModuleAux(moduleStatus) =
+    onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule)
+    var s: PLLStream
+    if sfMainModule in flags:
+      if graph.config.projectIsStdin: s = stdin.llStreamOpen
+      elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput)
+    discard processPipelineModule(graph, result, idGeneratorFromModule(result), s)
+  if result == nil:
+    var cachedModules: seq[FileIndex]
+    result = moduleFromRodFile(graph, fileIdx, cachedModules)
+    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+    if result == nil:
+      result = newModule(graph, fileIdx)
+      result.flags.incl flags
+      registerModule(graph, result)
+      processModuleAux("import")
+    else:
+      if sfSystemModule in flags:
+        graph.systemModule = result
+      partialInitModule(result, graph, fileIdx, filename)
+    for m in cachedModules:
+      registerModuleById(graph, m)
+      replayStateChanges(graph.packed[m.int].module, graph)
+      replayGenericCacheInformation(graph, m.int)
+  elif graph.isDirty(result):
+    result.flags.excl sfDirty
+    # reset module fields:
+    initStrTables(graph, result)
+    result.ast = nil
+    processModuleAux("import(dirty)")
+    graph.markClientsDirty(fileIdx)
+
+proc importPipelineModule(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
+  # this is called by the semantic checking phase
+  assert graph.config != nil
+  result = compilePipelineModule(graph, fileIdx, {}, s)
+  graph.addDep(s, fileIdx)
+  # keep track of import relationships
+  if graph.config.hcrOn:
+    graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx)
+  #if sfSystemModule in result.flags:
+  #  localError(result.info, errAttemptToRedefine, result.name.s)
+  # restore the notes for outer module:
+  graph.config.notes =
+    if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
+    else: graph.config.foreignPackageNotes
+
+proc connectPipelineCallbacks*(graph: ModuleGraph) =
+  graph.includeFileCallback = modules.includeModule
+  graph.importModuleCallback = importPipelineModule
+
+proc compilePipelineSystemModule*(graph: ModuleGraph) =
+  if graph.systemModule == nil:
+    connectPipelineCallbacks(graph)
+    graph.config.m.systemFileIdx = fileInfoIdx(graph.config,
+        graph.config.libpath / RelativeFile"system.nim")
+    discard graph.compilePipelineModule(graph.config.m.systemFileIdx, {sfSystemModule})
+
+proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
+  connectPipelineCallbacks(graph)
+  let conf = graph.config
+  wantMainModule(conf)
+  configComplete(graph)
+
+  let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
+  let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
+  conf.projectMainIdx2 = projectFile
+
+  let packSym = getPackage(graph, projectFile)
+  graph.config.mainPackageId = packSym.getPackageId
+  graph.importStack.add projectFile
+
+  if projectFile == systemFileIdx:
+    discard graph.compilePipelineModule(projectFile, {sfMainModule, sfSystemModule})
+  else:
+    graph.compilePipelineSystemModule()
+    discard graph.compilePipelineModule(projectFile, {sfMainModule})

compiler/pipelineutils.nim

@@ -0,0 +1,26 @@
+import ast, options, lineinfos, pathutils, msgs, modulegraphs, packages
+
+proc skipCodegen*(config: ConfigRef; n: PNode): bool {.inline.} =
+  # can be used by codegen passes to determine whether they should do
+  # something with `n`. Currently, this ignores `n` and uses the global
+  # error count instead.
+  result = config.errorCounter > 0
+
+proc resolveMod*(conf: ConfigRef; module, relativeTo: string): FileIndex =
+  let fullPath = findModule(conf, module, relativeTo)
+  if fullPath.isEmpty:
+    result = InvalidFileIdx
+  else:
+    result = fileInfoIdx(conf, fullPath)
+
+proc prepareConfigNotes*(graph: ModuleGraph; module: PSym) =
+  # don't be verbose unless the module belongs to the main package:
+  if graph.config.belongsToProjectPackage(module):
+    graph.config.notes = graph.config.mainPackageNotes
+  else:
+    if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes
+    graph.config.notes = graph.config.foreignPackageNotes
+
+proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
+  result = true
+  #module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")

compiler/platform.nim

@@ -26,7 +26,8 @@ type
     osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris,
     osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osCrossos, osAix, osPalmos, osQnx,
     osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks
-    osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osZephyr, osAny
+    osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osZephyr,
+    osNuttX, osAny
 
 type
   TInfoOSProp* = enum
@@ -193,6 +194,10 @@ const
       objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
       scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
       props: {ospPosix}),
+     (name: "NuttX", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
+      objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
+      scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
+      props: {ospPosix}),
      (name: "Any", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
       objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
       scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",

compiler/plugins/itersgen.nim

@@ -31,7 +31,7 @@ proc iterToProcImpl*(c: PContext, n: PNode): PNode =
     return
   let body = liftIterToProc(c.graph, iter.sym, getBody(c.graph, iter.sym), t, c.idgen)
 
-  let prc = newSym(skProc, n[3].ident, nextSymId c.idgen, iter.sym.owner, iter.sym.info)
+  let prc = newSym(skProc, n[3].ident, c.idgen, iter.sym.owner, iter.sym.info)
   prc.typ = copyType(iter.sym.typ, nextTypeId c.idgen, prc)
   excl prc.typ.flags, tfCapturesEnv
   prc.typ.n.add newSymNode(getEnvParam(iter.sym))

compiler/plugins/locals.nim

@@ -26,7 +26,7 @@ proc semLocals*(c: PContext, n: PNode): PNode =
             {tyVarargs, tyOpenArray, tyTypeDesc, tyStatic, tyUntyped, tyTyped, tyEmpty}:
 
         if it.owner == owner:
-          var field = newSym(skField, it.name, nextSymId c.idgen, owner, n.info)
+          var field = newSym(skField, it.name, c.idgen, owner, n.info)
           field.typ = it.typ.skipTypes({tyVar})
           field.position = counter
           inc(counter)

compiler/pragmas.nim

@@ -12,7 +12,7 @@
 import
   os, condsyms, ast, astalgo, idents, semdata, msgs, renderer,
   wordrecg, ropes, options, strutils, extccomp, math, magicsys, trees,
-  types, lookups, lineinfos, pathutils, linter
+  types, lookups, lineinfos, pathutils, linter, modulepaths
 
 when defined(nimPreviewSlimSystem):
   import std/assertions
@@ -34,7 +34,7 @@ const
     wAsmNoStackFrame, wDiscardable, wNoInit, wCodegenDecl,
     wGensym, wInject, wRaises, wEffectsOf, wTags, wForbids, wLocks, wDelegator, wGcSafe,
     wConstructor, wLiftLocals, wStackTrace, wLineTrace, wNoDestroy,
-    wRequires, wEnsures, wEnforceNoRaises, wSystemRaisesDefect}
+    wRequires, wEnsures, wEnforceNoRaises, wSystemRaisesDefect, wVirtual, wQuirky}
   converterPragmas* = procPragmas
   methodPragmas* = procPragmas+{wBase}-{wImportCpp}
   templatePragmas* = {wDeprecated, wError, wGensym, wInject, wDirty,
@@ -71,7 +71,8 @@ const
     wPure, wHeader, wCompilerProc, wCore, wFinal, wSize, wShallow,
     wIncompleteStruct, wCompleteStruct, wByCopy, wByRef,
     wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked,
-    wCppNonPod, wBorrow, wGcSafe, wPartial, wExplain, wPackage}
+    wCppNonPod, wBorrow, wGcSafe, wPartial, wExplain, wPackage, wCodegenDecl,
+    wSendable}
   fieldPragmas* = declPragmas + {wGuard, wBitsize, wCursor,
     wRequiresInit, wNoalias, wAlign} - {wExportNims, wNodecl} # why exclude these?
   varPragmas* = declPragmas + {wVolatile, wRegister, wThreadVar,
@@ -83,7 +84,7 @@ const
     wGensym, wInject,
     wIntDefine, wStrDefine, wBoolDefine, wDefine,
     wCompilerProc, wCore}
-  paramPragmas* = {wNoalias, wInject, wGensym}
+  paramPragmas* = {wNoalias, wInject, wGensym, wByRef, wByCopy, wCodegenDecl}
   letPragmas* = varPragmas
   procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNoSideEffect,
                       wThread, wRaises, wEffectsOf, wLocks, wTags, wForbids, wGcSafe,
@@ -116,7 +117,15 @@ const
 proc invalidPragma*(c: PContext; n: PNode) =
   localError(c.config, n.info, "invalid pragma: " & renderTree(n, {renderNoComments}))
 proc illegalCustomPragma*(c: PContext, n: PNode, s: PSym) =
-  localError(c.config, n.info, "cannot attach a custom pragma to '" & s.name.s & "'")
+  var msg = "cannot attach a custom pragma to '" & s.name.s & "'"
+  if s != nil:
+    msg.add("; custom pragmas are not supported for ")
+    case s.kind
+    of skForVar: msg.add("`for` loop variables")
+    of skEnumField: msg.add("enum fields")
+    of skModule: msg.add("modules")
+    else: msg.add("symbol kind " & $s.kind)
+  localError(c.config, n.info, msg)
 
 proc pragmaProposition(c: PContext, n: PNode) =
   if n.kind notin nkPragmaCallKinds or n.len != 2:
@@ -131,7 +140,7 @@ proc pragmaEnsures(c: PContext, n: PNode) =
     openScope(c)
     let o = getCurrOwner(c)
     if o.kind in routineKinds and o.typ != nil and o.typ.sons[0] != nil:
-      var s = newSym(skResult, getIdent(c.cache, "result"), nextSymId(c.idgen), o, n.info)
+      var s = newSym(skResult, getIdent(c.cache, "result"), c.idgen, o, n.info)
       s.typ = o.typ.sons[0]
       incl(s.flags, sfUsed)
       addDecl(c, s)
@@ -165,9 +174,7 @@ proc setExternName(c: PContext; s: PSym, extname: string, info: TLineInfo) =
       localError(c.config, info, "invalid extern name: '" & extname & "'. (Forgot to escape '$'?)")
   when hasFFI:
     s.cname = $s.loc.r
-  if c.config.cmd == cmdNimfix and '$' notin extname:
-    # note that '{.importc.}' is transformed into '{.importc: "$1".}'
-    s.loc.flags.incl(lfFullExternalName)
+
 
 proc makeExternImport(c: PContext; s: PSym, extname: string, info: TLineInfo) =
   setExternName(c, s, extname, info)
@@ -202,9 +209,9 @@ proc processImportObjC(c: PContext; s: PSym, extname: string, info: TLineInfo) =
   let m = s.getModule()
   incl(m.flags, sfCompileToObjc)
 
-proc newEmptyStrNode(c: PContext; n: PNode): PNode {.noinline.} =
+proc newEmptyStrNode(c: PContext; n: PNode, strVal: string = ""): PNode {.noinline.} =
   result = newNodeIT(nkStrLit, n.info, getSysType(c.graph, n.info, tyString))
-  result.strVal = ""
+  result.strVal = strVal
 
 proc getStrLitNode(c: PContext, n: PNode): PNode =
   if n.kind notin nkPragmaCallKinds or n.len != 2:
@@ -236,8 +243,17 @@ proc getOptionalStr(c: PContext, n: PNode, defaultStr: string): string =
   if n.kind in nkPragmaCallKinds: result = expectStrLit(c, n)
   else: result = defaultStr
 
+proc processVirtual(c: PContext, n: PNode, s: PSym) =
+  s.constraint = newEmptyStrNode(c, n, getOptionalStr(c, n, "$1"))
+  s.constraint.strVal = s.constraint.strVal % s.name.s
+  s.flags.incl {sfVirtual, sfInfixCall, sfExportc, sfMangleCpp}
+
+  s.typ.callConv = ccNoConvention
+  incl c.config.globalOptions, optMixedMode
+
 proc processCodegenDecl(c: PContext, n: PNode, sym: PSym) =
   sym.constraint = getStrLitNode(c, n)
+  sym.flags.incl sfCodegenDecl
 
 proc processMagic(c: PContext, n: PNode, s: PSym) =
   #if sfSystemModule notin c.module.flags:
@@ -270,7 +286,7 @@ proc onOff(c: PContext, n: PNode, op: TOptions, resOptions: var TOptions) =
   if isTurnedOn(c, n): resOptions.incl op
   else: resOptions.excl op
 
-proc pragmaNoForward(c: PContext, n: PNode; flag=sfNoForward) =
+proc pragmaNoForward*(c: PContext, n: PNode; flag=sfNoForward) =
   if isTurnedOn(c, n):
     incl(c.module.flags, flag)
     c.features.incl codeReordering
@@ -302,7 +318,7 @@ proc getLib(c: PContext, kind: TLibKind, path: PNode): PLib =
   result.path = path
   c.libs.add result
   if path.kind in {nkStrLit..nkTripleStrLit}:
-    result.isOverriden = options.isDynlibOverride(c.config, path.strVal)
+    result.isOverridden = options.isDynlibOverride(c.config, path.strVal)
 
 proc expectDynlibNode(c: PContext, n: PNode): PNode =
   if n.kind notin nkPragmaCallKinds or n.len != 2:
@@ -322,12 +338,12 @@ proc expectDynlibNode(c: PContext, n: PNode): PNode =
 proc processDynLib(c: PContext, n: PNode, sym: PSym) =
   if (sym == nil) or (sym.kind == skModule):
     let lib = getLib(c, libDynamic, expectDynlibNode(c, n))
-    if not lib.isOverriden:
+    if not lib.isOverridden:
       c.optionStack[^1].dynlib = lib
   else:
     if n.kind in nkPragmaCallKinds:
       var lib = getLib(c, libDynamic, expectDynlibNode(c, n))
-      if not lib.isOverriden:
+      if not lib.isOverridden:
         addToLib(lib, sym)
         incl(sym.loc.flags, lfDynamicLib)
     else:
@@ -364,7 +380,7 @@ proc processNote(c: PContext, n: PNode) =
     else: invalidPragma(c, n)
   else: invalidPragma(c, n)
 
-proc pragmaToOptions(w: TSpecialWord): TOptions {.inline.} =
+proc pragmaToOptions*(w: TSpecialWord): TOptions {.inline.} =
   case w
   of wChecks: ChecksOptions
   of wObjChecks: {optObjCheck}
@@ -390,6 +406,7 @@ proc pragmaToOptions(w: TSpecialWord): TOptions {.inline.} =
   of wImplicitStatic: {optImplicitStatic}
   of wPatterns, wTrMacros: {optTrMacros}
   of wSinkInference: {optSinkInference}
+  of wQuirky: {optQuirky}
   else: {}
 
 proc processExperimental(c: PContext; n: PNode) =
@@ -509,18 +526,25 @@ proc relativeFile(c: PContext; n: PNode; ext=""): AbsoluteFile =
       if result.isEmpty: result = AbsoluteFile s
 
 proc processCompile(c: PContext, n: PNode) =
+  ## This pragma can take two forms. The first is a simple file input:
+  ##     {.compile: "file.c".}
+  ## The second is a tuple where the second arg is the output name strutils formatter:
+  ##     {.compile: ("file.c", "$1.o").}
   proc docompile(c: PContext; it: PNode; src, dest: AbsoluteFile; customArgs: string) =
     var cf = Cfile(nimname: splitFile(src).name,
                    cname: src, obj: dest, flags: {CfileFlag.External},
                    customArgs: customArgs)
-    extccomp.addExternalFileToCompile(c.config, cf)
-    recordPragma(c, it, "compile", src.string, dest.string, customArgs)
+    if not fileExists(src):
+      localError(c.config, n.info, "cannot find: " & src.string)
+    else:
+      extccomp.addExternalFileToCompile(c.config, cf)
+      recordPragma(c, it, "compile", src.string, dest.string, customArgs)
 
   proc getStrLit(c: PContext, n: PNode; i: int): string =
     n[i] = c.semConstExpr(c, n[i])
     case n[i].kind
     of nkStrLit, nkRStrLit, nkTripleStrLit:
-      when defined(gcArc) or defined(gcOrc):
+      when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc):
         result = n[i].strVal
       else:
         shallowCopy(result, n[i].strVal)
@@ -554,7 +578,8 @@ proc processCompile(c: PContext, n: PNode) =
       else:
         found = findFile(c.config, s)
         if found.isEmpty: found = AbsoluteFile s
-    let obj = toObjFile(c.config, completeCfilePath(c.config, found, false))
+    let mangled = completeCfilePath(c.config, mangleModuleName(c.config, found).AbsoluteFile)
+    let obj = toObjFile(c.config, mangled)
     docompile(c, it, found, obj, customArgs)
 
 proc processLink(c: PContext, n: PNode) =
@@ -606,7 +631,7 @@ proc pragmaEmit(c: PContext, n: PNode) =
     if n1.kind == nkBracket:
       var b = newNodeI(nkBracket, n1.info, n1.len)
       for i in 0..<n1.len:
-        b[i] = c.semExpr(c, n1[i])
+        b[i] = c.semExprWithType(c, n1[i], {efTypeAllowed})
       n[1] = b
     else:
       n[1] = c.semConstExpr(c, n1)
@@ -659,7 +684,8 @@ proc processPragma(c: PContext, n: PNode, i: int) =
   elif it.safeLen != 2 or it[0].kind != nkIdent or it[1].kind != nkIdent:
     invalidPragma(c, n)
 
-  var userPragma = newSym(skTemplate, it[1].ident, nextSymId(c.idgen), c.module, it.info, c.config.options)
+  var userPragma = newSym(skTemplate, it[1].ident, c.idgen, c.module, it.info, c.config.options)
+  styleCheckDef(c, userPragma)
   userPragma.ast = newTreeI(nkPragma, n.info, n.sons[i+1..^1])
   strTableAdd(c.userPragmas, userPragma)
 
@@ -719,19 +745,8 @@ proc deprecatedStmt(c: PContext; outerPragma: PNode) =
     return
   if pragma.kind != nkBracket:
     localError(c.config, pragma.info, "list of key:value pairs expected"); return
-  for n in pragma:
-    if n.kind in nkPragmaCallKinds and n.len == 2:
-      let dest = qualifiedLookUp(c, n[1], {checkUndeclared})
-      if dest == nil or dest.kind in routineKinds:
-        localError(c.config, n.info, warnUser, "the .deprecated pragma is unreliable for routines")
-      let src = considerQuotedIdent(c, n[0])
-      let alias = newSym(skAlias, src, nextSymId(c.idgen), dest, n[0].info, c.config.options)
-      incl(alias.flags, sfExported)
-      if sfCompilerProc in dest.flags: markCompilerProc(c, alias)
-      addInterfaceDecl(c, alias)
-      n[1] = newSymNode(dest)
-    else:
-      localError(c.config, n.info, "key:value pair expected")
+  message(c.config, pragma.info, warnDeprecated,
+    "deprecated statement is now a no-op, use regular deprecated pragma")
 
 proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym =
   if it.kind notin nkPragmaCallKinds or it.len != 2:
@@ -747,12 +762,12 @@ proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym =
       # We return a dummy symbol; later passes over the type will repair it.
       # Generic instantiation needs to know about this too. But we're lazy
       # and perform the lookup on demand instead.
-      result = newSym(skUnknown, considerQuotedIdent(c, n), nextSymId(c.idgen), nil, n.info,
+      result = newSym(skUnknown, considerQuotedIdent(c, n), c.idgen, nil, n.info,
         c.config.options)
   else:
     result = qualifiedLookUp(c, n, {checkUndeclared})
 
-proc semCustomPragma(c: PContext, n: PNode): PNode =
+proc semCustomPragma(c: PContext, n: PNode, sym: PSym): PNode =
   var callNode: PNode
 
   if n.kind in {nkIdent, nkSym}:
@@ -772,6 +787,11 @@ proc semCustomPragma(c: PContext, n: PNode): PNode =
     invalidPragma(c, n)
     return n
 
+  # we have a valid custom pragma
+  if sym != nil and sym.kind in {skEnumField, skForVar, skModule}:
+    illegalCustomPragma(c, n, sym)
+    return n
+
   result = r
   # Transform the nkCall node back to its original form if possible
   if n.kind == nkIdent and r.len == 1:
@@ -805,7 +825,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
                   validPragmas: TSpecialWords,
                   comesFromPush, isStatement: bool): bool =
   var it = n[i]
-  var key = if it.kind in nkPragmaCallKinds and it.len > 1: it[0] else: it
+  let keyDeep = it.kind in nkPragmaCallKinds and it.len > 1
+  var key = if keyDeep: it[0] else: it
   if key.kind == nkBracketExpr:
     processNote(c, it)
     return
@@ -819,7 +840,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
     else: discard
     return
   elif key.kind notin nkIdentKinds:
-    n[i] = semCustomPragma(c, it)
+    n[i] = semCustomPragma(c, it, sym)
     return
   let ident = considerQuotedIdent(c, key)
   var userPragma = strTableGet(c.userPragmas, ident)
@@ -831,6 +852,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
     if c.instCounter > 100:
       globalError(c.config, it.info, "recursive dependency: " & userPragma.name.s)
 
+    if keyDeep:
+      localError(c.config, it.info, "user pragma cannot have arguments")
+
     pragma(c, sym, userPragma.ast, validPragmas, isStatement)
     n.sons[i..i] = userPragma.ast.sons # expand user pragma with its content
     i.inc(userPragma.ast.len - 1) # inc by -1 is ok, user pragmas was empty
@@ -838,7 +862,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
   else:
     let k = whichKeyword(ident)
     if k in validPragmas:
-      checkPragmaUse(c.config, key.info, k, ident.s)
+      checkPragmaUse(c, key.info, k, ident.s, (if sym != nil: sym else: c.module))
       case k
       of wExportc, wExportCpp:
         makeExternExport(c, sym, getOptionalStr(c, it, "$1"), it.info)
@@ -926,7 +950,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wThreadVar:
         noVal(c, it)
         incl(sym.flags, {sfThread, sfGlobal})
-      of wDeadCodeElimUnused: discard  # deprecated, dead code elim always on
+      of wDeadCodeElimUnused:
+        warningDeprecated(c.config, n.info, "'{.deadcodeelim: on.}' is deprecated, now a noop")  # deprecated, dead code elim always on
       of wNoForward: pragmaNoForward(c, it)
       of wReorder: pragmaNoForward(c, it, flag = sfReorder)
       of wMagic: processMagic(c, it, sym)
@@ -946,8 +971,12 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         # only supported for backwards compat, doesn't do anything anymore
         noVal(c, it)
       of wConstructor:
-        noVal(c, it)
         incl(sym.flags, sfConstructor)
+        if sfImportc notin sym.flags:
+          sym.constraint = newEmptyStrNode(c, it, getOptionalStr(c, it, ""))
+          sym.constraint.strVal = sym.constraint.strVal
+          sym.flags.incl {sfExportc, sfMangleCpp}
+          sym.typ.callConv = ccNoConvention
       of wHeader:
         var lib = getLib(c, libHeader, getStrLitNode(c, it))
         addToLib(lib, sym)
@@ -1041,6 +1070,12 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         if sym.typ != nil:
           incl(sym.typ.flags, tfThread)
           if sym.typ.callConv == ccClosure: sym.typ.callConv = ccNimCall
+      of wSendable:
+        noVal(c, it)
+        if sym != nil and sym.typ != nil:
+          incl(sym.typ.flags, tfSendable)
+        else:
+          invalidPragma(c, it)
       of wGcSafe:
         noVal(c, it)
         if sym != nil:
@@ -1165,13 +1200,17 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
           invalidPragma(c, it)
       of wByRef:
         noVal(c, it)
-        if sym == nil or sym.typ == nil:
+        if sym != nil and sym.kind == skParam:
+          sym.options.incl optByRef
+        elif sym == nil or sym.typ == nil:
           processOption(c, it, c.config.options)
         else:
           incl(sym.typ.flags, tfByRef)
       of wByCopy:
         noVal(c, it)
-        if sym.kind != skType or sym.typ == nil: invalidPragma(c, it)
+        if sym.kind == skParam:
+          incl(sym.flags, sfByCopy)
+        elif sym.kind != skType or sym.typ == nil: invalidPragma(c, it)
         else: incl(sym.typ.flags, tfByCopy)
       of wPartial:
         noVal(c, it)
@@ -1236,21 +1275,19 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         pragmaProposition(c, it)
       of wEnsures:
         pragmaEnsures(c, it)
-      of wEnforceNoRaises:
+      of wEnforceNoRaises, wQuirky:
         sym.flags.incl sfNeverRaises
       of wSystemRaisesDefect:
         sym.flags.incl sfSystemRaisesDefect
+      of wVirtual:
+        processVirtual(c, it, sym)
+
       else: invalidPragma(c, it)
     elif comesFromPush and whichKeyword(ident) != wInvalid:
       discard "ignore the .push pragma; it doesn't apply"
     else:
-      if sym == nil or (sym.kind in {skVar, skLet, skConst, skParam, skIterator,
-                        skField, skProc, skFunc, skConverter, skMethod, skType}):
-        n[i] = semCustomPragma(c, it)
-      elif sym != nil:
-        illegalCustomPragma(c, it, sym)
-      else:
-        invalidPragma(c, it)
+      # semCustomPragma gives appropriate error for invalid pragmas
+      n[i] = semCustomPragma(c, it, sym)
 
 proc overwriteLineInfo(n: PNode; info: TLineInfo) =
   n.info = info

compiler/readme.md

@@ -4,4 +4,4 @@
 - Note that this code has been translated from a bootstrapping version written in Pascal.
 - So the code is **not** a poster child of good Nim code.
 
-See [Internals of the Nim Compiler](https://nim-lang.org/docs/intern.html) for more information.
+See [Internals of the Nim Compiler](https://nim-lang.github.io/Nim/intern.html) for more information.

compiler/renderer.nim

@@ -9,10 +9,9 @@
 
 # This module implements the renderer of the standard Nim representation.
 
-when defined(nimHasUsed):
-  # 'import renderer' is so useful for debugging
-  # that Nim shouldn't produce a warning for that:
-  {.used.}
+# 'import renderer' is so useful for debugging
+# that Nim shouldn't produce a warning for that:
+{.used.}
 
 import
   lexer, options, idents, strutils, ast, msgs, lineinfos
@@ -24,13 +23,18 @@ type
   TRenderFlag* = enum
     renderNone, renderNoBody, renderNoComments, renderDocComments,
     renderNoPragmas, renderIds, renderNoProcDefs, renderSyms, renderRunnableExamples,
-    renderIr, renderExpandUsing
+    renderIr, renderNonExportedFields, renderExpandUsing
+
   TRenderFlags* = set[TRenderFlag]
   TRenderTok* = object
     kind*: TokType
     length*: int16
     sym*: PSym
 
+  Section = enum
+    GenericParams
+    ObjectDef
+
   TRenderTokSeq* = seq[TRenderTok]
   TSrcGen* = object
     indent*: int
@@ -45,7 +49,7 @@ type
     pendingWhitespace: int
     comStack*: seq[PNode]  # comment stack
     flags*: TRenderFlags
-    inGenericParams: bool
+    inside: set[Section] # Keeps track of contexts we are in
     checkAnon: bool        # we're in a context that can contain sfAnon
     inPragma: int
     when defined(nimpretty):
@@ -73,6 +77,16 @@ proc isKeyword*(i: PIdent): bool =
       (i.id <= ord(tokKeywordHigh) - ord(tkSymbol)):
     result = true
 
+proc isExported(n: PNode): bool =
+  ## Checks if an ident is exported.
+  ## This is meant to be used with idents in nkIdentDefs.
+  case n.kind
+  of nkPostfix:
+    n[0].ident.s == "*" and n[1].kind == nkIdent
+  of nkPragmaExpr:
+    n[0].isExported()
+  else: false
+
 proc renderDefinitionName*(s: PSym, noQuotes = false): string =
   ## Returns the definition name of the symbol.
   ##
@@ -85,6 +99,25 @@ proc renderDefinitionName*(s: PSym, noQuotes = false): string =
   else:
     result = '`' & x & '`'
 
+template inside(g: var TSrcGen, section: Section, body: untyped) =
+  ## Runs `body` with `section` included in `g.inside`.
+  ## Removes it at the end of the body if `g` wasn't inside it
+  ## before the template.
+  let wasntInSection = section notin g.inside
+  g.inside.incl section
+  body
+  if wasntInSection:
+    g.inside.excl section
+
+template outside(g: var TSrcGen, section: Section, body: untyped) =
+  ## Temporarily removes `section` from `g.inside`. Adds it back
+  ## at the end of the body if `g` was inside it before the template
+  let wasInSection = section in g.inside
+  g.inside.excl section
+  body
+  if wasInSection:
+    g.inside.incl section
+
 const
   IndentWidth = 2
   longIndentWid = IndentWidth * 2
@@ -121,7 +154,7 @@ proc initSrcGen(g: var TSrcGen, renderFlags: TRenderFlags; config: ConfigRef) =
   g.flags = renderFlags
   g.pendingNL = -1
   g.pendingWhitespace = -1
-  g.inGenericParams = false
+  g.inside = {}
   g.config = config
 
 proc addTok(g: var TSrcGen, kind: TokType, s: string; sym: PSym = nil) =
@@ -379,6 +412,7 @@ proc atom(g: TSrcGen; n: PNode): string =
   of nkEmpty: result = ""
   of nkIdent: result = n.ident.s
   of nkSym: result = n.sym.name.s
+  of nkClosedSymChoice, nkOpenSymChoice: result = n[0].sym.name.s
   of nkStrLit: result = ""; result.addQuoted(n.strVal)
   of nkRStrLit: result = "r\"" & replace(n.strVal, "\"", "\"\"") & '\"'
   of nkTripleStrLit: result = "\"\"\"" & n.strVal & "\"\"\""
@@ -619,7 +653,9 @@ proc isHideable(config: ConfigRef, n: PNode): bool =
   # xxx compare `ident` directly with `getIdent(cache, wRaises)`, but
   # this requires a `cache`.
   case n.kind
-  of nkExprColonExpr: result = n[0].kind == nkIdent and n[0].ident.s in ["raises", "tags", "extern", "deprecated", "forbids"]
+  of nkExprColonExpr:
+    result = n[0].kind == nkIdent and
+             n[0].ident.s.nimIdentNormalize in ["raises", "tags", "extern", "deprecated", "forbids", "stacktrace"]
   of nkIdent: result = n.ident.s in ["gcsafe", "deprecated"]
   else: result = false
 
@@ -651,7 +687,7 @@ proc gcommaAux(g: var TSrcGen, n: PNode, ind: int, start: int = 0,
     inHideable = false
 
 proc gcomma(g: var TSrcGen, n: PNode, c: TContext, start: int = 0,
-            theEnd: int = - 1) =
+            theEnd: int = -1) =
   var ind: int
   if rfInConstExpr in c.flags:
     ind = g.indent + IndentWidth
@@ -828,14 +864,12 @@ proc gproc(g: var TSrcGen, n: PNode) =
 
   if n[patternPos].kind != nkEmpty:
     gpattern(g, n[patternPos])
-  let oldInGenericParams = g.inGenericParams
-  g.inGenericParams = true
-  if renderNoBody in g.flags and n[miscPos].kind != nkEmpty and
-      n[miscPos][1].kind != nkEmpty:
-    gsub(g, n[miscPos][1])
-  else:
-    gsub(g, n[genericParamsPos])
-  g.inGenericParams = oldInGenericParams
+  g.inside(GenericParams):
+    if renderNoBody in g.flags and n[miscPos].kind != nkEmpty and
+        n[miscPos][1].kind != nkEmpty:
+      gsub(g, n[miscPos][1])
+    else:
+      gsub(g, n[genericParamsPos])
   gsub(g, n[paramsPos])
   if renderNoPragmas notin g.flags:
     gsub(g, n[pragmasPos])
@@ -913,7 +947,7 @@ proc gasm(g: var TSrcGen, n: PNode) =
     gsub(g, n[1])
 
 proc gident(g: var TSrcGen, n: PNode) =
-  if g.inGenericParams and n.kind == nkSym:
+  if GenericParams in g.inside and n.kind == nkSym:
     if sfAnon in n.sym.flags or
       (n.typ != nil and tfImplicitTypeParam in n.typ.flags): return
 
@@ -1004,7 +1038,7 @@ proc accentedName(g: var TSrcGen, n: PNode) =
     gsub(g, n)
 
 proc infixArgument(g: var TSrcGen, n: PNode, i: int) =
-  if i < 1 and i > 2: return
+  if i < 1 or i > 2: return
   var needsParenthesis = false
   let nNext = n[i].skipHiddenNodes
   if nNext.kind == nkInfix:
@@ -1301,14 +1335,31 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) =
     gsub(g, n, pragmasPos)
     put(g, tkColon, ":")
     gsub(g, n, bodyPos)
-  of nkConstDef, nkIdentDefs:
+  of nkIdentDefs:
+    # Skip if this is a property in a type and its not exported
+    # (While also not allowing rendering of non exported fields)
+    if ObjectDef in g.inside and (not n[0].isExported() and renderNonExportedFields notin g.flags):
+      return
+    # We render the identDef without being inside the section incase we render something like
+    # y: proc (x: string) # (We wouldn't want to check if x is exported)
+    g.outside(ObjectDef):
+      gcomma(g, n, 0, -3)
+      if n.len >= 2 and n[^2].kind != nkEmpty:
+        putWithSpace(g, tkColon, ":")
+        gsub(g, n[^2], c)
+      elif n.referencesUsing and renderExpandUsing in g.flags:
+        putWithSpace(g, tkColon, ":")
+        gsub(g, newSymNode(n.origUsingType), c)
+
+      if n.len >= 1 and n[^1].kind != nkEmpty:
+        put(g, tkSpaces, Space)
+        putWithSpace(g, tkEquals, "=")
+        gsub(g, n[^1], c)
+  of nkConstDef:
     gcomma(g, n, 0, -3)
     if n.len >= 2 and n[^2].kind != nkEmpty:
       putWithSpace(g, tkColon, ":")
       gsub(g, n[^2], c)
-    elif n.referencesUsing and renderExpandUsing in g.flags:
-      putWithSpace(g, tkColon, ":")
-      gsub(g, newSymNode(n.origUsingType), c)
 
     if n.len >= 1 and n[^1].kind != nkEmpty:
       put(g, tkSpaces, Space)
@@ -1331,6 +1382,10 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) =
     putWithSpace(g, tkColon, ":")
     gsub(g, n, 1)
   of nkInfix:
+    if n.len < 3:
+      var i = 0
+      put(g, tkOpr, "Too few children for nkInfix")
+      return
     let oldLineLen = g.lineLen # we cache this because lineLen gets updated below
     infixArgument(g, n, 1)
     put(g, tkSpaces, Space)
@@ -1378,15 +1433,9 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) =
   of nkAccQuoted:
     put(g, tkAccent, "`")
     for i in 0..<n.len:
-      proc getStrVal(n: PNode): string =
-        # pending https://github.com/nim-lang/Nim/pull/17540, use `getStrVal`
-        case n.kind
-        of nkIdent: n.ident.s
-        of nkSym: n.sym.name.s
-        else: ""
       proc isAlpha(n: PNode): bool =
         if n.kind in {nkIdent, nkSym}:
-          let tmp = n.getStrVal
+          let tmp = n.getPIdent.s
           result = tmp.len > 0 and tmp[0] in {'a'..'z', 'A'..'Z'}
       var useSpace = false
       if i == 1 and n[0].kind == nkIdent and n[0].ident.s in ["=", "'"]:
@@ -1471,10 +1520,11 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) =
   of nkObjectTy:
     if n.len > 0:
       putWithSpace(g, tkObject, "object")
-      gsub(g, n[0])
-      gsub(g, n[1])
-      gcoms(g)
-      gsub(g, n[2])
+      g.inside(ObjectDef):
+        gsub(g, n[0])
+        gsub(g, n[1])
+        gcoms(g)
+        gsub(g, n[2])
     else:
       put(g, tkObject, "object")
   of nkRecList:

compiler/reorder.nim

@@ -106,6 +106,7 @@ proc computeDeps(cache: IdentCache; n: PNode, declares, uses: var IntSet; topLev
     if a.kind == nkExprColonExpr and a[0].kind == nkIdent and a[0].ident.s == "pragma":
       # user defined pragma
       decl(a[1])
+      for i in 1..<n.safeLen: deps(n[i])
     else:
       for i in 0..<n.safeLen: deps(n[i])
   of nkMixinStmt, nkBindStmt: discard

compiler/ropes.nim

@@ -21,8 +21,8 @@ type
                        # though it is not necessary)
   Rope* = string
 
-proc newRopeAppender*(): string {.inline.} =
-  result = newString(0)
+proc newRopeAppender*(cap = 80): string {.inline.} =
+  result = newStringOfCap(cap)
 
 proc freeze*(r: Rope) {.inline.} = discard
 
@@ -102,12 +102,9 @@ proc runtimeFormat*(frmt: FormatStr, args: openArray[Rope]): Rope =
         inc(i)
       else:
         doAssert false, "invalid format string: " & frmt
-    var start = i
-    while i < frmt.len:
-      if frmt[i] != '$': inc(i)
-      else: break
-    if i - 1 >= start:
-      result.add(substr(frmt, start, i - 1))
+    else:
+      result.add(frmt[i])
+      inc(i)
 
 proc `%`*(frmt: static[FormatStr], args: openArray[Rope]): Rope =
   runtimeFormat(frmt, args)

compiler/scriptconfig.nim

@@ -11,13 +11,13 @@
 ## language.
 
 import
-  ast, modules, idents, passes, condsyms,
-  options, sem, llstream, vm, vmdef, commands,
+  ast, modules, idents, condsyms,
+  options, llstream, vm, vmdef, commands,
   os, times, osproc, wordrecg, strtabs, modulegraphs,
-  pathutils
+  pathutils, pipelines
 
 when defined(nimPreviewSlimSystem):
-  import std/syncio
+  import std/[syncio, assertions]
 
 # we support 'cmpIgnoreStyle' natively for efficiency:
 from strutils import cmpIgnoreStyle, contains
@@ -197,20 +197,18 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile;
   conf.symbolFiles = disabledSf
 
   let graph = newModuleGraph(cache, conf)
-  connectCallbacks(graph)
+  connectPipelineCallbacks(graph)
   if freshDefines: initDefines(conf.symbols)
 
   defineSymbol(conf.symbols, "nimscript")
   defineSymbol(conf.symbols, "nimconfig")
-  registerPass(graph, semPass)
-  registerPass(graph, evalPass)
 
   conf.searchPaths.add(conf.libpath)
 
   let oldGlobalOptions = conf.globalOptions
   let oldSelectedGC = conf.selectedGC
-  undefSymbol(conf.symbols, "nimv2")
-  conf.globalOptions.excl {optTinyRtti, optOwnedRefs, optSeqDestructors}
+  unregisterArcOrc(conf)
+  conf.globalOptions.excl optOwnedRefs
   conf.selectedGC = gcUnselected
 
   var m = graph.makeModule(scriptName)
@@ -218,8 +216,9 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile;
   var vm = setupVM(m, cache, scriptName.string, graph, idgen)
   graph.vm = vm
 
-  graph.compileSystemModule()
-  discard graph.processModule(m, vm.idgen, stream)
+  graph.setPipeLinePass(EvalPass)
+  graph.compilePipelineSystemModule()
+  discard graph.processPipelineModule(m, vm.idgen, stream)
 
   # watch out, "newruntime" can be set within NimScript itself and then we need
   # to remember this:
@@ -228,9 +227,20 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile;
   if optOwnedRefs in oldGlobalOptions:
     conf.globalOptions.incl {optTinyRtti, optOwnedRefs, optSeqDestructors}
     defineSymbol(conf.symbols, "nimv2")
-  if conf.selectedGC in {gcArc, gcOrc}:
+  if conf.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
     conf.globalOptions.incl {optTinyRtti, optSeqDestructors}
     defineSymbol(conf.symbols, "nimv2")
+    defineSymbol(conf.symbols, "gcdestructors")
+    defineSymbol(conf.symbols, "nimSeqsV2")
+    case conf.selectedGC
+    of gcArc:
+      defineSymbol(conf.symbols, "gcarc")
+    of gcOrc:
+      defineSymbol(conf.symbols, "gcorc")
+    of gcAtomicArc:
+      defineSymbol(conf.symbols, "gcatomicarc")
+    else:
+      doAssert false, "unreachable"
 
   # ensure we load 'system.nim' again for the real non-config stuff!
   resetSystemArtifacts(graph)

compiler/sem.nim

@@ -9,18 +9,18 @@
 
 # This module implements the semantic checking pass.
 
+import tables
+
 import
   ast, strutils, options, astalgo, trees,
   wordrecg, ropes, msgs, idents, renderer, types, platform, math,
   magicsys, nversion, nimsets, semfold, modulepaths, importer,
-  procfind, lookups, pragmas, passes, semdata, semtypinst, sigmatch,
+  procfind, lookups, pragmas, semdata, semtypinst, sigmatch,
   intsets, transf, vmdef, vm, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity,
   lowerings, plugins/active, lineinfos, strtabs, int128,
   isolation_check, typeallowed, modulegraphs, enumtostr, concepts, astmsgs
 
-when defined(nimfix):
-  import nimfix/prettybase
 
 when not defined(leanCompiler):
   import spawn
@@ -219,7 +219,7 @@ proc commonType*(c: PContext; x: PType, y: PNode): PType =
   commonType(c, x, y.typ)
 
 proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym =
-  result = newSym(kind, considerQuotedIdent(c, n), nextSymId c.idgen, getCurrOwner(c), n.info)
+  result = newSym(kind, considerQuotedIdent(c, n), c.idgen, getCurrOwner(c), n.info)
   when defined(nimsuggest):
     suggestDecl(c, n, result)
 
@@ -242,7 +242,7 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
     # template; we must fix it here: see #909
     result.owner = getCurrOwner(c)
   else:
-    result = newSym(kind, considerQuotedIdent(c, n), nextSymId c.idgen, getCurrOwner(c), n.info)
+    result = newSym(kind, considerQuotedIdent(c, n), c.idgen, getCurrOwner(c), n.info)
   #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule:
   #  incl(result.flags, sfGlobal)
   when defined(nimsuggest):
@@ -281,7 +281,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
 
 proc symFromType(c: PContext; t: PType, info: TLineInfo): PSym =
   if t.sym != nil: return t.sym
-  result = newSym(skType, getIdent(c.cache, "AnonType"), nextSymId c.idgen, t.owner, info)
+  result = newSym(skType, getIdent(c.cache, "AnonType"), c.idgen, t.owner, info)
   result.flags.incl sfAnon
   result.typ = t
 
@@ -405,8 +405,7 @@ proc semExprFlagDispatched(c: PContext, n: PNode, flags: TExprFlags; expectedTyp
       evaluated = evalAtCompileTime(c, result)
       if evaluated != nil: return evaluated
 
-when not defined(nimHasSinkInference):
-  {.pragma: nosinks.}
+proc semGenericStmt(c: PContext, n: PNode): PNode
 
 include hlo, seminst, semcall
 
@@ -468,8 +467,11 @@ proc semAfterMacroCall(c: PContext, call, macroResult: PNode,
         retType = generateTypeInstance(c, paramTypes,
                                        macroResult.info, retType)
 
-      result = semExpr(c, result, flags, expectedType)
-      result = fitNode(c, retType, result, result.info)
+      if retType.kind == tyVoid:
+        result = semStmt(c, result, flags)
+      else:
+        result = semExpr(c, result, flags, expectedType)
+        result = fitNode(c, retType, result, result.info)
       #globalError(s.info, errInvalidParamKindX, typeToString(s.typ[0]))
   dec(c.config.evalTemplateCounter)
   discard c.friendModules.pop()
@@ -513,8 +515,6 @@ proc semConstBoolExpr(c: PContext, n: PNode): PNode =
   result = forceBool(c, semConstExpr(c, n, getSysType(c.graph, n.info, tyBool)))
   if result.kind != nkIntLit:
     localError(c.config, n.info, errConstExprExpected)
-
-proc semGenericStmt(c: PContext, n: PNode): PNode
 proc semConceptBody(c: PContext, n: PNode): PNode
 
 include semtypes
@@ -553,18 +553,17 @@ proc pickCaseBranchIndex(caseExpr, matched: PNode): int =
   if endsWithElse:
     return caseExpr.len - 1
 
-proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, id: var IntSet): seq[PNode]
-proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, id: var IntSet): PNode
-proc defaultNodeField(c: PContext, a: PNode): PNode
-proc defaultNodeField(c: PContext, a: PNode, aTyp: PType): PNode
+proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, checkDefault: bool): seq[PNode]
+proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, checkDefault: bool): PNode
+proc defaultNodeField(c: PContext, a: PNode, checkDefault: bool): PNode
 
 const defaultFieldsSkipTypes = {tyGenericInst, tyAlias, tySink}
 
-proc defaultFieldsForTuple(c: PContext, recNode: PNode, id: var IntSet, hasDefault: var bool): seq[PNode] =
+proc defaultFieldsForTuple(c: PContext, recNode: PNode, hasDefault: var bool, checkDefault: bool): seq[PNode] =
   case recNode.kind
   of nkRecList:
     for field in recNode:
-      result.add defaultFieldsForTuple(c, field, id, hasDefault)
+      result.add defaultFieldsForTuple(c, field, hasDefault, checkDefault)
   of nkSym:
     let field = recNode.sym
     let recType = recNode.typ.skipTypes(defaultFieldsSkipTypes)
@@ -573,30 +572,30 @@ proc defaultFieldsForTuple(c: PContext, recNode: PNode, id: var IntSet, hasDefau
       result.add newTree(nkExprColonExpr, recNode, field.ast)
     else:
       if recType.kind in {tyObject, tyArray, tyTuple}:
-        let asgnExpr = defaultNodeField(c, recNode, recNode.typ, id)
+        let asgnExpr = defaultNodeField(c, recNode, recNode.typ, checkDefault)
         if asgnExpr != nil:
           hasDefault = true
-          asgnExpr.flags.incl nfUseDefaultField
+          asgnExpr.flags.incl nfSkipFieldChecking
           result.add newTree(nkExprColonExpr, recNode, asgnExpr)
           return
 
-      let asgnType = newType(tyTypeDesc, nextTypeId(c.idgen), recType.owner)
-      rawAddSon(asgnType, recType)
+      let asgnType = newType(tyTypeDesc, nextTypeId(c.idgen), recNode.typ.owner)
+      rawAddSon(asgnType, recNode.typ)
       let asgnExpr = newTree(nkCall,
                       newSymNode(getSysMagic(c.graph, recNode.info, "zeroDefault", mZeroDefault)),
                       newNodeIT(nkType, recNode.info, asgnType)
                     )
-      asgnExpr.flags.incl nfUseDefaultField
-      asgnExpr.typ = recType
+      asgnExpr.flags.incl nfSkipFieldChecking
+      asgnExpr.typ = recNode.typ
       result.add newTree(nkExprColonExpr, recNode, asgnExpr)
   else:
     doAssert false
 
-proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, id: var IntSet): seq[PNode] =
+proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, checkDefault: bool): seq[PNode] =
   case recNode.kind
   of nkRecList:
     for field in recNode:
-      result.add defaultFieldsForTheUninitialized(c, field, id)
+      result.add defaultFieldsForTheUninitialized(c, field, checkDefault)
   of nkRecCase:
     let discriminator = recNode[0]
     var selectedBranch: int
@@ -605,65 +604,60 @@ proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, id: var IntSe
       # None of the branches were explicitly selected by the user and no value
       # was given to the discrimator. We can assume that it will be initialized
       # to zero and this will select a particular branch as a result:
+      if checkDefault: # don't add defaults when checking whether a case branch has default fields
+        return
       defaultValue = newIntNode(nkIntLit#[c.graph]#, 0)
       defaultValue.typ = discriminator.typ
     selectedBranch = recNode.pickCaseBranchIndex defaultValue
-    defaultValue.flags.incl nfUseDefaultField
+    defaultValue.flags.incl nfSkipFieldChecking
     result.add newTree(nkExprColonExpr, discriminator, defaultValue)
-    result.add defaultFieldsForTheUninitialized(c, recNode[selectedBranch][^1], id)
+    result.add defaultFieldsForTheUninitialized(c, recNode[selectedBranch][^1], checkDefault)
   of nkSym:
     let field = recNode.sym
     let recType = recNode.typ.skipTypes(defaultFieldsSkipTypes)
     if field.ast != nil: #Try to use default value
       result.add newTree(nkExprColonExpr, recNode, field.ast)
     elif recType.kind in {tyObject, tyArray, tyTuple}:
-      let asgnExpr = defaultNodeField(c, recNode, recType, id)
+      let asgnExpr = defaultNodeField(c, recNode, recNode.typ, checkDefault)
       if asgnExpr != nil:
-        asgnExpr.typ = recType
-        asgnExpr.flags.incl nfUseDefaultField
+        asgnExpr.typ = recNode.typ
+        asgnExpr.flags.incl nfSkipFieldChecking
         result.add newTree(nkExprColonExpr, recNode, asgnExpr)
   else:
     doAssert false
 
-proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, id: var IntSet): PNode =
+proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, checkDefault: bool): PNode =
   let aTypSkip = aTyp.skipTypes(defaultFieldsSkipTypes)
   if aTypSkip.kind == tyObject:
-    if id.containsOrIncl(aTypSkip.id):
-      return
-    let child = defaultFieldsForTheUninitialized(c, aTypSkip.n, id)
+    let child = defaultFieldsForTheUninitialized(c, aTypSkip.n, checkDefault)
     if child.len > 0:
-      var asgnExpr = newTree(nkObjConstr, newNodeIT(nkType, a.info, aTypSkip))
-      asgnExpr.typ = aTypSkip
+      var asgnExpr = newTree(nkObjConstr, newNodeIT(nkType, a.info, aTyp))
+      asgnExpr.typ = aTyp
       asgnExpr.sons.add child
       result = semExpr(c, asgnExpr)
   elif aTypSkip.kind == tyArray:
-    let child = defaultNodeField(c, a, aTypSkip[1], id)
+    let child = defaultNodeField(c, a, aTypSkip[1], checkDefault)
 
     if child != nil:
       let node = newNode(nkIntLit)
       node.intVal = toInt64(lengthOrd(c.graph.config, aTypSkip))
-      result = semExpr(c, newTree(nkCall, newSymNode(getCompilerProc(c.graph, "nimArrayWith"), a.info),
-        semExprWithType(c, child),
-        node
-          ))
+      result = semExpr(c, newTree(nkCall, newSymNode(getSysSym(c.graph, a.info, "arrayWith"), a.info),
+              semExprWithType(c, child),
+              node
+                ))
       result.typ = aTyp
   elif aTypSkip.kind == tyTuple:
     var hasDefault = false
     if aTypSkip.n != nil:
-      let children = defaultFieldsForTuple(c, aTypSkip.n, id, hasDefault)
+      let children = defaultFieldsForTuple(c, aTypSkip.n, hasDefault, checkDefault)
       if hasDefault and children.len > 0:
         result = newNodeI(nkTupleConstr, a.info)
         result.typ = aTyp
         result.sons.add children
         result = semExpr(c, result)
 
-proc defaultNodeField(c: PContext, a: PNode, aTyp: PType): PNode =
-  var s = initIntSet()
-  defaultNodeField(c, a, aTyp, s)
-
-proc defaultNodeField(c: PContext, a: PNode): PNode =
-  var s = initIntSet()
-  result = defaultNodeField(c, a, a.typ, s)
+proc defaultNodeField(c: PContext, a: PNode, checkDefault: bool): PNode =
+  result = defaultNodeField(c, a, a.typ, checkDefault)
 
 include semtempl, semgnrc, semstmts, semexprs
 
@@ -677,41 +671,47 @@ proc addCodeForGenerics(c: PContext, n: PNode) =
         n.add prc.ast
   c.lastGenericIdx = c.generics.len
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
-  var c = newContext(graph, module)
-  c.idgen = idgen
-  c.enforceVoidContext = newType(tyTyped, nextTypeId(idgen), nil)
-  c.voidType = newType(tyVoid, nextTypeId(idgen), nil)
+proc preparePContext*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PContext =
+  result = newContext(graph, module)
+  result.idgen = idgen
+  result.enforceVoidContext = newType(tyTyped, nextTypeId(idgen), nil)
+  result.voidType = newType(tyVoid, nextTypeId(idgen), nil)
 
-  if c.p != nil: internalError(graph.config, module.info, "sem.myOpen")
-  c.semConstExpr = semConstExpr
-  c.semExpr = semExpr
-  c.semTryExpr = tryExpr
-  c.semTryConstExpr = tryConstExpr
-  c.computeRequiresInit = computeRequiresInit
-  c.semOperand = semOperand
-  c.semConstBoolExpr = semConstBoolExpr
-  c.semOverloadedCall = semOverloadedCall
-  c.semInferredLambda = semInferredLambda
-  c.semGenerateInstance = generateInstance
-  c.semTypeNode = semTypeNode
-  c.instTypeBoundOp = sigmatch.instTypeBoundOp
-  c.hasUnresolvedArgs = hasUnresolvedArgs
-  c.templInstCounter = new int
+  if result.p != nil: internalError(graph.config, module.info, "sem.preparePContext")
+  result.semConstExpr = semConstExpr
+  result.semExpr = semExpr
+  result.semExprWithType = semExprWithType
+  result.semTryExpr = tryExpr
+  result.semTryConstExpr = tryConstExpr
+  result.computeRequiresInit = computeRequiresInit
+  result.semOperand = semOperand
+  result.semConstBoolExpr = semConstBoolExpr
+  result.semOverloadedCall = semOverloadedCall
+  result.semInferredLambda = semInferredLambda
+  result.semGenerateInstance = generateInstance
+  result.semTypeNode = semTypeNode
+  result.instTypeBoundOp = sigmatch.instTypeBoundOp
+  result.hasUnresolvedArgs = hasUnresolvedArgs
+  result.templInstCounter = new int
 
-  pushProcCon(c, module)
-  pushOwner(c, c.module)
+  pushProcCon(result, module)
+  pushOwner(result, result.module)
 
-  c.moduleScope = openScope(c)
-  c.moduleScope.addSym(module) # a module knows itself
+  result.moduleScope = openScope(result)
+  result.moduleScope.addSym(module) # a module knows itself
 
   if sfSystemModule in module.flags:
     graph.systemModule = module
-  c.topLevelScope = openScope(c)
-  result = c
+  result.topLevelScope = openScope(result)
 
 proc isImportSystemStmt(g: ModuleGraph; n: PNode): bool =
   if g.systemModule == nil: return false
+  var n = n
+  if n.kind == nkStmtList:
+    for i in 0..<n.len-1:
+      if n[i].kind notin {nkCommentStmt, nkEmpty}:
+        n = n[i]
+        break
   case n.kind
   of nkImportStmt:
     for x in n:
@@ -776,8 +776,7 @@ proc recoverContext(c: PContext) =
   while getCurrOwner(c).kind != skModule: popOwner(c)
   while c.p != nil and c.p.owner.kind != skModule: c.p = c.p.next
 
-proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} =
-  var c = PContext(context)
+proc semWithPContext*(c: PContext, n: PNode): PNode =
   # no need for an expensive 'try' if we stop after the first error anyway:
   if c.config.errorMax <= 1:
     result = semStmtAndGenerateGenerics(c, n)
@@ -798,13 +797,13 @@ proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} =
       #if c.config.cmd == cmdIdeTools: findSuggest(c, n)
   storeRodNode(c, result)
 
+
 proc reportUnusedModules(c: PContext) =
   for i in 0..high(c.unusedImports):
     if sfUsed notin c.unusedImports[i][0].flags:
       message(c.config, c.unusedImports[i][1], warnUnusedImportX, c.unusedImports[i][0].name.s)
 
-proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
-  var c = PContext(context)
+proc closePContext*(graph: ModuleGraph; c: PContext, n: PNode): PNode =
   if c.config.cmd == cmdIdeTools and not c.suggestionsMade:
     suggestSentinel(c)
   closeScope(c)         # close module's scope
@@ -819,6 +818,3 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
   popOwner(c)
   popProcCon(c)
   sealRodFile(c)
-
-const semPass* = makePass(myOpen, myProcess, myClose,
-                          isFrontend = true)

compiler/semcall.nim

@@ -10,7 +10,8 @@
 ## This module implements semantic checking for calls.
 # included from sem.nim
 
-from algorithm import sort
+from std/algorithm import sort
+
 
 proc sameMethodDispatcher(a, b: PSym): bool =
   result = false
@@ -42,6 +43,7 @@ proc initCandidateSymbols(c: PContext, headSymbol: PNode,
                           best, alt: var TCandidate,
                           o: var TOverloadIter,
                           diagnostics: bool): seq[tuple[s: PSym, scope: int]] =
+  ## puts all overloads into a seq and prepares best+alt
   result = @[]
   var symx = initOverloadIter(o, c, headSymbol)
   while symx != nil:
@@ -63,36 +65,35 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
                        errors: var CandidateErrors,
                        diagnosticsFlag: bool,
                        errorsEnabled: bool, flags: TExprFlags) =
+  # `matches` may find new symbols, so keep track of count
+  var symCount = c.currentScope.symbols.counter
+
   var o: TOverloadIter
-  var sym = initOverloadIter(o, c, headSymbol)
-  var scope = o.lastOverloadScope
-  # Thanks to the lazy semchecking for operands, we need to check whether
-  # 'initCandidate' modifies the symbol table (via semExpr).
-  # This can occur in cases like 'init(a, 1, (var b = new(Type2); b))'
-  let counterInitial = c.currentScope.symbols.counter
-  var syms: seq[tuple[s: PSym, scope: int]]
-  var noSyms = true
-  var nextSymIndex = 0
-  while sym != nil:
-    if sym.kind in filter:
-      # Initialise 'best' and 'alt' with the first available symbol
-      initCandidate(c, best, sym, initialBinding, scope, diagnosticsFlag)
-      initCandidate(c, alt, sym, initialBinding, scope, diagnosticsFlag)
-      best.state = csNoMatch
-      break
-    else:
-      sym = nextOverloadIter(o, c, headSymbol)
-      scope = o.lastOverloadScope
-  var z: TCandidate
-  while sym != nil:
-    if sym.kind notin filter:
-      sym = nextOverloadIter(o, c, headSymbol)
-      scope = o.lastOverloadScope
-      continue
+  # https://github.com/nim-lang/Nim/issues/21272
+  # prevent mutation during iteration by storing them in a seq
+  # luckily `initCandidateSymbols` does just that
+  var syms = initCandidateSymbols(c, headSymbol, initialBinding, filter,
+                                  best, alt, o, diagnosticsFlag)
+  if len(syms) == 0:
+    return
+  # current overload being considered
+  var sym = syms[0].s
+  var scope = syms[0].scope
+
+  # starts at 1 because 0 is already done with setup, only needs checking
+  var nextSymIndex = 1
+  var z: TCandidate # current candidate
+  while true:
     determineType(c, sym)
     initCandidate(c, z, sym, initialBinding, scope, diagnosticsFlag)
-    if c.currentScope.symbols.counter == counterInitial or syms.len != 0:
+
+    # this is kinda backwards as without a check here the described
+    # problems in recalc would not happen, but instead it 100%
+    # does check forever in some cases
+    if c.currentScope.symbols.counter == symCount:
+      # may introduce new symbols with caveats described in recalc branch
       matches(c, n, orig, z)
+
       if z.state == csMatch:
         # little hack so that iterators are preferred over everything else:
         if sym.kind == skIterator:
@@ -112,22 +113,36 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
           firstMismatch: z.firstMismatch,
           diagnostics: z.diagnostics))
     else:
+      # this branch feels like a ticking timebomb
+      # one of two bad things could happen
+      # 1) new symbols are discovered but the loop ends before we recalc
+      # 2) new symbols are discovered and resemmed forever
+      # not 100% sure if these are possible though as they would rely
+      #  on somehow introducing a new overload during overload resolution
+
       # Symbol table has been modified. Restart and pre-calculate all syms
       # before any further candidate init and compare. SLOW, but rare case.
       syms = initCandidateSymbols(c, headSymbol, initialBinding, filter,
                                   best, alt, o, diagnosticsFlag)
-      noSyms = false
-    if noSyms:
-      sym = nextOverloadIter(o, c, headSymbol)
-      scope = o.lastOverloadScope
-    elif nextSymIndex < syms.len:
-      # rare case: retrieve the next pre-calculated symbol
-      sym = syms[nextSymIndex].s
-      scope = syms[nextSymIndex].scope
-      nextSymIndex += 1
-    else:
+
+      # reset counter because syms may be in a new order
+      symCount = c.currentScope.symbols.counter
+      nextSymIndex = 0
+
+      # just in case, should be impossible though
+      if syms.len == 0:
+        break
+
+    if nextSymIndex > high(syms):
+      # we have reached the end
       break
 
+    # advance to next sym
+    sym = syms[nextSymIndex].s
+    scope = syms[nextSymIndex].scope
+    inc(nextSymIndex)
+
+
 proc effectProblem(f, a: PType; result: var string; c: PContext) =
   if f.kind == tyProc and a.kind == tyProc:
     if tfThread in f.flags and tfThread notin a.flags:
@@ -192,7 +207,7 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
   # argument in order to remove plenty of candidates. This is
   # comparable to what C# does and C# is doing fine.
   var filterOnlyFirst = false
-  if optShowAllMismatches notin c.config.globalOptions:
+  if optShowAllMismatches notin c.config.globalOptions and verboseTypeMismatch in c.config.legacyFeatures:
     for err in errors:
       if err.firstMismatch.arg > 1:
         filterOnlyFirst = true
@@ -208,6 +223,10 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
     if filterOnlyFirst and err.firstMismatch.arg == 1:
       inc skipped
       continue
+
+    if verboseTypeMismatch notin c.config.legacyFeatures:
+      candidates.add "[" & $err.firstMismatch.arg & "] "
+
     if err.sym.kind in routineKinds and err.sym.ast != nil:
       candidates.add(renderTree(err.sym.ast,
             {renderNoBody, renderNoComments, renderNoPragmas}))
@@ -217,7 +236,7 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
     candidates.add("\n")
     let nArg = if err.firstMismatch.arg < n.len: n[err.firstMismatch.arg] else: nil
     let nameParam = if err.firstMismatch.formal != nil: err.firstMismatch.formal.name.s else: ""
-    if n.len > 1:
+    if n.len > 1 and verboseTypeMismatch in c.config.legacyFeatures:
       candidates.add("  first type mismatch at position: " & $err.firstMismatch.arg)
       # candidates.add "\n  reason: " & $err.firstMismatch.kind # for debugging
       case err.firstMismatch.kind
@@ -274,11 +293,28 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
 const
   errTypeMismatch = "type mismatch: got <"
   errButExpected = "but expected one of:"
+  errExpectedPosition = "Expected one of (first mismatch at [position]):"
   errUndeclaredField = "undeclared field: '$1'"
   errUndeclaredRoutine = "attempting to call undeclared routine: '$1'"
   errBadRoutine = "attempting to call routine: '$1'$2"
   errAmbiguousCallXYZ = "ambiguous call; both $1 and $2 match for: $3"
 
+proc describeParamList(c: PContext, n: PNode, startIdx = 1; prefer = preferName): string =
+  result = "Expression: " & $n
+  for i in startIdx..<n.len:
+    result.add "\n  [" & $i & "] " & renderTree(n[i]) & ": "
+    result.add describeArg(c, n, i, startIdx, prefer)
+  result.add "\n"
+
+template legacynotFoundError(c: PContext, n: PNode, errors: CandidateErrors) =
+  let (prefer, candidates) = presentFailedCandidates(c, n, errors)
+  var result = errTypeMismatch
+  result.add(describeArgs(c, n, 1, prefer))
+  result.add('>')
+  if candidates != "":
+    result.add("\n" & errButExpected & "\n" & candidates)
+  localError(c.config, n.info, result & "\nexpression: " & $n)
+
 proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) =
   # Gives a detailed error message; this is separated from semOverloadedCall,
   # as semOverloadedCall is already pretty slow (and we need this information
@@ -302,17 +338,19 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) =
     if n[0].kind in nkIdentKinds:
       let ident = considerQuotedIdent(c, n[0], n).s
       localError(c.config, n.info, errUndeclaredRoutine % ident)
-    else: 
+    else:
       localError(c.config, n.info, "expression '$1' cannot be called" % n[0].renderTree)
     return
 
-  let (prefer, candidates) = presentFailedCandidates(c, n, errors)
-  var result = errTypeMismatch
-  result.add(describeArgs(c, n, 1, prefer))
-  result.add('>')
-  if candidates != "":
-    result.add("\n" & errButExpected & "\n" & candidates)
-  localError(c.config, n.info, result & "\nexpression: " & $n)
+  if verboseTypeMismatch in c.config.legacyFeatures:
+    legacynotFoundError(c, n, errors)
+  else:
+    let (prefer, candidates) = presentFailedCandidates(c, n, errors)
+    var result = "type mismatch\n"
+    result.add describeParamList(c, n, 1, prefer)
+    if candidates != "":
+      result.add("\n" & errExpectedPosition & "\n" & candidates)
+    localError(c.config, n.info, result)
 
 proc bracketNotFoundError(c: PContext; n: PNode) =
   var errors: CandidateErrors = @[]
@@ -577,7 +615,7 @@ proc tryDeref(n: PNode): PNode =
   result.add n
 
 proc semOverloadedCall(c: PContext, n, nOrig: PNode,
-                       filter: TSymKinds, flags: TExprFlags): PNode {.nosinks.} =
+                       filter: TSymKinds, flags: TExprFlags): PNode =
   var errors: CandidateErrors = @[] # if efExplain in flags: @[] else: nil
   var r = resolveOverloads(c, n, nOrig, filter, flags, errors, efExplain in flags)
   if r.state == csMatch:
@@ -589,10 +627,13 @@ proc semOverloadedCall(c: PContext, n, nOrig: PNode,
               candidates)
     result = semResolvedCall(c, r, n, flags)
   else:
-    if efExplain notin flags:
+    if efDetermineType in flags and c.inGenericContext > 0 and c.matchedConcept == nil:
+      result = semGenericStmt(c, n)
+      result.typ = makeTypeFromExpr(c, result.copyTree)
+    elif efExplain notin flags:
       # repeat the overload resolution,
       # this time enabling all the diagnostic output (this should fail again)
-      discard semOverloadedCall(c, n, nOrig, filter, flags + {efExplain})
+      result = semOverloadedCall(c, n, nOrig, filter, flags + {efExplain})
     elif efNoUndeclared notin flags:
       notFoundError(c, n, errors)
 

compiler/semdata.nim

@@ -75,6 +75,7 @@ type
       # overload resolution.
     efNoDiagnostics,
     efTypeAllowed # typeAllowed will be called after
+    efWantNoDefaults
 
   TExprFlags* = set[TExprFlag]
 
@@ -125,6 +126,7 @@ type
     libs*: seq[PLib]           # all libs used by this module
     semConstExpr*: proc (c: PContext, n: PNode; expectedType: PType = nil): PNode {.nimcall.} # for the pragmas
     semExpr*: proc (c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType = nil): PNode {.nimcall.}
+    semExprWithType*: proc (c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType = nil): PNode {.nimcall.}
     semTryExpr*: proc (c: PContext, n: PNode, flags: TExprFlags = {}): PNode {.nimcall.}
     semTryConstExpr*: proc (c: PContext, n: PNode; expectedType: PType = nil): PNode {.nimcall.}
     computeRequiresInit*: proc (c: PContext, t: PType): bool {.nimcall.}
@@ -165,6 +167,8 @@ type
     lastTLineInfo*: TLineInfo
     sideEffects*: Table[int, seq[(TLineInfo, PSym)]] # symbol.id index
     inUncheckedAssignSection*: int
+    importModuleLookup*: Table[int, seq[int]] # (module.ident.id, [module.id])
+    skipTypes*: seq[PNode] # used to skip types between passes in type section. So far only used for inheritance and sets.
 
 template config*(c: PContext): ConfigRef = c.graph.config
 
@@ -429,7 +433,7 @@ proc makeTypeSymNode*(c: PContext, typ: PType, info: TLineInfo): PNode =
   incl typedesc.flags, tfCheckedForDestructor
   internalAssert(c.config, typ != nil)
   typedesc.addSonSkipIntLit(typ, c.idgen)
-  let sym = newSym(skType, c.cache.idAnon, nextSymId(c.idgen), getCurrOwner(c), info,
+  let sym = newSym(skType, c.cache.idAnon, c.idgen, getCurrOwner(c), info,
                    c.config.options).linkTo(typedesc)
   result = newSymNode(sym, info)
 

compiler/semexprs.nim

@@ -57,7 +57,14 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     # XXX tyGenericInst here?
     if result.typ.kind == tyProc and hasUnresolvedParams(result, {efOperand}):
       #and tfUnresolved in result.typ.flags:
-      localError(c.config, n.info, errProcHasNoConcreteType % n.renderTree)
+      let owner = result.typ.owner
+      let err =
+        # consistent error message with evaltempl/semMacroExpr
+        if owner != nil and owner.kind in {skTemplate, skMacro}:
+          errMissingGenericParamsForTemplate % n.renderTree
+        else:
+          errProcHasNoConcreteType % n.renderTree
+      localError(c.config, n.info, err)
     if result.typ.kind in {tyVar, tyLent}: result = newDeref(result)
   elif {efWantStmt, efAllowStmt} * flags != {}:
     result.typ = newTypeS(tyVoid, c)
@@ -109,7 +116,7 @@ proc ambiguousSymChoice(c: PContext, orig, n: PNode): PNode =
     result = n
 
 proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType = nil): PNode =
-  result = semExprCheck(c, n, flags, expectedType)
+  result = semExprCheck(c, n, flags-{efTypeAllowed}, expectedType)
   if result.typ == nil and efInTypeof in flags:
     result.typ = c.voidType
   elif (result.typ == nil or result.typ.kind == tyNone) and
@@ -123,6 +130,18 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType
   elif result.typ.kind == tyError:
     # associates the type error to the current owner
     result.typ = errorType(c)
+  elif efTypeAllowed in flags and result.typ.kind == tyProc and
+      hasUnresolvedParams(result, {}):
+    # mirrored with semOperand but only on efTypeAllowed
+    let owner = result.typ.owner
+    let err =
+      # consistent error message with evaltempl/semMacroExpr
+      if owner != nil and owner.kind in {skTemplate, skMacro}:
+        errMissingGenericParamsForTemplate % n.renderTree
+      else:
+        errProcHasNoConcreteType % n.renderTree
+    localError(c.config, n.info, err)
+    result.typ = errorType(c)
   else:
     if result.typ.kind in {tyVar, tyLent}: result = newDeref(result)
 
@@ -242,7 +261,7 @@ proc isCastable(c: PContext; dst, src: PType, info: TLineInfo): bool =
   if skipTypes(dst, abstractInst).kind == tyBuiltInTypeClass:
     return false
   let conf = c.config
-  if conf.selectedGC in {gcArc, gcOrc}:
+  if conf.selectedGC in {gcArc, gcOrc, gcAtomicArc}:
     let d = skipTypes(dst, abstractInst)
     let s = skipTypes(src, abstractInst)
     if d.kind == tyRef and s.kind == tyRef and s[0].isFinal != d[0].isFinal:
@@ -261,7 +280,7 @@ proc isCastable(c: PContext; dst, src: PType, info: TLineInfo): bool =
     return false
   elif srcSize < 0:
     return false
-  elif typeAllowed(dst, skParam, c) != nil:
+  elif typeAllowed(dst, skParam, c, {taIsCastable}) != nil:
     return false
   elif dst.kind == tyProc and dst.callConv == ccClosure:
     return src.kind == tyProc and src.callConv == ccClosure
@@ -269,11 +288,6 @@ proc isCastable(c: PContext; dst, src: PType, info: TLineInfo): bool =
     result = (dstSize >= srcSize) or
         (skipTypes(dst, abstractInst).kind in IntegralTypes) or
         (skipTypes(src, abstractInst-{tyTypeDesc}).kind in IntegralTypes)
-    if result and (dstSize > srcSize):
-      var warnMsg = "target type is larger than source type"
-      warnMsg.add("\n  target type: '$1' ($2)" % [$dst, if dstSize == 1: "1 byte" else: $dstSize & " bytes"])
-      warnMsg.add("\n  source type: '$1' ($2)" % [$src, if srcSize == 1: "1 byte" else: $srcSize & " bytes"])
-      message(conf, info, warnCastSizes, warnMsg)
   if result and src.kind == tyNil:
     return dst.size <= conf.target.ptrSize
 
@@ -302,7 +316,7 @@ proc semConv(c: PContext, n: PNode; expectedType: PType = nil): PNode =
   result = newNodeI(nkConv, n.info)
 
   var targetType = semTypeNode(c, n[0], nil)
-  case targetType.kind
+  case targetType.skipTypes({tyDistinct}).kind
   of tyTypeDesc:
     internalAssert c.config, targetType.len > 0
     if targetType.base.kind == tyNone:
@@ -319,6 +333,8 @@ proc semConv(c: PContext, n: PNode; expectedType: PType = nil): PNode =
       return evaluated
     else:
       targetType = targetType.base
+  of tyAnything, tyUntyped, tyTyped:
+    localError(c.config, n.info, "illegal type conversion to '$1'" % typeToString(targetType))
   else: discard
 
   maybeLiftType(targetType, c, n[0].info)
@@ -340,6 +356,9 @@ proc semConv(c: PContext, n: PNode; expectedType: PType = nil): PNode =
       targetType.skipTypes(abstractPtrs).kind == tyObject:
     localError(c.config, n.info, "object construction uses ':', not '='")
   var op = semExprWithType(c, n[1])
+  if op.kind == nkClosedSymChoice and op.len > 0 and
+      op[0].sym.kind == skEnumField: # resolves overloadedable enums
+    op = ambiguousSymChoice(c, n, op)
   if targetType.kind != tyGenericParam and targetType.isMetaType:
     let final = inferWithMetatype(c, targetType, op, true)
     result.add final
@@ -467,6 +486,7 @@ proc isOpImpl(c: PContext, n: PNode, flags: TExprFlags): PNode =
       res = t.kind == tyProc and
             t.callConv == ccClosure
     of "iterator":
+      # holdover from when `is iterator` didn't work
       let t = skipTypes(t1, abstractRange)
       res = t.kind == tyProc and
             t.callConv == ccClosure and
@@ -785,7 +805,7 @@ proc analyseIfAddressTaken(c: PContext, n: PNode, isOutParam: bool): PNode =
   else:
     result = newHiddenAddrTaken(c, n, isOutParam)
 
-proc analyseIfAddressTakenInCall(c: PContext, n: PNode) =
+proc analyseIfAddressTakenInCall(c: PContext, n: PNode, isConverter = false) =
   checkMinSonsLen(n, 1, c.config)
   const
     FakeVarParams = {mNew, mNewFinalize, mInc, ast.mDec, mIncl, mExcl,
@@ -793,10 +813,15 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) =
       mAppendSeqElem, mNewSeq, mReset, mShallowCopy, mDeepCopy, mMove,
       mWasMoved}
 
+  template checkIfConverterCalled(c: PContext, n: PNode) =
+    ## Checks if there is a converter call which wouldn't be checked otherwise
+    # Call can sometimes be wrapped in a deref
+    let node = if n.kind == nkHiddenDeref: n[0] else: n
+    if node.kind == nkHiddenCallConv:
+      analyseIfAddressTakenInCall(c, node, true)
   # get the real type of the callee
   # it may be a proc var with a generic alias type, so we skip over them
   var t = n[0].typ.skipTypes({tyGenericInst, tyAlias, tySink})
-
   if n[0].kind == nkSym and n[0].sym.magic in FakeVarParams:
     # BUGFIX: check for L-Value still needs to be done for the arguments!
     # note sometimes this is eval'ed twice so we check for nkHiddenAddr here:
@@ -811,6 +836,8 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) =
               discard "allow access within a cast(unsafeAssign) section"
             else:
               localError(c.config, it.info, errVarForOutParamNeededX % $it)
+        # Make sure to still check arguments for converters
+        c.checkIfConverterCalled(n[i])
     # bug #5113: disallow newSeq(result) where result is a 'var T':
     if n[0].sym.magic in {mNew, mNewFinalize, mNewSeq}:
       var arg = n[1] #.skipAddr
@@ -822,14 +849,13 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) =
     return
   for i in 1..<n.len:
     let n = if n.kind == nkHiddenDeref: n[0] else: n
-    if n[i].kind == nkHiddenCallConv:
-      # we need to recurse explicitly here as converters can create nested
-      # calls and then they wouldn't be analysed otherwise
-      analyseIfAddressTakenInCall(c, n[i])
+    c.checkIfConverterCalled(n[i])
     if i < t.len and
         skipTypes(t[i], abstractInst-{tyTypeDesc}).kind in {tyVar}:
-      if n[i].kind != nkHiddenAddr:
-        n[i] = analyseIfAddressTaken(c, n[i], isOutParam(skipTypes(t[i], abstractInst-{tyTypeDesc})))
+      # Converters wrap var parameters in nkHiddenAddr but they haven't been analysed yet.
+      # So we need to make sure we are checking them still when in a converter call
+      if n[i].kind != nkHiddenAddr or isConverter:
+        n[i] = analyseIfAddressTaken(c, n[i].skipAddr(), isOutParam(skipTypes(t[i], abstractInst-{tyTypeDesc})))
 
 include semmagic
 
@@ -940,7 +966,8 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
 
   if result != nil:
     if result[0].kind != nkSym:
-      internalError(c.config, "semOverloadedCallAnalyseEffects")
+      if not (efDetermineType in flags and c.inGenericContext > 0):
+        internalError(c.config, "semOverloadedCallAnalyseEffects")
       return
     let callee = result[0].sym
     case callee.kind
@@ -976,8 +1003,18 @@ proc setGenericParams(c: PContext, n: PNode) =
 proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags; expectedType: PType = nil): PNode =
   if efNoSemCheck notin flags and n.typ != nil and n.typ.kind == tyError:
     return errorNode(c, n)
+  if n.typ != nil and n.typ.kind == tyFromExpr and c.inGenericContext > 0:
+    return n
 
   result = n
+
+  when defined(nimsuggest):
+    if c.config.expandProgress:
+      if c.config.expandLevels == 0:
+        return n
+      else:
+        c.config.expandLevels -= 1
+
   let callee = result[0].sym
   case callee.kind
   of skMacro: result = semMacroExpr(c, result, orig, callee, flags, expectedType)
@@ -988,7 +1025,7 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags; expectedTy
     fixAbstractType(c, result)
     analyseIfAddressTakenInCall(c, result)
     if callee.magic != mNone:
-      result = magicsAfterOverloadResolution(c, result, flags)
+      result = magicsAfterOverloadResolution(c, result, flags, expectedType)
     when false:
       if result.typ != nil and
           not (result.typ.kind == tySequence and result.typ[0].kind == tyEmpty):
@@ -1023,7 +1060,7 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags; expectedType: PType
       if s != nil:
         setGenericParams(c, n[0])
         return semDirectOp(c, n, flags, expectedType)
-    elif isSymChoice(n[0]):
+    elif isSymChoice(n[0]) and nfDotField notin n.flags:
       # overloaded generic procs e.g. newSeq[int] can end up here
       return semDirectOp(c, n, flags, expectedType)
 
@@ -1212,7 +1249,7 @@ proc readTypeParameter(c: PContext, typ: PType,
             # This seems semantically correct and then we'll be able
             # to return the section symbol directly here
             let foundType = makeTypeDesc(c, def[2].typ)
-            return newSymNode(copySym(def[0].sym, nextSymId c.idgen).linkTo(foundType), info)
+            return newSymNode(copySym(def[0].sym, c.idgen).linkTo(foundType), info)
 
       of nkConstSection:
         for def in statement:
@@ -1237,11 +1274,12 @@ proc readTypeParameter(c: PContext, typ: PType,
             return c.graph.emptyNode
         else:
           let foundTyp = makeTypeDesc(c, rawTyp)
-          return newSymNode(copySym(tParam.sym, nextSymId c.idgen).linkTo(foundTyp), info)
+          return newSymNode(copySym(tParam.sym, c.idgen).linkTo(foundTyp), info)
 
   return nil
 
 proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
+  assert n.kind in nkIdentKinds + {nkDotExpr}
   let s = getGenSym(c, sym)
   case s.kind
   of skConst:
@@ -1276,9 +1314,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     else:
       result = newSymNode(s, n.info)
   of skMacro, skTemplate:
-    if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0 or
-       (n.kind notin nkCallKinds and s.requiredParams > 0) or
-       sfCustomPragma in sym.flags:
+    # check if we cannot use alias syntax (no required args or generic params)
+    if sfNoalias in s.flags:
       let info = getCallLineInfo(n)
       markUsed(c, info, s)
       onUse(info, s)
@@ -1336,6 +1373,12 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     markUsed(c, n.info, s)
     onUse(n.info, s)
     result = newSymNode(s, n.info)
+  of skModule:
+    # make sure type is None and not nil for discard checking
+    if efWantStmt in flags: s.typ = newTypeS(tyNone, c)
+    markUsed(c, n.info, s)
+    onUse(n.info, s)
+    result = newSymNode(s, n.info)
   else:
     let info = getCallLineInfo(n)
     #if efInCall notin flags:
@@ -1492,7 +1535,9 @@ proc builtinFieldAccess(c: PContext; n: PNode; flags: var TExprFlags): PNode =
     flags.incl efCannotBeDotCall
 
 proc dotTransformation(c: PContext, n: PNode): PNode =
-  if isSymChoice(n[1]):
+  if isSymChoice(n[1]) or
+      # generics usually leave field names as symchoices, but not types
+      (n[1].kind == nkSym and n[1].sym.kind == skType):
     result = newNodeI(nkDotCall, n.info)
     result.add n[1]
     result.add copyTree(n[0])
@@ -1565,9 +1610,8 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
     result.add(x[0])
     return
   checkMinSonsLen(n, 2, c.config)
-  # make sure we don't evaluate generic macros/templates
-  n[0] = semExprWithType(c, n[0],
-                              {efNoEvaluateGeneric})
+  # signal that generic parameters may be applied after
+  n[0] = semExprWithType(c, n[0], {efNoEvaluateGeneric})
   var arr = skipTypes(n[0].typ, {tyGenericInst, tyUserTypeClassInst, tyOwned,
                                       tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink})
   if arr.kind == tyStatic:
@@ -1617,7 +1661,10 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
         {tyInt..tyInt64}:
       let idx = getOrdValue(n[1])
       if idx >= 0 and idx < arr.len: n.typ = arr[toInt(idx)]
-      else: localError(c.config, n.info, "invalid index value for tuple subscript")
+      else:
+        localError(c.config, n.info,
+          "invalid index $1 in subscript for tuple of length $2" %
+            [$idx, $arr.len])
       result = n
     else:
       result = nil
@@ -1808,7 +1855,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
     result.add(n[1])
     return semExprNoType(c, result)
   of nkPar, nkTupleConstr:
-    if a.len >= 2:
+    if a.len >= 2 or a.kind == nkTupleConstr:
       # unfortunately we need to rewrite ``(x, y) = foo()`` already here so
       # that overloading of the assignment operator still works. Usually we
       # prefer to do these rewritings in transf.nim:
@@ -1888,6 +1935,9 @@ proc semReturn(c: PContext, n: PNode): PNode =
     localError(c.config, n.info, "'return' not allowed here")
 
 proc semProcBody(c: PContext, n: PNode; expectedType: PType = nil): PNode =
+  when defined(nimsuggest):
+    if c.graph.config.expandDone():
+      return n
   openScope(c)
   result = semExpr(c, n, expectedType = expectedType)
   if c.p.resultSym != nil and not isEmptyType(result.typ):
@@ -1945,6 +1995,8 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) =
             tupleConstr[i] = takeImplicitAddr(c, tupleConstr[i], e.kind == tyLent)
         else:
           localError(c.config, n[0].info, errXExpected, "tuple constructor")
+      elif e.kind == tyEmpty:
+        localError(c.config, n[0].info, errTypeExpected)
   else:
     when false:
       # XXX investigate what we really need here.
@@ -2059,7 +2111,7 @@ proc expectString(c: PContext, n: PNode): string =
     localError(c.config, n.info, errStringLiteralExpected)
 
 proc newAnonSym(c: PContext; kind: TSymKind, info: TLineInfo): PSym =
-  result = newSym(kind, c.cache.idAnon, nextSymId c.idgen, getCurrOwner(c), info)
+  result = newSym(kind, c.cache.idAnon, c.idgen, getCurrOwner(c), info)
 
 proc semExpandToAst(c: PContext, n: PNode): PNode =
   let macroCall = n[1]
@@ -2188,10 +2240,14 @@ proc semQuoteAst(c: PContext, n: PNode): PNode =
   if ids.len > 0:
     dummyTemplate[paramsPos] = newNodeI(nkFormalParams, n.info)
     dummyTemplate[paramsPos].add getSysSym(c.graph, n.info, "untyped").newSymNode # return type
-    ids.add getSysSym(c.graph, n.info, "untyped").newSymNode # params type
-    ids.add c.graph.emptyNode # no default value
-    dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids)
-
+    dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids[0], getSysSym(c.graph, n.info, "typed").newSymNode, c.graph.emptyNode)
+    for i in 1..<ids.len:
+      let exp = semExprWithType(c, quotes[i+1], {})
+      let typ = exp.typ
+      if tfTriggersCompileTime notin typ.flags and exp.kind == nkSym and exp.sym.kind notin routineKinds + {skType}:
+        dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids[i], newNodeIT(nkType, n.info, typ), c.graph.emptyNode)
+      else:
+        dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids[i], getSysSym(c.graph, n.info, "typed").newSymNode, c.graph.emptyNode)
   var tmpl = semTemplateDef(c, dummyTemplate)
   quotes[0] = tmpl[namePos]
   # This adds a call to newIdentNode("result") as the first argument to the template call
@@ -2532,34 +2588,39 @@ proc semSetConstr(c: PContext, n: PNode, expectedType: PType = nil): PNode =
     # only semantic checking for all elements, later type checking:
     var typ: PType = nil
     for i in 0..<n.len:
+      let doSetType = typ == nil
       if isRange(n[i]):
         checkSonsLen(n[i], 3, c.config)
         n[i][1] = semExprWithType(c, n[i][1], {efTypeAllowed}, expectedElementType)
         n[i][2] = semExprWithType(c, n[i][2], {efTypeAllowed}, expectedElementType)
-        if typ == nil:
+        if doSetType:
           typ = skipTypes(n[i][1].typ,
                           {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink})
-          if expectedElementType == nil:
-            expectedElementType = typ
         n[i].typ = n[i][2].typ # range node needs type too
       elif n[i].kind == nkRange:
         # already semchecked
-        if typ == nil:
+        if doSetType:
           typ = skipTypes(n[i][0].typ,
                           {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink})
-          if expectedElementType == nil:
-            expectedElementType = typ
       else:
         n[i] = semExprWithType(c, n[i], {efTypeAllowed}, expectedElementType)
-        if typ == nil:
+        if doSetType:
           typ = skipTypes(n[i].typ, {tyGenericInst, tyVar, tyLent, tyOrdinal, tyAlias, tySink})
-          if expectedElementType == nil:
-            expectedElementType = typ
-    if not isOrdinalType(typ, allowEnumWithHoles=true):
-      localError(c.config, n.info, errOrdinalTypeExpected % typeToString(typ, preferDesc))
-      typ = makeRangeType(c, 0, MaxSetElements-1, n.info)
-    elif lengthOrd(c.config, typ) > MaxSetElements:
-      typ = makeRangeType(c, 0, MaxSetElements-1, n.info)
+      if doSetType:
+        if not isOrdinalType(typ, allowEnumWithHoles=true):
+          localError(c.config, n.info, errOrdinalTypeExpected % typeToString(typ, preferDesc))
+          typ = makeRangeType(c, 0, MaxSetElements-1, n.info)
+        elif isIntLit(typ):
+          # set of int literal, use a default range smaller than the max range
+          typ = makeRangeType(c, 0, DefaultSetElements-1, n.info)
+        elif lengthOrd(c.config, typ) > MaxSetElements:
+          message(c.config, n.info, warnAboveMaxSizeSet, "type '" &
+            typeToString(typ, preferDesc) & "' is too big to be a `set` element, " &
+            "assuming a range of 0.." & $(MaxSetElements - 1) &
+            ", explicitly write this range to get rid of warning")
+          typ = makeRangeType(c, 0, MaxSetElements-1, n.info)
+        if expectedElementType == nil:
+          expectedElementType = typ
     addSonSkipIntLit(result.typ, typ, c.idgen)
     for i in 0..<n.len:
       var m: PNode
@@ -2823,7 +2884,7 @@ proc hoistParamsUsedInDefault(c: PContext, call, letSection, defExpr: var PNode)
     let paramPos = defExpr.sym.position + 1
 
     if call[paramPos].kind != nkSym:
-      let hoistedVarSym = newSym(skLet, getIdent(c.graph.cache, genPrefix), nextSymId c.idgen,
+      let hoistedVarSym = newSym(skLet, getIdent(c.graph.cache, genPrefix), c.idgen,
                                  c.p.owner, letSection.info, c.p.owner.options)
       hoistedVarSym.typ = call[paramPos].typ
 
@@ -2891,7 +2952,6 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType
     defer:
       if isCompilerDebug():
         echo ("<", c.config$n.info, n, ?.result.typ)
-
   template directLiteral(typeKind: TTypeKind) =
     if result.typ == nil:
       if expectedType != nil and (
@@ -2903,6 +2963,19 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType
         result.typ = getSysType(c.graph, n.info, typeKind)
 
   result = n
+  when defined(nimsuggest):
+    var expandStarted = false
+    if c.config.ideCmd == ideExpand and not c.config.expandProgress and
+        ((n.kind in {nkFuncDef, nkProcDef, nkIteratorDef, nkTemplateDef, nkMethodDef, nkConverterDef} and
+          n.info.exactEquals(c.config.expandPosition)) or
+         (n.kind in {nkCall, nkCommand} and
+          n[0].info.exactEquals(c.config.expandPosition))):
+      expandStarted = true
+      c.config.expandProgress = true
+      if c.config.expandLevels == 0:
+        c.config.expandNodeResult = $n
+        suggestQuit()
+
   if c.config.cmd == cmdIdeTools: suggestExpr(c, n)
   if nfSem in n.flags: return
   case n.kind
@@ -2924,6 +2997,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType
         else:
           {checkUndeclared, checkModule, checkAmbiguity, checkPureEnumFields}
       s = qualifiedLookUp(c, n, checks)
+      if s == nil:
+        return
     if c.matchedConcept == nil: semCaptureSym(s, c.p.owner)
     case s.kind
     of skProc, skFunc, skMethod, skConverter, skIterator:
@@ -3030,8 +3105,6 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType
     c.isAmbiguous = false
     var s = qualifiedLookUp(c, n[0], mode)
     if s != nil:
-      #if c.config.cmd == cmdNimfix and n[0].kind == nkDotExpr:
-      #  pretty.checkUse(n[0][1].info, s)
       case s.kind
       of skMacro, skTemplate:
         result = semDirectOp(c, n, flags, expectedType)
@@ -3209,7 +3282,9 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType
   of nkDefer:
     if c.currentScope == c.topLevelScope:
       localError(c.config, n.info, "defer statement not supported at top level")
+    openScope(c)
     n[0] = semExpr(c, n[0])
+    closeScope(c)
     if not n[0].typ.isEmptyType and not implicitlyDiscardable(n[0]):
       localError(c.config, n.info, "'defer' takes a 'void' expression")
     #localError(c.config, n.info, errGenerated, "'defer' not allowed in this context")
@@ -3230,3 +3305,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType
     localError(c.config, n.info, "invalid expression: " &
                renderTree(n, {renderNoComments}))
   if result != nil: incl(result.flags, nfSem)
+
+  when defined(nimsuggest):
+    if expandStarted:
+      c.config.expandNodeResult = $result
+      suggestQuit()

compiler/semfields.nim

@@ -109,7 +109,7 @@ proc semForFields(c: PContext, n: PNode, m: TMagic): PNode =
   var trueSymbol = systemModuleSym(c.graph, getIdent(c.cache, "true"))
   if trueSymbol == nil:
     localError(c.config, n.info, "system needs: 'true'")
-    trueSymbol = newSym(skUnknown, getIdent(c.cache, "true"), nextSymId c.idgen, getCurrOwner(c), n.info)
+    trueSymbol = newSym(skUnknown, getIdent(c.cache, "true"), c.idgen, getCurrOwner(c), n.info)
     trueSymbol.typ = getSysType(c.graph, n.info, tyBool)
 
   result[0] = newSymNode(trueSymbol, n.info)

compiler/semfold.nim

@@ -569,6 +569,8 @@ proc foldDefine(m, s: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
     else: result = copyTree(s.astdef) # unreachable
   else:
     result = copyTree(s.astdef)
+    if result != nil:
+      result.info = n.info
 
 proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode =
   result = nil
@@ -593,6 +595,8 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
         result = foldDefine(m, s, n, idgen, g)
       else:
         result = copyTree(s.astdef)
+        if result != nil:
+          result.info = n.info
     of skProc, skFunc, skMethod:
       result = n
     of skParam:
@@ -681,7 +685,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
       n[0] = a
   of nkBracket, nkCurly:
     result = copyNode(n)
-    for i, son in n.pairs:
+    for son in n.items:
       var a = getConstExpr(m, son, idgen, g)
       if a == nil: return nil
       result.add a
@@ -705,7 +709,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
     # tuple constructor
     result = copyNode(n)
     if (n.len > 0) and (n[0].kind == nkExprColonExpr):
-      for i, expr in n.pairs:
+      for expr in n.items:
         let exprNew = copyNode(expr) # nkExprColonExpr
         exprNew.add expr[0]
         let a = getConstExpr(m, expr[1], idgen, g)
@@ -713,7 +717,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
         exprNew.add a
         result.add exprNew
     else:
-      for i, expr in n.pairs:
+      for expr in n.items:
         let a = getConstExpr(m, expr, idgen, g)
         if a == nil: return nil
         result.add a

compiler/semgnrc.nim

@@ -50,13 +50,6 @@ proc semGenericStmtScope(c: PContext, n: PNode,
   result = semGenericStmt(c, n, flags, ctx)
   closeScope(c)
 
-template macroToExpand(s): untyped =
-  s.kind in {skMacro, skTemplate} and (s.typ.len == 1 or sfAllUntyped in s.flags)
-
-template macroToExpandSym(s): untyped =
-  sfCustomPragma notin s.flags and s.kind in {skMacro, skTemplate} and
-    (s.typ.len == 1) and not fromDotExpr
-
 template isMixedIn(sym): bool =
   let s = sym
   s.name.id in ctx.toMixin or (withinConcept in flags and
@@ -68,26 +61,32 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
                           fromDotExpr=false): PNode =
   semIdeForTemplateOrGenericCheck(c.config, n, ctx.cursorInBody)
   incl(s.flags, sfUsed)
+  template maybeDotChoice(c: PContext, n: PNode, s: PSym, fromDotExpr: bool) =
+    if fromDotExpr:
+      result = symChoice(c, n, s, scForceOpen)
+      if result.kind == nkOpenSymChoice and result.len == 1:
+        result.transitionSonsKind(nkClosedSymChoice)
+    else:
+      result = symChoice(c, n, s, scOpen)
   case s.kind
   of skUnknown:
     # Introduced in this pass! Leave it as an identifier.
     result = n
-  of skProc, skFunc, skMethod, skIterator, skConverter, skModule:
-    result = symChoice(c, n, s, scOpen)
-  of skTemplate:
-    if macroToExpandSym(s):
+  of skProc, skFunc, skMethod, skIterator, skConverter, skModule, skEnumField:
+    maybeDotChoice(c, n, s, fromDotExpr)
+  of skTemplate, skMacro:
+    # alias syntax, see semSym for skTemplate, skMacro
+    if sfNoalias notin s.flags and not fromDotExpr:
       onUse(n.info, s)
-      result = semTemplateExpr(c, n, s, {efNoSemCheck})
+      case s.kind
+      of skTemplate: result = semTemplateExpr(c, n, s, {efNoSemCheck})
+      of skMacro: result = semMacroExpr(c, n, n, s, {efNoSemCheck})
+      else: discard # unreachable
+      c.friendModules.add(s.owner.getModule)
       result = semGenericStmt(c, result, {}, ctx)
+      discard c.friendModules.pop()
     else:
-      result = symChoice(c, n, s, scOpen)
-  of skMacro:
-    if macroToExpandSym(s):
-      onUse(n.info, s)
-      result = semMacroExpr(c, n, n, s, {efNoSemCheck})
-      result = semGenericStmt(c, result, {}, ctx)
-    else:
-      result = symChoice(c, n, s, scOpen)
+      maybeDotChoice(c, n, s, fromDotExpr)
   of skGenericParam:
     if s.typ != nil and s.typ.kind == tyStatic:
       if s.typ.n != nil:
@@ -107,8 +106,6 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
     else:
       result = n
     onUse(n.info, s)
-  of skEnumField:
-    result = symChoice(c, n, s, scOpen)
   else:
     result = newSymNode(s, n.info)
     onUse(n.info, s)
@@ -118,7 +115,7 @@ proc lookup(c: PContext, n: PNode, flags: TSemGenericFlags,
   result = n
   let ident = considerQuotedIdent(c, n)
   var amb = false
-  var s = searchInScopes(c, ident, amb).skipAlias(n, c.config)
+  var s = searchInScopes(c, ident, amb)
   if s == nil:
     s = strTableGet(c.pureEnumFields, ident)
     #if s != nil and contains(c.ambiguousSymbols, s.id):
@@ -141,7 +138,8 @@ proc newDot(n, b: PNode): PNode =
   result.add(b)
 
 proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
-                 ctx: var GenericCtx; isMacro: var bool): PNode =
+                 ctx: var GenericCtx; isMacro: var bool;
+                 inCall = false): PNode =
   assert n.kind == nkDotExpr
   semIdeForTemplateOrGenericCheck(c.config, n, ctx.cursorInBody)
 
@@ -155,22 +153,22 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
     result = n
     let n = n[1]
     let ident = considerQuotedIdent(c, n)
-    var candidates = searchInScopesFilterBy(c, ident, routineKinds) # .skipAlias(n, c.config)
+    # could be type conversion if like a.T and not a.T()
+    let symKinds = if inCall: routineKinds else: routineKinds+{skType}
+    var candidates = searchInScopesFilterBy(c, ident, symKinds)
     if candidates.len > 0:
       let s = candidates[0] # XXX take into account the other candidates!
       isMacro = s.kind in {skTemplate, skMacro}
       if withinBind in flags or s.id in ctx.toBind:
-        result = newDot(result, symChoice(c, n, s, scClosed))
+        if s.kind == skType: # don't put types in sym choice
+          result = newDot(result, semGenericStmtSymbol(c, n, s, ctx, flags, fromDotExpr=true))
+        else:
+          result = newDot(result, symChoice(c, n, s, scClosed))
       elif s.isMixedIn:
         result = newDot(result, symChoice(c, n, s, scForceOpen))
       else:
         let syms = semGenericStmtSymbol(c, n, s, ctx, flags, fromDotExpr=true)
-        if syms.kind == nkSym:
-          let choice = symChoice(c, n, s, scForceOpen)
-          choice.transitionSonsKind(nkClosedSymChoice)
-          result = newDot(result, choice)
-        else:
-          result = newDot(result, syms)
+        result = newDot(result, syms)
 
 proc addTempDecl(c: PContext; n: PNode; kind: TSymKind) =
   let s = newSymS(skUnknown, getIdentNode(c, n), c)
@@ -241,20 +239,17 @@ proc semGenericStmt(c: PContext, n: PNode,
                         else: scOpen
       let sc = symChoice(c, fn, s, whichChoice)
       case s.kind
-      of skMacro:
-        if macroToExpand(s) and sc.safeLen <= 1:
+      of skMacro, skTemplate:
+        # unambiguous macros/templates are expanded if all params are untyped
+        if sfAllUntyped in s.flags and sc.safeLen <= 1:
           onUse(fn.info, s)
-          result = semMacroExpr(c, n, n, s, {efNoSemCheck})
-          result = semGenericStmt(c, result, flags, ctx)
-        else:
-          n[0] = sc
-          result = n
-        mixinContext = true
-      of skTemplate:
-        if macroToExpand(s) and sc.safeLen <= 1:
-          onUse(fn.info, s)
-          result = semTemplateExpr(c, n, s, {efNoSemCheck})
+          case s.kind
+          of skMacro: result = semMacroExpr(c, n, n, s, {efNoSemCheck})
+          of skTemplate: result = semTemplateExpr(c, n, s, {efNoSemCheck})
+          else: discard # unreachable
+          c.friendModules.add(s.owner.getModule)
           result = semGenericStmt(c, result, flags, ctx)
+          discard c.friendModules.pop()
         else:
           n[0] = sc
           result = n
@@ -288,7 +283,7 @@ proc semGenericStmt(c: PContext, n: PNode,
         onUse(fn.info, s)
         first = 1
     elif fn.kind == nkDotExpr:
-      result[0] = fuzzyLookup(c, fn, flags, ctx, mixinContext)
+      result[0] = fuzzyLookup(c, fn, flags, ctx, mixinContext, inCall = true)
       first = 1
     # Consider 'when declared(globalsSlot): ThreadVarSetValue(globalsSlot, ...)'
     # in threads.nim: the subtle preprocessing here binds 'globalsSlot' which
@@ -476,7 +471,7 @@ proc semGenericStmt(c: PContext, n: PNode,
                                               flags, ctx)
     if n[paramsPos].kind != nkEmpty:
       if n[paramsPos][0].kind != nkEmpty:
-        addPrelimDecl(c, newSym(skUnknown, getIdent(c.cache, "result"), nextSymId c.idgen, nil, n.info))
+        addPrelimDecl(c, newSym(skUnknown, getIdent(c.cache, "result"), c.idgen, nil, n.info))
       n[paramsPos] = semGenericStmt(c, n[paramsPos], flags, ctx)
     n[pragmasPos] = semGenericStmt(c, n[pragmasPos], flags, ctx)
     var body: PNode
@@ -493,6 +488,20 @@ proc semGenericStmt(c: PContext, n: PNode,
   of nkExprColonExpr, nkExprEqExpr:
     checkMinSonsLen(n, 2, c.config)
     result[1] = semGenericStmt(c, n[1], flags, ctx)
+  of nkObjConstr:
+    for i in 0..<n.len:
+      result[i] = semGenericStmt(c, n[i], flags, ctx)
+    if result[0].kind == nkSym:
+      let fmoduleId = getModule(result[0].sym).id
+      var isVisable = false
+      for module in c.friendModules:
+        if module.id == fmoduleId:
+          isVisable = true
+          break
+      if isVisable:
+        for i in 1..<result.len:
+          if result[i].kind == nkExprColonExpr:
+            result[i][1].flags.incl nfSkipFieldChecking
   else:
     for i in 0..<n.len:
       result[i] = semGenericStmt(c, n[i], flags, ctx)

compiler/seminst.nim

@@ -40,12 +40,12 @@ const
 
 iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable): PSym =
   internalAssert c.config, n.kind == nkGenericParams
-  for i, a in n.pairs:
+  for a in n.items:
     internalAssert c.config, a.kind == nkSym
     var q = a.sym
     if q.typ.kind in {tyTypeDesc, tyGenericParam, tyStatic, tyConcept}+tyTypeClasses:
       let symKind = if q.typ.kind == tyStatic: skConst else: skType
-      var s = newSym(symKind, q.name, nextSymId(c.idgen), getCurrOwner(c), q.info)
+      var s = newSym(symKind, q.name, c.idgen, getCurrOwner(c), q.info)
       s.flags.incl {sfUsed, sfFromGeneric}
       var t = PType(idTableGet(pt, q.typ))
       if t == nil:
@@ -100,7 +100,7 @@ proc freshGenSyms(c: PContext; n: PNode, owner, orig: PSym, symMap: var TIdTable
       n.sym = x
     elif s.owner == nil or s.owner.kind == skPackage:
       #echo "copied this ", s.name.s
-      x = copySym(s, nextSymId c.idgen)
+      x = copySym(s, c.idgen)
       x.owner = owner
       idTablePut(symMap, s, x)
       n.sym = x
@@ -127,11 +127,18 @@ proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
         if sfGenSym in param.flags:
           idTablePut(symMap, params[i].sym, result.typ.n[param.position+1].sym)
     freshGenSyms(c, b, result, orig, symMap)
-    
-    if sfBorrow notin orig.flags: 
+
+    if sfBorrow notin orig.flags:
       # We do not want to generate a body for generic borrowed procs.
       # As body is a sym to the borrowed proc.
-      b = semProcBody(c, b)
+      let resultType = # todo probably refactor it into a function
+        if result.kind == skMacro:
+          sysTypeFromName(c.graph, n.info, "NimNode")
+        elif not isInlineIterator(result.typ):
+          result.typ[0]
+        else:
+          nil
+      b = semProcBody(c, b, resultType)
     result.ast[bodyPos] = hloBody(c, b)
     excl(result.flags, sfForward)
     trackProc(c, result, result.ast[bodyPos])
@@ -186,7 +193,7 @@ proc instGenericContainer(c: PContext, info: TLineInfo, header: PType,
     var param: PSym
 
     template paramSym(kind): untyped =
-      newSym(kind, genParam.sym.name, nextSymId c.idgen, genericTyp.sym, genParam.sym.info)
+      newSym(kind, genParam.sym.name, c.idgen, genericTyp.sym, genParam.sym.info)
 
     if genParam.kind == tyStatic:
       param = paramSym skConst
@@ -256,7 +263,7 @@ proc instantiateProcType(c: PContext, pt: TIdTable,
 
     internalAssert c.config, originalParams[i].kind == nkSym
     let oldParam = originalParams[i].sym
-    let param = copySym(oldParam, nextSymId c.idgen)
+    let param = copySym(oldParam, c.idgen)
     param.owner = prc
     param.typ = result[i]
 
@@ -314,7 +321,7 @@ proc fillMixinScope(c: PContext) =
     p = p.next
 
 proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
-                      info: TLineInfo): PSym {.nosinks.} =
+                      info: TLineInfo): PSym =
   ## Generates a new instance of a generic procedure.
   ## The `pt` parameter is a type-unsafe mapping table used to link generic
   ## parameters to their concrete types within the generic instance.
@@ -333,7 +340,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   c.matchedConcept = nil
   let oldScope = c.currentScope
   while not isTopLevel(c): c.currentScope = c.currentScope.parent
-  result = copySym(fn, nextSymId c.idgen)
+  result = copySym(fn, c.idgen)
   incl(result.flags, sfFromGeneric)
   result.owner = fn
   result.ast = n
@@ -345,7 +352,9 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
 
   openScope(c)
   let gp = n[genericParamsPos]
-  internalAssert c.config, gp.kind == nkGenericParams
+  if gp.kind != nkGenericParams:
+    # bug #22137
+    globalError(c.config, info, "generic instantiation too nested")
   n[namePos] = newSymNode(result)
   pushInfoContext(c.config, info, fn.detailedInfo)
   var entry = TInstantiation.new
@@ -381,8 +390,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
       pragma(c, result, n[pragmasPos], allRoutinePragmas)
     if isNil(n[bodyPos]):
       n[bodyPos] = copyTree(getBody(c.graph, fn))
-    if c.inGenericContext == 0:
-      instantiateBody(c, n, fn.typ.n, result, fn)
+    instantiateBody(c, n, fn.typ.n, result, fn)
     sideEffectsCheck(c, result)
     if result.magic notin {mSlice, mTypeOf}:
       # 'toOpenArray' is special and it is allowed to return 'openArray':